refactor: reactions grouping and other fixes

packages/app/src/Cache/EventInteractionCache.ts

@@ -30,7 +30,6 @@ export class EventInteractionCache extends FeedCache<EventInteraction> {
       });
       await this.bulkSet(toImport);
 
-      console.debug(`Imported dumb-zap-cache events: `, toImport.length);
       window.localStorage.removeItem("zap-cache");
     }
     await this.buffer([...this.onTable]);

packages/app/src/Cache/FollowsFeed.ts

@@ -1,6 +1,5 @@
 import { unixNow, unixNowMs } from "@snort/shared";
 import { EventKind, RequestBuilder, SystemInterface, TaggedNostrEvent } from "@snort/system";
-import debug from "debug";
 
 import { db } from "@/Db";
 import { Day, Hour } from "@/Utils/Const";
@@ -68,8 +67,7 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
     const oldest = await this.table?.orderBy("created_at").first();
     this.#oldest = oldest?.created_at;
     this.emit("change", latest?.map(a => this.key(a)) ?? []);
-
+    this.log(`Loaded %d/%d in %d ms`, latest?.length ?? 0, keys.length, (unixNowMs() - start).toLocaleString());
-    debug(this.name)(`Loaded %d/%d in %d ms`, latest?.length ?? 0, keys.length, (unixNowMs() - start).toLocaleString());
   }
 
   async loadMore(system: SystemInterface, session: LoginSession, before: number) {
@@ -132,7 +130,7 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
       const allKeys = new Set(everything?.map(a => a.pubkey));
       const missingKeys = keys.filter(a => !allKeys.has(a));
       await this.backFill(system, missingKeys);
-      debug(this.name)(`Backfilled %d keys in %d ms`, missingKeys.length, (unixNowMs() - start).toLocaleString());
+      this.log(`Backfilled %d keys in %d ms`, missingKeys.length, (unixNowMs() - start).toLocaleString());
     }
   }
 }

packages/app/src/Components/Event/LongFormText.tsx

@@ -32,13 +32,8 @@ export function LongFormText(props: LongFormTextProps) {
   const [reading, setReading] = useState(false);
   const [showMore, setShowMore] = useState(false);
   const ref = useRef<HTMLDivElement>(null);
-  const related = useReactions(
+  const related = useReactions("note:reactions", [NostrLink.fromEvent(props.ev)], undefined, false);
-    NostrLink.fromEvent(props.ev).id + "related",
+  const { reactions, reposts, zaps } = useEventReactions(NostrLink.fromEvent(props.ev), related);
-    [NostrLink.fromEvent(props.ev)],
-    undefined,
-    false,
-  );
-  const { reactions, reposts, zaps } = useEventReactions(NostrLink.fromEvent(props.ev), related.data ?? []);
 
   function previewText() {
     return (

packages/app/src/Components/Event/NostrFileHeader.tsx

@@ -10,8 +10,8 @@ import { findTag } from "@/Utils";
 export default function NostrFileHeader({ link }: { link: NostrLink }) {
   const ev = useEventFeed(link);
 
-  if (!ev.data) return <PageSpinner />;
+  if (!ev) return <PageSpinner />;
-  return <NostrFileElement ev={ev.data} />;
+  return <NostrFileElement ev={ev} />;
 }
 
 export function NostrFileElement({ ev }: { ev: NostrEvent }) {

packages/app/src/Components/Event/Note/NoteFooter.tsx

@@ -36,8 +36,8 @@ export default function NoteFooter(props: NoteFooterProps) {
   const link = useMemo(() => NostrLink.fromEvent(ev), [ev.id]);
   const ids = useMemo(() => [link], [link]);
 
-  const related = useReactions(link.id + "related", ids, undefined, false);
+  const related = useReactions("note:reactions", ids, undefined, false);
-  const { reactions, zaps, reposts } = useEventReactions(link, related.data ?? []);
+  const { reactions, zaps, reposts } = useEventReactions(link, related);
   const { positive } = reactions;
 
   const { formatMessage } = useIntl();

packages/app/src/Components/Event/Note/NoteQuote.tsx

@@ -11,11 +11,11 @@ const options = {
 
 export default function NoteQuote({ link, depth }: { link: NostrLink; depth?: number }) {
   const ev = useEventFeed(link);
-  if (!ev.data)
+  if (!ev)
     return (
       <div className="note-quote flex items-center justify-center h-[110px]">
         <PageSpinner />
       </div>
     );
-  return <Note data={ev.data} className="note-quote" depth={(depth ?? 0) + 1} options={options} />;
+  return <Note data={ev} className="note-quote" depth={(depth ?? 0) + 1} options={options} />;
 }

packages/app/src/Components/Event/Note/ReactionsModal.tsx

@@ -26,11 +26,11 @@ const ReactionsModal = ({ show, setShow, event }: ReactionsModalProps) => {
 
   const link = NostrLink.fromEvent(event);
 
-  const related = useReactions(link.id + "related", [link], undefined, false);
+  const related = useReactions("note:reactions", [link], undefined, false);
-  const { reactions, zaps, reposts } = useEventReactions(link, related.data ?? []);
+  const { reactions, zaps, reposts } = useEventReactions(link, related);
   const { positive, negative } = reactions;
 
-  const sortEvents = events =>
+  const sortEvents = (events: Array<TaggedNostrEvent>) =>
     events.sort(
       (a, b) => socialGraphInstance.getFollowDistance(a.pubkey) - socialGraphInstance.getFollowDistance(b.pubkey),
     );

packages/app/src/Components/Feed/Articles.tsx

@@ -16,7 +16,7 @@ export default function Articles() {
 
   return (
     <>
-      {orderDescending(data.data ?? []).map(a => (
+      {orderDescending(data).map(a => (
         <Note
           data={a}
           key={a.id}

packages/app/src/Components/Feed/Generic.tsx

@@ -1,4 +1,4 @@
-import { NostrLink, NoteCollection, ReqFilter, RequestBuilder } from "@snort/system";
+import { NostrLink, ReqFilter, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -6,7 +6,6 @@ import { TimelineRenderer } from "@/Components/Feed/TimelineRenderer";
 
 export function GenericFeed({ link }: { link: NostrLink }) {
   const sub = useMemo(() => {
-    console.debug(link);
     const sub = new RequestBuilder("generic");
     sub.withOptions({ leaveOpen: true });
     const reqs = JSON.parse(link.id) as Array<ReqFilter>;
@@ -17,11 +16,11 @@ export function GenericFeed({ link }: { link: NostrLink }) {
     return sub;
   }, [link]);
 
-  const evs = useRequestBuilder(NoteCollection, sub);
+  const evs = useRequestBuilder(sub);
 
   return (
     <TimelineRenderer
-      frags={[{ events: evs.data ?? [], refTime: 0 }]}
+      frags={[{ events: evs, refTime: 0 }]}
       latest={[]}
       showLatest={() => {
         //nothing

packages/app/src/Components/Feed/TimelineFollows.tsx

@@ -70,7 +70,7 @@ const TimelineFollows = (props: TimelineFollowsProps) => {
   };
   const mixinFiltered = useMemo(() => {
     const mainFeedIds = new Set(mainFeed.map(a => a.id));
-    return (mixin.data.data ?? [])
+    return (mixin.data ?? [])
       .filter(a => !mainFeedIds.has(a.id) && postsOnly(a) && !isEventMuted(a))
       .filter(a => a.tags.filter(a => a[0] === "t").length < 5)
       .filter(a => !oldest || a.created_at >= oldest)

packages/app/src/Components/SearchBox/SearchBox.tsx

@@ -38,7 +38,7 @@ export default function SearchBox() {
   const subject: TimelineSubject = {
     type: "profile_keyword",
     discriminator: search,
-    items: [search],
+    items: search ? [search] : [],
     relay: undefined,
     streams: false,
   };

packages/app/src/Components/SendSats/SendSats.tsx

@@ -84,10 +84,8 @@ export default function SendSats(props: SendSatsProps) {
   useEffect(() => {
     if (props.targets && props.show) {
       try {
-        console.debug("loading zapper");
         const zapper = new Zapper(system, publisher);
         zapper.load(props.targets).then(() => {
-          console.debug(zapper);
           setZapper(zapper);
         });
       } catch (e) {

packages/app/src/Components/Trending/TrendingPosts.tsx

@@ -31,7 +31,7 @@ export default function TrendingNotes({ count = Infinity, small = false }: { cou
     return removeUndefined(
       data.notes.map(a => {
         const ev = a.event;
-        if (!System.Optimizer.schnorrVerify(ev)) {
+        if (!System.optimizer.schnorrVerify(ev)) {
           console.error(`Event with invalid sig\n\n${ev}\n\nfrom ${trendingNotesUrl}`);
           return;
         }

packages/app/src/Components/User/ProfileLink.tsx

@@ -18,7 +18,8 @@ export function ProfileLink({
   children?: ReactNode;
 } & Omit<LinkProps, "to">) {
   const system = useContext(SnortContext);
-  const relays = system.relayCache.getFromCache(pubkey)
+  const relays = system.relayCache
+    .getFromCache(pubkey)
     ?.relays?.filter(a => a.settings.write)
     ?.map(a => a.url);
 

packages/app/src/Feed/BadgesFeed.ts

@@ -20,11 +20,8 @@ export default function useProfileBadges(pubkey?: HexKey) {
   const profileBadges = useRequestBuilder(sub);
 
   const profile = useMemo(() => {
-    if (profileBadges.data) {
+    if (profileBadges) {
-      return chunks(
+      return chunks(profileBadges[0]?.tags.filter(t => t[0] === "a" || t[0] === "e"), 2).reduce((acc, [a, e]) => {
-        profileBadges.data[0].tags.filter(t => t[0] === "a" || t[0] === "e"),
-        2,
-      ).reduce((acc, [a, e]) => {
         return {
           ...acc,
           [e[1]]: a[1],
@@ -60,8 +57,8 @@ export default function useProfileBadges(pubkey?: HexKey) {
   const awards = useRequestBuilder(awardsSub);
 
   const result = useMemo(() => {
-    if (awards.data) {
+    if (awards) {
-      return awards.data
+      return awards
         .map((award, _, arr) => {
           const [, pubkey, d] =
             award.tags

packages/app/src/Feed/FollowersFeed.ts

@@ -13,7 +13,7 @@ export default function useFollowersFeed(pubkey?: HexKey) {
   const followersFeed = useRequestBuilder(sub);
 
   const followers = useMemo(() => {
-    const contactLists = followersFeed.data?.filter(
+    const contactLists = followersFeed?.filter(
       a => a.kind === EventKind.ContactList && a.tags.some(b => b[0] === "p" && b[1] === pubkey),
     );
     return [...new Set(contactLists?.map(a => a.pubkey))].sort((a, b) => {

packages/app/src/Feed/FollowsFeed.ts

@@ -21,7 +21,7 @@ export default function useFollowsFeed(pubkey?: HexKey) {
       return follows.item;
     }
 
-    return getFollowing(contactFeed.data ?? [], pubkey);
+    return getFollowing(contactFeed ?? [], pubkey);
   }, [contactFeed, follows, pubkey]);
 }
 

packages/app/src/Feed/LoginFeed.ts

@@ -100,8 +100,8 @@ export default function useLoginFeed() {
 
   // update relays and follow lists
   useEffect(() => {
-    if (loginFeed.data) {
+    if (loginFeed) {
-      const contactList = getNewest(loginFeed.data.filter(a => a.kind === EventKind.ContactList));
+      const contactList = getNewest(loginFeed.filter(a => a.kind === EventKind.ContactList));
       if (contactList) {
         const pTags = contactList.tags.filter(a => a[0] === "p").map(a => a[1]);
         setFollows(login.id, pTags, contactList.created_at * 1000);
@@ -109,17 +109,17 @@ export default function useLoginFeed() {
         FollowsFeed.backFillIfMissing(system, pTags);
       }
 
-      const relays = getNewest(loginFeed.data.filter(a => a.kind === EventKind.Relays));
+      const relays = getNewest(loginFeed.filter(a => a.kind === EventKind.Relays));
       if (relays) {
         const parsedRelays = parseRelayTags(relays.tags.filter(a => a[0] === "r")).map(a => [a.url, a.settings]);
         setRelays(login, Object.fromEntries(parsedRelays), relays.created_at * 1000);
       }
 
-      Nip4Chats.onEvent(loginFeed.data);
+      Nip4Chats.onEvent(loginFeed);
-      Nip28Chats.onEvent(loginFeed.data);
+      Nip28Chats.onEvent(loginFeed);
 
       if (publisher) {
-        const subs = loginFeed.data.filter(
+        const subs = loginFeed.filter(
           a => a.kind === EventKind.SnortSubscriptions && a.pubkey === bech32ToHex(SnortPubKey),
         );
         Promise.all(
@@ -135,7 +135,7 @@ export default function useLoginFeed() {
           }),
         ).then(a => addSubscription(login, ...a.filter(a => a !== undefined).map(unwrap)));
 
-        const appData = getNewest(loginFeed.data.filter(a => a.kind === EventKind.AppData));
+        const appData = getNewest(loginFeed.filter(a => a.kind === EventKind.AppData));
         if (appData) {
           publisher.decryptGeneric(appData.content, appData.pubkey).then(d => {
             setAppData(login, JSON.parse(d) as SnortAppData, appData.created_at * 1000);
@@ -200,20 +200,20 @@ export default function useLoginFeed() {
   }
 
   useEffect(() => {
-    if (loginFeed.data) {
+    if (loginFeed) {
-      const mutedFeed = loginFeed.data.filter(a => a.kind === EventKind.MuteList);
+      const mutedFeed = loginFeed.filter(a => a.kind === EventKind.MuteList);
       handleMutedFeed(mutedFeed);
 
-      const pinnedFeed = loginFeed.data.filter(a => a.kind === EventKind.PinList);
+      const pinnedFeed = loginFeed.filter(a => a.kind === EventKind.PinList);
       handlePinnedFeed(pinnedFeed);
 
-      const tagsFeed = loginFeed.data.filter(a => a.kind === EventKind.InterestsList);
+      const tagsFeed = loginFeed.filter(a => a.kind === EventKind.InterestsList);
       handleTagFeed(tagsFeed);
 
-      const bookmarkFeed = loginFeed.data.filter(a => a.kind === EventKind.BookmarksList);
+      const bookmarkFeed = loginFeed.filter(a => a.kind === EventKind.BookmarksList);
       handleBookmarkFeed(bookmarkFeed);
 
-      const publicChatsFeed = loginFeed.data.filter(a => a.kind === EventKind.PublicChatsList);
+      const publicChatsFeed = loginFeed.filter(a => a.kind === EventKind.PublicChatsList);
       handlePublicChatsListFeed(publicChatsFeed);
     }
   }, [loginFeed]);

packages/app/src/Feed/RelaysFeed.tsx

@@ -11,5 +11,5 @@ export default function useRelaysFeed(pubkey?: HexKey) {
   }, [pubkey]);
 
   const relays = useRequestBuilder(sub);
-  return parseRelayTags(relays.data?.[0].tags.filter(a => a[0] === "r") ?? []);
+  return parseRelayTags(relays[0]?.tags.filter(a => a[0] === "r") ?? []);
 }

packages/app/src/Feed/StatusFeed.ts

@@ -20,7 +20,7 @@ export function useStatusFeed(id?: string, leaveOpen = false) {
 
   const status = useRequestBuilder(sub);
 
-  const statusFiltered = status.data?.filter(a => {
+  const statusFiltered = status.filter(a => {
     const exp = Number(findTag(a, "expiration"));
     return isNaN(exp) || exp >= unixNow();
   });

packages/app/src/Feed/ThreadFeed.ts

@@ -33,8 +33,8 @@ export default function useThreadFeed(link: NostrLink) {
   const store = useRequestBuilder(sub);
 
   useEffect(() => {
-    if (store.data) {
+    if (store) {
-      const links = store.data
+      const links = store
         .map(a => [
           NostrLink.fromEvent(a),
           ...a.tags.filter(a => a[0] === "e" || a[0] === "a").map(v => NostrLink.fromTag(v)),
@@ -42,7 +42,7 @@ export default function useThreadFeed(link: NostrLink) {
         .flat();
       setAllEvents(links);
 
-      const current = store.data.find(a => link.matchesEvent(a));
+      const current = store.find(a => link.matchesEvent(a));
       if (current) {
         const t = EventExt.extractThread(current);
         if (t) {
@@ -60,12 +60,12 @@ export default function useThreadFeed(link: NostrLink) {
         }
       }
     }
-  }, [store.data?.length]);
+  }, [store?.length]);
 
   const reactions = useReactions(`thread:${link.id.slice(0, 12)}:reactions`, [link, ...allEvents]);
 
   return {
-    thread: store.data ?? [],
+    thread: store ?? [],
-    reactions: reactions.data ?? [],
+    reactions: reactions ?? [],
   };
 }

packages/app/src/Feed/TimelineFeed.ts

@@ -1,7 +1,7 @@
 import { unixNow } from "@snort/shared";
 import { EventKind, RequestBuilder } from "@snort/system";
-import { useRequestBuilder } from "@snort/system-react";
+import { useRequestBuilderAdvanced } from "@snort/system-react";
-import { useCallback, useMemo } from "react";
+import { useCallback, useMemo, useSyncExternalStore } from "react";
 
 import useLogin from "@/Hooks/useLogin";
 import useTimelineWindow from "@/Hooks/useTimelineWindow";
@@ -116,7 +116,16 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
     return rb?.builder ?? null;
   }, [until, since, options.method, pref, createBuilder]);
 
-  const main = useRequestBuilder(sub);
+  const mainQuery = useRequestBuilderAdvanced(sub);
+  const main = useSyncExternalStore(
+    h => {
+      mainQuery?.on("event", h);
+      return () => {
+        mainQuery?.off("event", h);
+      };
+    },
+    () => mainQuery?.snapshot,
+  );
 
   const subRealtime = useMemo(() => {
     const rb = createBuilder();
@@ -130,17 +139,25 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
     return rb?.builder ?? null;
   }, [pref.autoShowLatest, createBuilder]);
 
-  const latest = useRequestBuilder(subRealtime);
+  const latestQuery = useRequestBuilderAdvanced(subRealtime);
+  const latest = useSyncExternalStore(
+    h => {
+      latestQuery?.on("event", h);
+      return () => {
+        latestQuery?.off("event", h);
+      };
+    },
+    () => latestQuery?.snapshot,
+  );
 
   return {
-    main: main.data,
+    main: main,
-    latest: latest.data,
+    latest: latest,
-    loading: main.loading(),
     loadMore: () => {
-      if (main.data) {
+      if (main) {
         console.debug("Timeline load more!");
         if (options.method === "LIMIT_UNTIL") {
-          const oldest = main.data.reduce((acc, v) => (acc = v.created_at < acc ? v.created_at : acc), unixNow());
+          const oldest = main.reduce((acc, v) => (acc = v.created_at < acc ? v.created_at : acc), unixNow());
           setUntil(oldest);
         } else {
           older();
@@ -148,9 +165,9 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
       }
     },
     showLatest: () => {
-      if (latest.data) {
+      if (latest) {
-        main.add(latest.data);
+        mainQuery?.feed.add(latest);
-        latest.clear();
+        latestQuery?.feed.clear();
       }
     },
   };

packages/app/src/Feed/ZapsFeed.ts

@@ -13,8 +13,8 @@ export default function useZapsFeed(link?: NostrLink) {
   const zapsFeed = useRequestBuilder(sub);
 
   const zaps = useMemo(() => {
-    if (zapsFeed.data) {
+    if (zapsFeed) {
-      const profileZaps = zapsFeed.data.map(a => parseZap(a)).filter(z => z.valid);
+      const profileZaps = zapsFeed.map(a => parseZap(a)).filter(z => z.valid);
       profileZaps.sort((a, b) => b.amount - a.amount);
       return profileZaps;
     }

packages/app/src/Hooks/useCommunityLeaders.tsx

@@ -27,7 +27,6 @@ export function useCommunityLeaders() {
   });
 
   useEffect(() => {
-    console.debug("CommunityLeaders", list);
     LeadersStore.setLeaders(list.map(a => a.id));
   }, [list]);
 }

packages/app/src/Hooks/useLists.tsx

@@ -1,4 +1,4 @@
-import { EventKind, NostrLink, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, NostrLink, RequestBuilder } from "@snort/system";
 import { useEventsFeed, useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -12,18 +12,18 @@ export function useLinkList(id: string, fn: (rb: RequestBuilder) => void) {
     return rb;
   }, [id, fn]);
 
-  const listStore = useRequestBuilder(NoteCollection, sub);
+  const listStore = useRequestBuilder(sub);
   return useMemo(() => {
-    if (listStore.data && listStore.data.length > 0) {
+    if (listStore && listStore.length > 0) {
-      return listStore.data.map(e => NostrLink.fromTags(e.tags)).flat();
+      return listStore.map(e => NostrLink.fromTags(e.tags)).flat();
     }
     return [];
-  }, [listStore.data]);
+  }, [listStore]);
 }
 
 export function useLinkListEvents(id: string, fn: (rb: RequestBuilder) => void) {
   const links = useLinkList(id, fn);
-  return useEventsFeed(`${id}:events`, links).data ?? [];
+  return useEventsFeed(`${id}:events`, links);
 }
 
 export function usePinList(pubkey: string | undefined) {

packages/app/src/Hooks/useRates.tsx

@@ -1,8 +1,10 @@
 import { bech32ToHex } from "@snort/shared";
-import { EventKind, ReplaceableNoteStore, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
+import { getNewest } from "@/Utils";
+
 // Snort backend publishes rates
 const SnortPubkey = "npub1sn0rtcjcf543gj4wsg7fa59s700d5ztys5ctj0g69g2x6802npjqhjjtws";
 
@@ -20,12 +22,13 @@ export function useRates(symbol: string, leaveOpen = true) {
     return rb;
   }, [symbol]);
 
-  const data = useRequestBuilder(ReplaceableNoteStore, sub);
+  const feed = useRequestBuilder(sub);
+  const ev = getNewest(feed);
 
-  const tag = data?.data?.tags.find(a => a[0] === "d" && a[1] === symbol);
+  const tag = ev?.tags.find(a => a[0] === "d" && a[1] === symbol);
   if (!tag) return undefined;
   return {
-    time: data.data?.created_at,
+    time: ev?.created_at,
     ask: Number(tag[2]),
     bid: Number(tag[3]),
     low: Number(tag[4]),

packages/app/src/Hooks/useRefreshFeedcache.tsx

@@ -1,5 +1,5 @@
 import { unwrap } from "@snort/shared";
-import { NoopStore, RequestBuilder, TaggedNostrEvent } from "@snort/system";
+import { RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { useEffect, useMemo } from "react";
 
 import { RefreshFeedCache } from "@/Cache/RefreshFeedCache";
@@ -25,23 +25,14 @@ export function useRefreshFeedCache<T>(c: RefreshFeedCache<T>, leaveOpen = false
 
   useEffect(() => {
     if (sub) {
-      const q = system.Query(NoopStore, sub);
+      const q = system.Query(sub);
-      let t: ReturnType<typeof setTimeout> | undefined;
+      const handler = (evs: Array<TaggedNostrEvent>) => {
-      let tBuf: Array<TaggedNostrEvent> = [];
+        c.onEvent(evs, unwrap(login.publicKey), publisher);
-      q.on("event", evs => {
+      };
-        if (!t) {
+      q.on("event", handler);
-          tBuf = [...evs];
-          t = setTimeout(() => {
-            t = undefined;
-            c.onEvent(tBuf, unwrap(login.publicKey), publisher);
-          }, 100);
-        } else {
-          tBuf.push(...evs);
-        }
-      });
       q.uncancel();
       return () => {
-        q.off("event");
+        q.off("event", handler);
         q.cancel();
       };
     }

packages/app/src/Pages/HashTagsPage.tsx

@@ -1,5 +1,5 @@
 import { dedupe } from "@snort/shared";
-import { EventKind, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import classNames from "classnames";
 import { useMemo } from "react";
@@ -59,8 +59,8 @@ export function HashTagHeader({ tag, events, className }: { tag: string; events?
     rb.withFilter().kinds([EventKind.InterestsList]).tag("t", [tag.toLowerCase()]);
     return rb;
   }, [tag]);
-  const followsTag = useRequestBuilder(NoteCollection, sub);
+  const followsTag = useRequestBuilder(sub);
-  const pubkeys = dedupe((followsTag.data ?? []).map(a => a.pubkey));
+  const pubkeys = dedupe(followsTag.map(a => a.pubkey));
 
   return (
     <div className={classNames("flex flex-col", className)}>

packages/app/src/Pages/Layout/Header.tsx

@@ -103,10 +103,10 @@ function NoteTitle({ link }: { link: NostrLink }) {
   const ev = useEventFeed(link);
 
   const values = useMemo(() => {
-    return { name: <DisplayName pubkey={ev.data?.pubkey ?? ""} /> };
+    return { name: <DisplayName pubkey={ev?.pubkey ?? ""} /> };
-  }, [ev.data?.pubkey]);
+  }, [ev?.pubkey]);
 
-  if (!ev.data?.pubkey) {
+  if (!ev?.pubkey) {
     return <FormattedMessage defaultMessage="Note" id="qMePPG" />;
   }
 

packages/app/src/Pages/Messages/Nip28ChatProfile.tsx

@@ -6,8 +6,8 @@ import ProfilePreview from "@/Components/User/ProfilePreview";
 
 export default function Nip28ChatProfile({ id, onClick }: { id: string; onClick: (id: string) => void }) {
   const channel = useEventFeed(new NostrLink(CONFIG.eventLinkPrefix, id, 40));
-  if (channel?.data) {
+  if (channel) {
-    const meta = JSON.parse(channel.data.content) as UserMetadata;
+    const meta = JSON.parse(channel.content) as UserMetadata;
     return (
       <ProfilePreview
         pubkey=""

packages/app/src/Pages/Notifications/Notifications.tsx

@@ -252,7 +252,7 @@ function NotificationGroup({ evs, onClick }: { evs: Array<TaggedNostrEvent>; onC
 }
 
 function NotificationContext({ link, onClick }: { link: NostrLink; onClick: () => void }) {
-  const { data: ev } = useEventFeed(link);
+  const ev = useEventFeed(link);
   if (link.type === NostrPrefix.PublicKey) {
     return <ProfilePreview pubkey={link.id} actions={<></>} />;
   }

packages/app/src/Pages/settings/Preferences.tsx

@@ -415,7 +415,6 @@ const PreferencesPage = () => {
           value={perf.reactionEmoji}
           onChange={e => {
             const split = e.target.value.match(/[\p{L}\S]{1}/u);
-            console.debug(e.target.value, split);
             updatePreferences(id, {
               ...perf,
               reactionEmoji: split?.[0] ?? "",

packages/app/src/Pages/settings/wallet/Alby.tsx

@@ -19,7 +19,6 @@ export default function AlbyOAuth() {
   async function setupWallet(token: string) {
     try {
       const auth = await alby.getToken(token);
-      console.debug(auth);
       const connection = new AlbyWallet(auth, () => {});
       const info = await connection.getInfo();
 

packages/app/src/Utils/Login/MultiAccountStore.ts

@@ -221,7 +221,6 @@ export class MultiAccountStore extends ExternalStore<LoginSession> {
   updateSession(s: LoginSession) {
     if (this.#accounts.has(s.id)) {
       this.#accounts.set(s.id, s);
-      console.debug("SET SESSION", s);
       this.#save();
     }
   }

packages/app/src/Utils/Notifications.ts

@@ -78,8 +78,7 @@ export async function subscribeToNotifications(publisher: EventPublisher) {
   if ("Notification" in window) {
     try {
       if (Notification.permission !== "granted") {
-        const res = await Notification.requestPermission();
+        await Notification.requestPermission();
-        console.debug(res);
       }
     } catch (e) {
       console.error(e);

packages/app/src/Utils/ZapPoolController.ts

@@ -1,4 +1,5 @@
 import { ExternalStore, LNURL, unixNow } from "@snort/shared";
+import debug from "debug";
 
 import { UserCache } from "@/Cache";
 import { Toastore } from "@/Components/Toaster/Toaster";
@@ -21,6 +22,7 @@ export interface ZapPoolRecipient {
 }
 
 class ZapPool extends ExternalStore<Array<ZapPoolRecipient>> {
+  #log = debug("ZapPool");
   #store = new Map<string, ZapPoolRecipient>();
   #isPayoutInProgress = false;
   #lastPayout = 0;
@@ -50,7 +52,7 @@ class ZapPool extends ExternalStore<Array<ZapPoolRecipient>> {
         const invoice = await svc.getInvoice(amtSend, `SnortZapPool: ${x.split}%`);
         if (invoice.pr) {
           const result = await wallet.payInvoice(invoice.pr);
-          console.debug("ZPC", invoice, result);
+          this.#log("%o %o", invoice, result);
           if (result.state === WalletInvoiceState.Paid) {
             x.sum -= amtSend;
             Toastore.push({

packages/app/src/Utils/index.ts

@@ -55,7 +55,6 @@ export async function openFile(): Promise<File | undefined> {
       () => {
         setTimeout(() => {
           if (!lock) {
-            console.debug("FOCUS WINDOW UPLOAD");
             resolve(undefined);
           }
         }, 300);

packages/shared/src/feed-cache.ts

@@ -20,7 +20,7 @@ export interface FeedCacheEvents {
 export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
   readonly name: string;
   #snapshot: Array<TCached> = [];
-  #log: ReturnType<typeof debug>;
+  protected log: ReturnType<typeof debug>;
   #hits = 0;
   #miss = 0;
   protected table?: DexieTableLike<TCached>;
@@ -31,9 +31,9 @@ export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
     super();
     this.name = name;
     this.table = table;
-    this.#log = debug(name);
+    this.log = debug(name);
     setInterval(() => {
-      this.#log(
+      this.log(
         "%d loaded, %d on-disk, %d hooks, %d% hit",
         this.cache.size,
         this.onTable.size,
@@ -55,21 +55,15 @@ export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
   }
 
   hook(fn: HookFn, key: string | undefined) {
-    if (key) {
+    const handle = (keys: Array<string>) => {
-      const handle = (keys: Array<string>) => {
+      if (!key || keys.includes(key)) {
-        if (keys.includes(key)) {
+        fn();
-          fn();
+      }
-        }
-      };
-      this.on("change", handle);
-      return () => this.off("change", handle);
-    }
-
-    return () => {
-      // noop
     };
+    this.on("change", handle);
+    return () => this.off("change", handle);
   }
-  
+
   keysOnTable() {
     return [...this.onTable];
   }
@@ -161,7 +155,7 @@ export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
       }
       return "no_change";
     })();
-    this.#log("Updating %s %s %o", k, updateType, m);
+    this.log("Updating %s %s %o", k, updateType, m);
     if (updateType !== "no_change") {
       const updated = {
         ...existing,
@@ -193,7 +187,7 @@ export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
         "change",
         fromCache.map(a => this.key(a)),
       );
-      this.#log(`Loaded %d/%d in %d ms`, fromCache.length, keys.length, (unixNowMs() - start).toLocaleString());
+      this.log(`Loaded %d/%d in %d ms`, fromCache.length, keys.length, (unixNowMs() - start).toLocaleString());
       return mapped.filter(a => !a.has).map(a => a.key);
     }
 

packages/shared/src/lnurl.ts

@@ -136,7 +136,6 @@ export class LNURL {
       const rsp = await fetch(`${baseUrl}?${queryJoined}`);
       if (rsp.ok) {
         const data: LNURLInvoice = await rsp.json();
-        console.debug("[LNURL]: ", data);
         if (data.status === "ERROR") {
           throw new Error(data.reason);
         } else {

packages/system-react/README.md

@@ -4,7 +4,7 @@ React hooks for @snort/system
 
 ### Available hooks
 
-#### `useRequestBuilder(NoteStore, RequestBuilder)`
+#### `useRequestBuilder(RequestBuilder)`
 
 The main hook which allows you to subscribe to nostr relays and returns a reactive store.
 
@@ -63,10 +63,10 @@ export function UserPosts(props: { pubkey: string }) {
     return rb;
   }, [props.pubkey]);
 
-  const data = useRequestBuilder(NoteCollection, sub);
+  const data = useRequestBuilder(sub);
   return (
     <>
-      {data.data.map(a => (
+      {data.map(a => (
         <Note ev={a} />
       ))}
     </>

packages/system-react/example/example.tsx

@@ -1,7 +1,7 @@
 import { useMemo } from "react";
 import { SnortContext, useRequestBuilder, useUserProfile } from "../src";
 
-import { NostrSystem, NoteCollection, RequestBuilder, TaggedNostrEvent } from "@snort/system";
+import { NostrSystem, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 
 const System = new NostrSystem({});
 
@@ -13,7 +13,7 @@ export function Note({ ev }: { ev: TaggedNostrEvent }) {
 
   return (
     <div>
-      Post by: {profile.name ?? profile.display_name}
+      Post by: {profile?.name ?? profile?.display_name}
       <p>{ev.content}</p>
     </div>
   );
@@ -27,10 +27,10 @@ export function UserPosts(props: { pubkey: string }) {
     return rb;
   }, [props.pubkey]);
 
-  const data = useRequestBuilder(NoteCollection, sub);
+  const data = useRequestBuilder(sub);
   return (
     <>
-      {data.data.map(a => (
+      {data.map(a => (
         <Note ev={a} />
       ))}
     </>

packages/system-react/src/useEventFeed.ts

@@ -9,7 +9,7 @@ export function useEventFeed(link: NostrLink) {
     return b;
   }, [link]);
 
-  return useRequestBuilder(sub);
+  return useRequestBuilder(sub).at(0);
 }
 
 export function useEventsFeed(id: string, links: Array<NostrLink>) {

packages/system-react/src/useReactions.ts

@@ -27,7 +27,7 @@ export function useReactions(
       }
     }
     others?.(rb);
-    return rb.numFilters > 0 ? rb : null;
+    return rb.numFilters > 0 ? rb : undefined;
   }, [ids]);
 
   return useRequestBuilder(sub);

packages/system-react/src/useRequestBuilder.tsx

@@ -1,40 +1,55 @@
-import { useContext, useSyncExternalStore } from "react";
+import { useCallback, useContext, useEffect, useMemo, useSyncExternalStore } from "react";
-import { RequestBuilder, EmptySnapshot, NoteStore, StoreSnapshot } from "@snort/system";
+import { EmptySnapshot, RequestBuilder } from "@snort/system";
-import { unwrap } from "@snort/shared";
 import { SnortContext } from "./context";
 
 /**
  * Send a query to the relays and wait for data
  */
-const useRequestBuilder = (
+export function useRequestBuilder(rb: RequestBuilder | null | undefined) {
-  rb: RequestBuilder | null,
-) => {
   const system = useContext(SnortContext);
-  const subscribe = (onChanged: () => void) => {
+  return useSyncExternalStore(
+    v => {
+      if (rb) {
+        const q = system.Query(rb);
+        q.on("event", v);
+        q.uncancel();
+        return () => {
+          q.off("event", v);
+          q.cancel();
+        };
+      }
+      return () => {
+        // noop
+      };
+    },
+    () => {
+      const q = system.GetQuery(rb?.id ?? "");
+      if (q) {
+        return q.snapshot;
+      } else {
+        return EmptySnapshot;
+      }
+    },
+  );
+}
+
+/**
+ * More advanced hook which returns the Query object
+ */
+export function useRequestBuilderAdvanced(rb: RequestBuilder | null | undefined) {
+  const system = useContext(SnortContext);
+  const q = useMemo(() => {
     if (rb) {
       const q = system.Query(rb);
-      q.on("event", onChanged);
       q.uncancel();
-      return () => {
+      return q;
-        q.off("event", onChanged);
-        q.cancel();
-      };
     }
+  }, [rb]);
+  useEffect(() => {
     return () => {
-      // noop
+      q?.cancel();
     };
-  };
+  }, [q]);
-  const getState = () => {
-    const q = system.GetQuery(rb?.id ?? "");
-    if (q) {
-      return q.snapshot;
-    }
-    return EmptySnapshot;
-  };
-  return useSyncExternalStore(
-    v => subscribe(v),
-    () => getState(),
-  );
-};
 
-export { useRequestBuilder };
+  return q;
+}

packages/system-svelte/src/request-builder.ts

@@ -1,19 +1,19 @@
-import { type NoteStore, type RequestBuilder, type StoreSnapshot, type SystemInterface } from "@snort/system";
+import { TaggedNostrEvent, type RequestBuilder, type SystemInterface } from "@snort/system";
 import { getContext } from "svelte";
 
-export function useRequestBuilder<T extends NoteStore>(type: new () => T, rb: RequestBuilder) {
+export function useRequestBuilder(rb: RequestBuilder) {
   const system = getContext("snort") as SystemInterface;
-  type TSnap = StoreSnapshot<ReturnType<T["getSnapshotData"]>>;
   return {
-    subscribe: (set: (value: TSnap) => void) => {
+    subscribe: (set: (value: Array<TaggedNostrEvent>) => void) => {
-      const q = system.Query(type, rb);
+      const q = system.Query(rb);
+      const handle = () => {
+        set(q.snapshot);
+      };
       q.uncancel();
-      const release = q.feed.hook(() => {
+      q.on("event", handle);
-        set(q.feed.snapshot as TSnap);
-      });
       return () => {
+        q.off("event", handle);
         q.cancel();
-        release();
       };
     },
   };

packages/system/README.md

@@ -5,12 +5,7 @@ A collection of caching and querying techniquies used by https://snort.social to
 Simple example:
 
 ```js
-import {
+import { NostrSystem, RequestBuilder, StoreSnapshot, NoteCollection } from "@snort/system";
-    NostrSystem,
-    RequestBuilder,
-    StoreSnapshot,
-    NoteCollection
-} from "@snort/system"
 
 // Singleton instance to store all connections and access query fetching system
 const System = new NostrSystem({});
@@ -30,25 +25,11 @@ const System = new NostrSystem({});
     .kinds([1])
     .limit(10);
 
-  const q = System.Query(NoteCollection, rb);
+  const q = System.Query(rb);
   // basic usage using "onEvent", fired every 100ms
-  q.feed.onEvent(evs => {
+  q.on("event", evs => {
     console.log(evs);
     // something else..
   });
-
-  // Hookable type using change notification, limited to every 500ms
-  const release = q.feed.hook(() => {
-    // since we use the NoteCollection we expect NostrEvent[]
-    // other stores provide different data, like a single event instead of an array (latest version)
-    const state = q.feed.snapshot as StoreSnapshot<ReturnType<NoteCollection["getSnapshotData"]>>;
-
-    // do something with snapshot of store
-    console.log(`We have ${state.data?.length} events now!`);
-  });
-
-  // release the hook when its not needed anymore
-  // these patterns will be managed in @snort/system-react to make it easier to use react or other UI frameworks
-  release();
 })();
 ```

packages/system/examples/simple.ts

@@ -18,24 +18,10 @@ const System = new NostrSystem({});
     .kinds([1])
     .limit(10);
 
-  const q = System.Query(NoteCollection, rb);
+  const q = System.Query(rb);
   // basic usage using "onEvent", fired every 100ms
-  q.feed.onEvent(evs => {
+  q.on("event", evs => {
     console.log(evs);
     // something else..
   });
-
-  // Hookable type using change notification, limited to every 500ms
-  const release = q.feed.hook(() => {
-    // since we use the FlatNoteStore we expect NostrEvent[]
-    // other stores provide different data, like a single event instead of an array (latest version)
-    const state = q.feed.snapshot as StoreSnapshot<ReturnType<NoteCollection["getSnapshotData"]>>;
-
-    // do something with snapshot of store
-    console.log(`We have ${state.data?.length} events now!`);
-  });
-
-  // release the hook when its not needed anymore
-  // these patterns will be managed in @snort/system-react to make it easier to use react or other UI frameworks
-  release();
 })();

packages/system/src/background-loader.ts

@@ -76,16 +76,17 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
     } else {
       return await new Promise<T>((resolve, reject) => {
         this.TrackKeys(key);
-        this.cache.on("change", keys => {
+        const handler = (keys: Array<string>) => {
           if (keys.includes(key)) {
             const existing = this.cache.getFromCache(key);
             if (existing) {
               resolve(existing);
               this.UntrackKeys(key);
-              this.cache.off("change");
+              this.cache.off("change", handler);
             }
           }
-        });
+        };
+        this.cache.on("change", handler);
       });
     }
   }
@@ -103,6 +104,7 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
           missing.filter(a => !found.some(b => a === this.cache.key(b))).map(a => this.makePlaceholder(a)),
         );
         if (noResult.length > 0) {
+          this.#log("Adding placeholders for %O", noResult);
           await Promise.all(noResult.map(a => this.cache.update(a)));
         }
       } catch (e) {
@@ -115,19 +117,24 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
   }
 
   async #loadData(missing: Array<string>) {
+    this.#log("Loading data", missing);
     if (this.loaderFn) {
       const results = await this.loaderFn(missing);
       await Promise.all(results.map(a => this.cache.update(a)));
       return results;
     } else {
+      const hookHandled = new Set<string>();
       const v = await this.#system.Fetch(this.buildSub(missing), async e => {
+        this.#log("Callback handled %o", e);
         for (const pe of e) {
           const m = this.onEvent(pe);
           if (m) {
             await this.cache.update(m);
+            hookHandled.add(pe.id);
           }
         }
       });
+      this.#log("Got data", v);
       return removeUndefined(v.map(this.onEvent));
     }
   }

packages/system/src/cache/index.ts

@@ -50,6 +50,7 @@ export interface UsersRelays {
 }
 
 export function mapEventToProfile(ev: NostrEvent) {
+  if (ev.kind !== 0) return;
   try {
     const data: UserMetadata = JSON.parse(ev.content);
     let ret = {

packages/system/src/connection-pool.ts

@@ -30,7 +30,9 @@ export type ConnectionPool = {
 /**
  * Simple connection pool containing connections to multiple nostr relays
  */
-export class NostrConnectionPool extends EventEmitter<NostrConnectionPoolEvents> implements ConnectionPool {
+export class DefaultConnectionPool extends EventEmitter<NostrConnectionPoolEvents> implements ConnectionPool {
+  #system: SystemInterface;
+
   #log = debug("NostrConnectionPool");
 
   /**
@@ -38,6 +40,11 @@ export class NostrConnectionPool extends EventEmitter<NostrConnectionPoolEvents>
    */
   #sockets = new Map<string, Connection>();
 
+  constructor(system: SystemInterface) {
+    super();
+    this.#system = system;
+  }
+
   /**
    * Get basic state information from the pool
    */
@@ -63,7 +70,13 @@ export class NostrConnectionPool extends EventEmitter<NostrConnectionPoolEvents>
         const c = new Connection(addr, options, ephemeral);
         this.#sockets.set(addr, c);
 
-        c.on("event", (s, e) => this.emit("event", addr, s, e));
+        c.on("event", (s, e) => {
+          if (this.#system.checkSigs && !this.#system.optimizer.schnorrVerify(e)) {
+            this.#log("Reject invalid event %o", e);
+            return;
+          }
+          this.emit("event", addr, s, e);
+        });
         c.on("eose", s => this.emit("eose", addr, s));
         c.on("disconnect", code => this.emit("disconnect", addr, code));
         c.on("connected", r => this.emit("connected", addr, r));

packages/system/src/connection.ts

@@ -9,6 +9,7 @@ import { ConnectionStats } from "./connection-stats";
 import { NostrEvent, ReqCommand, ReqFilter, TaggedNostrEvent, u256 } from "./nostr";
 import { RelayInfo } from "./relay-info";
 import EventKind from "./event-kind";
+import { EventExt } from "./event-ext";
 
 /**
  * Relay settings
@@ -225,10 +226,16 @@ export class Connection extends EventEmitter<ConnectionEvents> {
           break;
         }
         case "EVENT": {
-          this.emit("event", msg[1] as string, {
+          const ev = {
             ...(msg[2] as NostrEvent),
             relays: [this.Address],
-          });
+          } as TaggedNostrEvent;
+
+          if (!EventExt.isValid(ev)) {
+            //this.#log("Rejecting invalid event %O", ev);
+            return;
+          }
+          this.emit("event", msg[1] as string, ev);
           this.Stats.EventsReceived++;
           this.notifyChange();
           break;

packages/system/src/index.ts

@@ -1,13 +1,14 @@
 import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
 import { RequestBuilder } from "./request-builder";
-import { NoteCollection, NoteStore, NoteStoreSnapshotData, StoreSnapshot } from "./note-collection";
 import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr";
 import { ProfileLoaderService } from "./profile-cache";
 import { RelayCache, RelayMetadataLoader } from "./outbox-model";
 import { Optimizer } from "./query-optimizer";
 import { base64 } from "@scure/base";
 import { FeedCache } from "@snort/shared";
-import { ConnectionPool } from "nostr-connection-pool";
+import { ConnectionPool } from "./connection-pool";
+import EventEmitter from "eventemitter3";
+import { QueryEvents } from "./query";
 
 export { NostrSystem } from "./nostr-system";
 export { default as EventKind } from "./event-kind";
@@ -46,13 +47,16 @@ export * from "./cache/relay-metric";
 
 export * from "./worker/system-worker";
 
-export interface QueryLike {
+export type QueryLike = {
-  on: (event: "event", fn?: (evs: Array<TaggedNostrEvent>) => void) => void;
+  get progress(): number;
-  off: (event: "event", fn?: (evs: Array<TaggedNostrEvent>) => void) => void;
+  feed: {
+    add: (evs: Array<TaggedNostrEvent>) => void;
+    clear: () => void;
+  };
   cancel: () => void;
   uncancel: () => void;
-  get snapshot(): StoreSnapshot<Array<TaggedNostrEvent>>;
+  get snapshot(): Array<TaggedNostrEvent>;
-}
+} & EventEmitter<QueryEvents>;
 
 export interface SystemInterface {
   /**
@@ -87,7 +91,7 @@ export interface SystemInterface {
    * @param req Request to send to relays
    * @param cb A callback which will fire every 100ms when new data is received
    */
-  Fetch(req: RequestBuilder, cb?: (evs: ReadonlyArray<TaggedNostrEvent>) => void): Promise<Array<TaggedNostrEvent>>;
+  Fetch(req: RequestBuilder, cb?: (evs: Array<TaggedNostrEvent>) => void): Promise<Array<TaggedNostrEvent>>;
 
   /**
    * Create a new permanent connection to a relay

packages/system/src/nostr-system.ts

@@ -23,8 +23,8 @@ import {
 import { EventsCache } from "./cache/events";
 import { RelayMetadataLoader } from "./outbox-model";
 import { Optimizer, DefaultOptimizer } from "./query-optimizer";
-import { NostrConnectionPool } from "./nostr-connection-pool";
+import { ConnectionPool, DefaultConnectionPool } from "./connection-pool";
-import { NostrQueryManager } from "./nostr-query-manager";
+import { QueryManager } from "./query-manager";
 
 export interface NostrSystemEvents {
   change: (state: SystemSnapshot) => void;
@@ -48,7 +48,7 @@ export interface NostrsystemProps {
  */
 export class NostrSystem extends EventEmitter<NostrSystemEvents> implements SystemInterface {
   #log = debug("System");
-  #queryManager: NostrQueryManager;
+  #queryManager: QueryManager;
 
   /**
    * Storage class for user relay lists
@@ -80,7 +80,7 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
    */
   readonly optimizer: Optimizer;
 
-  readonly pool = new NostrConnectionPool();
+  readonly pool: ConnectionPool;
   readonly eventsCache: FeedCache<NostrEvent>;
   readonly relayLoader: RelayMetadataLoader;
 
@@ -102,7 +102,8 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
     this.relayLoader = new RelayMetadataLoader(this, this.relayCache);
     this.checkSigs = props.checkSigs ?? true;
 
-    this.#queryManager = new NostrQueryManager(this);
+    this.pool = new DefaultConnectionPool(this);
+    this.#queryManager = new QueryManager(this);
 
     // hook connection pool
     this.pool.on("connected", (id, wasReconnect) => {
@@ -121,18 +122,6 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
     });
     this.pool.on("event", (_, sub, ev) => {
       ev.relays?.length && this.relayMetricsHandler.onEvent(ev.relays[0]);
-
-      if (!EventExt.isValid(ev)) {
-        this.#log("Rejecting invalid event %O", ev);
-        return;
-      }
-      if (this.checkSigs) {
-        if (!this.optimizer.schnorrVerify(ev)) {
-          this.#log("Invalid sig %O", ev);
-          return;
-        }
-      }
-
       this.emit("event", sub, ev);
     });
     this.pool.on("disconnect", (id, code) => {
@@ -160,17 +149,6 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
     this.#queryManager.on("trace", t => {
       this.relayMetricsHandler.onTraceReport(t);
     });
-
-    // internal handler for on-event
-    this.on("event", (sub, ev) => {
-      for (const [, v] of this.#queryManager) {
-        const trace = v.handleEvent(sub, ev);
-        // inject events to cache if query by id
-        if (trace && trace.filters.some(a => a.ids)) {
-          this.eventsCache.set(ev);
-        }
-      }
-    });
   }
 
   get Sockets(): ConnectionStateSnapshot[] {
@@ -200,15 +178,15 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
   }
 
   GetQuery(id: string): QueryLike | undefined {
-    return this.#queryManager.get(id) as QueryLike;
+    return this.#queryManager.get(id);
   }
 
-  Fetch(req: RequestBuilder, cb?: (evs: ReadonlyArray<TaggedNostrEvent>) => void) {
+  Fetch(req: RequestBuilder, cb?: (evs: Array<TaggedNostrEvent>) => void) {
     return this.#queryManager.fetch(req, cb);
   }
 
   Query(req: RequestBuilder): QueryLike {
-    return this.#queryManager.query(req) as QueryLike;
+    return this.#queryManager.query(req);
   }
 
   HandleEvent(ev: TaggedNostrEvent) {

packages/system/src/note-collection.ts

@@ -1,27 +1,10 @@
-import { appendDedupe, SortedMap } from "@snort/shared";
+import { SortedMap } from "@snort/shared";
-import { EventExt, EventType, TaggedNostrEvent, u256 } from ".";
+import { EventExt, EventType, TaggedNostrEvent } from ".";
 import { findTag } from "./utils";
 import EventEmitter from "eventemitter3";
 
-export interface StoreSnapshot<TSnapshot extends NoteStoreSnapshotData> {
+export const EmptySnapshot: NoteStoreSnapshotData = [];
-  data: TSnapshot | undefined;
+export type NoteStoreSnapshotData = Array<TaggedNostrEvent>;
-  clear: () => void;
-  loading: () => boolean;
-  add: (ev: Readonly<TaggedNostrEvent> | Readonly<Array<TaggedNostrEvent>>) => void;
-}
-
-export const EmptySnapshot = {
-  data: undefined,
-  clear: () => {
-    // empty
-  },
-  loading: () => true,
-  add: () => {
-    // empty
-  },
-} as StoreSnapshot<Array<TaggedNostrEvent>>;
-
-export type NoteStoreSnapshotData = Array<TaggedNostrEvent> | TaggedNostrEvent;
 export type NoteStoreHook = () => void;
 export type NoteStoreHookRelease = () => void;
 export type OnEventCallback = (e: Readonly<Array<TaggedNostrEvent>>) => void;
@@ -30,8 +13,7 @@ export type OnEoseCallback = (c: string) => void;
 export type OnEoseCallbackRelease = () => void;
 
 export interface NosteStoreEvents {
-  progress: (loading: boolean) => void;
+  event: (evs: Array<TaggedNostrEvent>) => void;
-  event: (evs: Readonly<Array<TaggedNostrEvent>>) => void;
 }
 
 /**
@@ -40,133 +22,40 @@ export interface NosteStoreEvents {
 export abstract class NoteStore extends EventEmitter<NosteStoreEvents> {
   abstract add(ev: Readonly<TaggedNostrEvent> | Readonly<Array<TaggedNostrEvent>>): void;
   abstract clear(): void;
-  abstract getSnapshotData(): NoteStoreSnapshotData | undefined;
 
-  abstract get snapshot(): StoreSnapshot<NoteStoreSnapshotData>;
+  abstract get snapshot(): NoteStoreSnapshotData;
-  abstract get loading(): boolean;
-  abstract set loading(v: boolean);
 }
 
-export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> extends NoteStore {
+export abstract class HookedNoteStore extends NoteStore {
-  #loading = true;
+  #storeSnapshot: NoteStoreSnapshotData = [];
-  #storeSnapshot: StoreSnapshot<TSnapshot> = {
+  #nextEmit?: ReturnType<typeof setTimeout>;
-    clear: () => this.clear(),
-    loading: () => this.loading,
-    add: ev => this.add(ev),
-    data: undefined,
-  };
-  #needsSnapshot = true;
-  #nextNotifyTimer?: ReturnType<typeof setTimeout>;
   #bufEmit: Array<TaggedNostrEvent> = [];
 
   get snapshot() {
-    this.#updateSnapshot();
     return this.#storeSnapshot;
   }
 
-  get loading() {
-    return this.#loading;
-  }
-
-  set loading(v: boolean) {
-    this.#loading = v;
-    this.emit("progress", v);
-  }
-
   abstract override add(ev: Readonly<TaggedNostrEvent> | Readonly<Array<TaggedNostrEvent>>): void;
   abstract override clear(): void;
+  protected abstract takeSnapshot(): NoteStoreSnapshotData | undefined;
 
-  getSnapshotData() {
+  protected onChange(changes: Array<TaggedNostrEvent>): void {
-    this.#updateSnapshot();
+    this.#storeSnapshot = this.takeSnapshot() ?? [];
-    return this.#storeSnapshot.data;
-  }
-
-  protected abstract takeSnapshot(): TSnapshot | undefined;
-
-  protected onChange(changes: Readonly<Array<TaggedNostrEvent>>): void {
-    this.#needsSnapshot = true;
     this.#bufEmit.push(...changes);
-    if (!this.#nextNotifyTimer) {
+    if (!this.#nextEmit) {
-      this.#nextNotifyTimer = setTimeout(() => {
+      this.#nextEmit = setTimeout(() => {
-        this.#nextNotifyTimer = undefined;
+        this.#nextEmit = undefined;
         this.emit("event", this.#bufEmit);
         this.#bufEmit = [];
-      }, 500);
+      }, 300);
     }
   }
-
-  #updateSnapshot() {
-    if (this.#needsSnapshot) {
-      this.#storeSnapshot = {
-        ...this.#storeSnapshot,
-        data: this.takeSnapshot(),
-      };
-      this.#needsSnapshot = false;
-    }
-  }
-}
-
-/**
- * A store which doesnt store anything, useful for hooks only
- */
-export class NoopStore extends HookedNoteStore<Array<TaggedNostrEvent>> {
-  override add(ev: readonly TaggedNostrEvent[] | Readonly<TaggedNostrEvent>): void {
-    this.onChange(Array.isArray(ev) ? ev : [ev]);
-  }
-
-  override clear(): void {
-    // nothing to do
-  }
-
-  protected override takeSnapshot(): TaggedNostrEvent[] | undefined {
-    // nothing to do
-    return undefined;
-  }
-}
-
-/**
- * A simple flat container of events with no duplicates
- */
-export class FlatNoteStore extends HookedNoteStore<Array<TaggedNostrEvent>> {
-  #events: Array<TaggedNostrEvent> = [];
-  #ids: Set<u256> = new Set();
-
-  add(ev: TaggedNostrEvent | Array<TaggedNostrEvent>) {
-    ev = Array.isArray(ev) ? ev : [ev];
-    const changes: Array<TaggedNostrEvent> = [];
-    ev.forEach(a => {
-      if (!this.#ids.has(a.id)) {
-        this.#events.push(a);
-        this.#ids.add(a.id);
-        changes.push(a);
-      } else {
-        const existing = this.#events.findIndex(b => b.id === a.id);
-        if (existing !== -1) {
-          this.#events[existing].relays = appendDedupe(this.#events[existing].relays, a.relays);
-        }
-      }
-    });
-
-    if (changes.length > 0) {
-      this.onChange(changes);
-    }
-  }
-
-  clear() {
-    this.#events = [];
-    this.#ids.clear();
-    this.onChange([]);
-  }
-
-  takeSnapshot() {
-    return [...this.#events];
-  }
 }
 
 /**
  * A note store that holds a single replaceable event for a given user defined key generator function
  */
-export class KeyedReplaceableNoteStore extends HookedNoteStore<Array<TaggedNostrEvent>> {
+export class KeyedReplaceableNoteStore extends HookedNoteStore {
   #keyFn: (ev: TaggedNostrEvent) => string;
   #events: SortedMap<string, TaggedNostrEvent> = new SortedMap([], (a, b) => b[1].created_at - a[1].created_at);
 
@@ -201,39 +90,6 @@ export class KeyedReplaceableNoteStore extends HookedNoteStore<Array<TaggedNostr
   }
 }
 
-/**
- * A note store that holds a single replaceable event
- */
-export class ReplaceableNoteStore extends HookedNoteStore<Readonly<TaggedNostrEvent>> {
-  #event?: TaggedNostrEvent;
-
-  add(ev: TaggedNostrEvent | Array<TaggedNostrEvent>) {
-    ev = Array.isArray(ev) ? ev : [ev];
-    const changes: Array<TaggedNostrEvent> = [];
-    ev.forEach(a => {
-      const existingCreated = this.#event?.created_at ?? 0;
-      if (a.created_at > existingCreated) {
-        this.#event = a;
-        changes.push(a);
-      }
-    });
-    if (changes.length > 0) {
-      this.onChange(changes);
-    }
-  }
-
-  clear() {
-    this.#event = undefined;
-    this.onChange([]);
-  }
-
-  takeSnapshot() {
-    if (this.#event) {
-      return { ...this.#event };
-    }
-  }
-}
-
 /**
  * General use note store based on kind ranges
  */

packages/system/src/profile-cache.ts

@@ -2,7 +2,6 @@ import { unixNowMs } from "@snort/shared";
 import { EventKind, TaggedNostrEvent, RequestBuilder } from ".";
 import { ProfileCacheExpire } from "./const";
 import { mapEventToProfile, CachedMetadata } from "./cache";
-import { v4 as uuid } from "uuid";
 import { BackgroundLoader } from "./background-loader";
 
 export class ProfileLoaderService extends BackgroundLoader<CachedMetadata> {
@@ -19,14 +18,8 @@ export class ProfileLoaderService extends BackgroundLoader<CachedMetadata> {
   }
 
   override buildSub(missing: string[]): RequestBuilder {
-    const sub = new RequestBuilder(`profiles-${uuid()}`);
+    const sub = new RequestBuilder(`profiles`);
-    sub
+    sub.withFilter().kinds([EventKind.SetMetadata]).authors(missing);
-      .withOptions({
-        skipDiff: true,
-      })
-      .withFilter()
-      .kinds([EventKind.SetMetadata])
-      .authors(missing);
     return sub;
   }
 

packages/system/src/query-manager.ts

@@ -2,11 +2,10 @@ import debug from "debug";
 import EventEmitter from "eventemitter3";
 import { BuiltRawReqFilter, RequestBuilder, SystemInterface, TaggedNostrEvent } from ".";
 import { Query, TraceReport } from "./query";
-import { unwrap } from "@snort/shared";
 import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer";
 import { trimFilters } from "./request-trim";
 
-interface NostrQueryManagerEvents {
+interface QueryManagerEvents {
   change: () => void;
   trace: (report: TraceReport) => void;
 }
@@ -14,8 +13,8 @@ interface NostrQueryManagerEvents {
 /**
  * Query manager handles sending requests to the nostr network
  */
-export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
+export class QueryManager extends EventEmitter<QueryManagerEvents> {
-  #log = debug("NostrQueryManager");
+  #log = debug("QueryManager");
 
   /**
    * All active queries
@@ -70,20 +69,27 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
   /**
    * Async fetch results
    */
-  fetch(req: RequestBuilder, cb?: (evs: ReadonlyArray<TaggedNostrEvent>) => void) {
+  async fetch(req: RequestBuilder, cb?: (evs: Array<TaggedNostrEvent>) => void) {
-    const q = this.query(req);
+    const q = new Query(this.#system, req);
-    return new Promise<Array<TaggedNostrEvent>>(resolve => {
+    q.on("trace", r => this.emit("trace", r));
-      if (cb) {
+    q.on("filters", fx => {
-        q.feed.on("event", cb);
+      this.#send(q, fx);
-      }
+    });
-      q.feed.on("progress", loading => {
+    if (cb) {
+      q.on("event", evs => cb(evs));
+    }
+    await new Promise<void>(resolve => {
+      q.on("loading", loading => {
+        this.#log("loading %s %o", q.id, loading);
         if (!loading) {
-          q.feed.off("event");
+          resolve();
-          q.cancel();
-          resolve(unwrap(q.snapshot.data));
         }
       });
     });
+    const results = q.feed.takeSnapshot();
+    q.cleanup();
+    this.#log("Fetch results for %s %o", q.id, results);
+    return results;
   }
 
   *[Symbol.iterator]() {
@@ -105,12 +111,13 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
     // check for empty filters
     const fNew = trimFilters(qSend.filters);
     if (fNew.length === 0) {
+      this.#log("Dropping %s %o", q.id, qSend);
       return;
     }
     qSend.filters = fNew;
 
     if (qSend.relay) {
-      this.#log("Sending query to %s %O", qSend.relay, qSend);
+      this.#log("Sending query to %s %s %O", qSend.relay, q.id, qSend);
       const s = this.#system.pool.getConnection(qSend.relay);
       if (s) {
         const qt = q.sendToRelay(s, qSend);
@@ -132,7 +139,7 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
       const ret = [];
       for (const [a, s] of this.#system.pool) {
         if (!s.Ephemeral) {
-          this.#log("Sending query to %s %O", a, qSend);
+          this.#log("Sending query to %s %s %O", a, q.id, qSend);
           const qt = q.sendToRelay(s, qSend);
           if (qt) {
             ret.push(qt);
@@ -142,6 +149,7 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
       return ret;
     }
   }
+
   #cleanup() {
     let changed = false;
     for (const [k, v] of this.#queries) {

packages/system/src/query.ts

@@ -4,7 +4,7 @@ import EventEmitter from "eventemitter3";
 import { unixNowMs, unwrap } from "@snort/shared";
 
 import { Connection, ReqFilter, Nips, TaggedNostrEvent, SystemInterface } from ".";
-import { NoteCollection, NoteStore } from "./note-collection";
+import { NoteCollection } from "./note-collection";
 import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
 import { eventMatchesFilter } from "./request-matcher";
 
@@ -97,25 +97,26 @@ export interface TraceReport {
   responseTime: number;
 }
 
-interface QueryEvents {
+export interface QueryEvents {
+  loading: (v: boolean) => void;
   trace: (report: TraceReport) => void;
   filters: (req: BuiltRawReqFilter) => void;
-  event: (evs: ReadonlyArray<TaggedNostrEvent>) => void;
+  event: (evs: Array<TaggedNostrEvent>) => void;
+  end: () => void;
 }
 
 /**
  * Active or queued query on the system
  */
 export class Query extends EventEmitter<QueryEvents> {
-  /**
+  get id() {
-   * Unique id of this query
+    return this.request.id;
-   */
+  }
-  readonly id: string;
 
   /**
    * RequestBuilder instance
    */
-  requests: Array<RequestBuilder> = [];
+  request: RequestBuilder;
 
   /**
    * Nostr system interface
@@ -166,8 +167,7 @@ export class Query extends EventEmitter<QueryEvents> {
 
   constructor(system: SystemInterface, req: RequestBuilder) {
     super();
-    this.id = uuid();
+    this.request = req;
-    this.requests.push(req);
     this.#system = system;
     this.#feed = new NoteCollection();
     this.#leaveOpen = req.options?.leaveOpen ?? false;
@@ -176,32 +176,21 @@ export class Query extends EventEmitter<QueryEvents> {
     this.#checkTraces();
 
     this.feed.on("event", evs => this.emit("event", evs));
+    this.#start();
   }
 
   /**
    * Adds another request to this one
    */
   addRequest(req: RequestBuilder) {
-    if (this.#groupTimeout) {
+    if (req.instance === this.request.instance) {
-      clearTimeout(this.#groupTimeout);
+      // same requst, do nothing
-      this.#groupTimeout = undefined;
+      this.#log("Same query %O === %O", req, this.request);
-    }
+      return;
-    if (this.requests.some(a => a.instance === req.instance)) {
-      // already exists, nothing to add
-      return false;
-    }
-    if (this.requests.some(a => a.options?.skipDiff !== req.options?.skipDiff)) {
-      throw new Error("Mixing skipDiff option is not supported");
-    }
-    this.requests.push(req);
-
-    if (this.#groupingDelay) {
-      this.#groupTimeout = setTimeout(() => {
-        this.#emitFilters();
-      }, this.#groupingDelay);
-    } else {
-      this.#emitFilters();
     }
+    this.#log("Add query %O to %s", req, this.id);
+    this.request.add(req);
+    this.#start();
     return true;
   }
 
@@ -228,12 +217,11 @@ export class Query extends EventEmitter<QueryEvents> {
     return this.#feed.snapshot;
   }
 
-  handleEvent(sub: string, e: TaggedNostrEvent) {
+  #handleEvent(sub: string, e: TaggedNostrEvent) {
     for (const t of this.#tracing) {
       if (t.id === sub || sub === "*") {
         if (t.filters.some(v => eventMatchesFilter(e, v))) {
           this.feed.add(e);
-          return t;
         } else {
           this.#log("Event did not match filter, rejecting %O %O", e, t);
         }
@@ -254,7 +242,12 @@ export class Query extends EventEmitter<QueryEvents> {
   }
 
   cleanup() {
+    if (this.#groupTimeout) {
+      clearTimeout(this.#groupTimeout);
+      this.#groupTimeout = undefined;
+    }
     this.#stopCheckTraces();
+    this.emit("end");
   }
 
   /**
@@ -316,35 +309,46 @@ export class Query extends EventEmitter<QueryEvents> {
     return thisProgress;
   }
 
-  #emitFilters() {
+  #start() {
-    if (this.requests.every(a => !!a.options?.skipDiff)) {
+    if (this.#groupTimeout) {
-      const existing = this.filters;
+      clearTimeout(this.#groupTimeout);
-      const rb = new RequestBuilder(this.id);
+      this.#groupTimeout = undefined;
-      this.requests.forEach(a => rb.add(a));
+    }
-      const filters = rb.buildDiff(this.#system, existing);
+    if (this.#groupingDelay) {
-      filters.forEach(f => this.emit("filters", f));
+      this.#groupTimeout = setTimeout(() => {
-      this.requests = [];
+        this.#emitFilters();
+      }, this.#groupingDelay);
     } else {
-      // send without diff
+      this.#emitFilters();
-      const rb = new RequestBuilder(this.id);
+    }
-      this.requests.forEach(a => rb.add(a));
+  }
-      const filters = rb.build(this.#system);
+
+  #emitFilters() {
+    this.#log("Starting emit of %s", this.id);
+    const existing = this.filters;
+    if (!(this.request.options?.skipDiff ?? false) && existing.length > 0) {
+      const filters = this.request.buildDiff(this.#system, existing);
+      this.#log("Build %s %O", this.id, filters);
+      filters.forEach(f => this.emit("filters", f));
+    } else {
+      const filters = this.request.build(this.#system);
+      this.#log("Build %s %O", this.id, filters);
       filters.forEach(f => this.emit("filters", f));
-      this.requests = [];
     }
   }
 
   #onProgress() {
     const isFinished = this.progress === 1;
-    if (this.feed.loading !== isFinished) {
+    if (isFinished) {
-      this.#log("%s loading=%s, progress=%d, traces=%O", this.id, this.feed.loading, this.progress, this.#tracing);
+      this.#log("%s loading=%s, progress=%d, traces=%O", this.id, !isFinished, this.progress, this.#tracing);
-      this.feed.loading = isFinished;
+      this.emit("loading", !isFinished);
     }
   }
 
   #stopCheckTraces() {
     if (this.#checkTrace) {
       clearInterval(this.#checkTrace);
+      this.#checkTrace = undefined;
     }
   }
 
@@ -398,6 +402,9 @@ export class Query extends EventEmitter<QueryEvents> {
         responseTime: qt.responseTime,
       } as TraceReport),
     );
+    const handler = (sub: string, ev: TaggedNostrEvent) => this.#handleEvent(sub, ev);
+    c.on("event", handler);
+    this.on("end", () => c.off("event", handler));
     this.#tracing.push(qt);
     c.QueueReq(["REQ", qt.id, ...qt.filters], () => qt.sentToRelay());
     return qt;

packages/system/src/worker/system-worker.ts

@@ -25,7 +25,7 @@ import { FeedCache } from "@snort/shared";
 import { EventsCache } from "../cache/events";
 import { RelayMetricHandler } from "../relay-metric-handler";
 import debug from "debug";
-import { ConnectionPool } from "nostr-connection-pool";
+import { ConnectionPool } from "connection-pool";
 
 export class SystemWorker extends EventEmitter<NostrSystemEvents> implements SystemInterface {
   #log = debug("SystemWorker");