refactor: Query emits Filters

packages/app/src/Cache/FollowListCache.ts

@@ -44,6 +44,6 @@ export class FollowListCache extends RefreshFeedCache<TaggedNostrEvent> {
 
   override async preload() {
     await super.preload();
-    this.snapshot().forEach(e => socialGraphInstance.handleEvent(e));
+    this.cache.forEach(e => socialGraphInstance.handleEvent(e));
   }
 }

packages/app/src/Cache/FollowsFeed.ts

@@ -43,7 +43,10 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
     const filtered = evs.filter(a => this.#kinds.includes(a.kind));
     if (filtered.length > 0) {
       await this.bulkSet(filtered);
-      this.notifyChange(filtered.map(a => this.key(a)));
+      this.emit(
+        "change",
+        filtered.map(a => this.key(a)),
+      );
     }
   }
 
@@ -64,7 +67,7 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
 
     const oldest = await this.table?.orderBy("created_at").first();
     this.#oldest = oldest?.created_at;
-    this.notifyChange(latest?.map(a => this.key(a)) ?? []);
+    this.emit("change", latest?.map(a => this.key(a)) ?? []);
 
     debug(this.name)(`Loaded %d/%d in %d ms`, latest?.length ?? 0, keys.length, (unixNowMs() - start).toLocaleString());
   }
@@ -96,7 +99,7 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
         this.onTable.add(k);
       });
 
-      this.notifyChange(latest?.map(a => this.key(a)) ?? []);
+      this.emit("change", latest?.map(a => this.key(a)) ?? []);
     }
   }
 

packages/app/src/Cache/Notifications.ts

@@ -33,7 +33,10 @@ export class NotificationsCache extends RefreshFeedCache<NostrEventForSession> {
           forSession: pubKey,
         })),
       );
-      this.notifyChange(filtered.map(v => this.key(v)));
+      this.emit(
+        "change",
+        filtered.map(v => this.key(v)),
+      );
     }
   }
 

packages/app/src/Cache/RefreshFeedCache.ts

@@ -24,7 +24,6 @@ export abstract class RefreshFeedCache<T> extends FeedCache<TWithCreated<T>> {
 
   override async preload(): Promise<void> {
     await super.preload();
-    // load all dms to memory
     await this.buffer([...this.onTable]);
   }
 }

packages/app/src/Components/IrisAccount/IrisAccount.tsx

@@ -285,15 +285,20 @@ class IrisAccount extends Component<Props> {
   componentDidMount() {
     const session = LoginStore.snapshot();
     const myPub = session.publicKey;
-    System.ProfileLoader.Cache.hook(() => {
+    if (myPub) {
-      const profile = System.ProfileLoader.Cache.getFromCache(myPub);
+      System.profileLoader.cache.on("change", keys => {
-      const irisToActive = profile && profile.nip05 && profile.nip05.endsWith("@iris.to");
+        if (keys.includes(myPub)) {
-      this.setState({ profile, irisToActive });
+          const profile = System.profileLoader.cache.getFromCache(myPub);
-      if (profile && !irisToActive) {
+          const irisToActive = profile && profile.nip05 && profile.nip05.endsWith("@iris.to");
-        this.checkExistingAccount(myPub);
+          this.setState({ profile, irisToActive });
-      }
+          if (profile && !irisToActive) {
-    }, myPub);
+            this.checkExistingAccount(myPub);
-    this.checkExistingAccount(myPub);
+          }
+        }
+      });
+
+      this.checkExistingAccount(myPub);
+    }
   }
 
   async checkExistingAccount(pub: any) {

packages/app/src/Components/User/ProfileLink.tsx

@@ -18,7 +18,7 @@ export function ProfileLink({
   children?: ReactNode;
 } & Omit<LinkProps, "to">) {
   const system = useContext(SnortContext);
-  const relays = system.RelayCache.getFromCache(pubkey)
+  const relays = system.relayCache.getFromCache(pubkey)
     ?.relays?.filter(a => a.settings.write)
     ?.map(a => a.url);
 

packages/app/src/Feed/ArticlesFeed.ts

@@ -1,4 +1,4 @@
-import { EventKind, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -15,5 +15,5 @@ export function useArticles() {
     return rb;
   }, [follows.timestamp]);
 
-  return useRequestBuilder(NoteCollection, sub);
+  return useRequestBuilder(sub);
 }

packages/app/src/Feed/BadgesFeed.ts

@@ -1,4 +1,4 @@
-import { EventKind, HexKey, NoteCollection, ReplaceableNoteStore, RequestBuilder } from "@snort/system";
+import { EventKind, HexKey, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -17,12 +17,12 @@ export default function useProfileBadges(pubkey?: HexKey) {
     return b;
   }, [pubkey]);
 
-  const profileBadges = useRequestBuilder(ReplaceableNoteStore, sub);
+  const profileBadges = useRequestBuilder(sub);
 
   const profile = useMemo(() => {
     if (profileBadges.data) {
       return chunks(
-        profileBadges.data.tags.filter(t => t[0] === "a" || t[0] === "e"),
+        profileBadges.data[0].tags.filter(t => t[0] === "a" || t[0] === "e"),
         2,
       ).reduce((acc, [a, e]) => {
         return {
@@ -57,7 +57,7 @@ export default function useProfileBadges(pubkey?: HexKey) {
     return b;
   }, [profile, ds]);
 
-  const awards = useRequestBuilder(NoteCollection, awardsSub);
+  const awards = useRequestBuilder(awardsSub);
 
   const result = useMemo(() => {
     if (awards.data) {

packages/app/src/Feed/FollowersFeed.ts

@@ -1,4 +1,4 @@
-import { EventKind, HexKey, NoteCollection, RequestBuilder, socialGraphInstance } from "@snort/system";
+import { EventKind, HexKey, RequestBuilder, socialGraphInstance } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -10,7 +10,7 @@ export default function useFollowersFeed(pubkey?: HexKey) {
     return b;
   }, [pubkey]);
 
-  const followersFeed = useRequestBuilder(NoteCollection, sub);
+  const followersFeed = useRequestBuilder(sub);
 
   const followers = useMemo(() => {
     const contactLists = followersFeed.data?.filter(

packages/app/src/Feed/FollowsFeed.ts

@@ -1,4 +1,4 @@
-import { EventKind, HexKey, NoteCollection, RequestBuilder, TaggedNostrEvent } from "@snort/system";
+import { EventKind, HexKey, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -15,7 +15,7 @@ export default function useFollowsFeed(pubkey?: HexKey) {
     return b;
   }, [isMe, pubkey]);
 
-  const contactFeed = useRequestBuilder(NoteCollection, sub);
+  const contactFeed = useRequestBuilder(sub);
   return useMemo(() => {
     if (isMe) {
       return follows.item;

packages/app/src/Feed/HashtagsFeed.ts

@@ -1,5 +1,5 @@
 import { unixNow } from "@snort/shared";
-import { EventKind, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -18,7 +18,7 @@ export default function useHashtagsFeed() {
   }, [hashtags]);
 
   return {
-    data: useRequestBuilder(NoteCollection, sub),
+    data: useRequestBuilder(sub),
     hashtags,
   };
 }

packages/app/src/Feed/LoginFeed.ts

@@ -1,9 +1,9 @@
-import { EventKind, NostrLink, NoteCollection, parseRelayTags, RequestBuilder, TaggedNostrEvent } from "@snort/system";
+import { EventKind, NostrLink, parseRelayTags, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { usePrevious } from "@uidotdev/usehooks";
 import { useEffect, useMemo } from "react";
 
-import { FollowLists, FollowsFeed, GiftsCache, Notifications, UserRelays } from "@/Cache";
+import { FollowLists, FollowsFeed, GiftsCache, Notifications } from "@/Cache";
 import { Nip4Chats, Nip28Chats } from "@/chat";
 import { Nip28ChatSystem } from "@/chat/nip28";
 import useEventPublisher from "@/Hooks/useEventPublisher";
@@ -96,7 +96,7 @@ export default function useLoginFeed() {
     return b;
   }, [login]);
 
-  const loginFeed = useRequestBuilder(NoteCollection, subLogin);
+  const loginFeed = useRequestBuilder(subLogin);
 
   // update relays and follow lists
   useEffect(() => {
@@ -219,7 +219,6 @@ export default function useLoginFeed() {
   }, [loginFeed]);
 
   useEffect(() => {
-    UserRelays.buffer(follows.item).catch(console.error);
+    system.profileLoader.TrackKeys(follows.item); // always track follows profiles
-    system.ProfileLoader.TrackKeys(follows.item); // always track follows profiles
   }, [follows.item]);
 }

packages/app/src/Feed/RelaysFeed.tsx

@@ -1,4 +1,4 @@
-import { EventKind, HexKey, parseRelayTags, ReplaceableNoteStore, RequestBuilder } from "@snort/system";
+import { EventKind, HexKey, parseRelayTags, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -10,6 +10,6 @@ export default function useRelaysFeed(pubkey?: HexKey) {
     return b;
   }, [pubkey]);
 
-  const relays = useRequestBuilder(ReplaceableNoteStore, sub);
+  const relays = useRequestBuilder(sub);
-  return parseRelayTags(relays.data?.tags.filter(a => a[0] === "r") ?? []);
+  return parseRelayTags(relays.data?.[0].tags.filter(a => a[0] === "r") ?? []);
 }

packages/app/src/Feed/StatusFeed.ts

@@ -1,5 +1,5 @@
 import { unixNow } from "@snort/shared";
-import { EventKind, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -18,7 +18,7 @@ export function useStatusFeed(id?: string, leaveOpen = false) {
     return rb;
   }, [id]);
 
-  const status = useRequestBuilder(NoteCollection, sub);
+  const status = useRequestBuilder(sub);
 
   const statusFiltered = status.data?.filter(a => {
     const exp = Number(findTag(a, "expiration"));

packages/app/src/Feed/ThreadFeed.ts

@@ -1,4 +1,4 @@
-import { EventExt, EventKind, NostrLink, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventExt, EventKind, NostrLink, RequestBuilder } from "@snort/system";
 import { useReactions, useRequestBuilder } from "@snort/system-react";
 import { useEffect, useMemo, useState } from "react";
 
@@ -30,7 +30,7 @@ export default function useThreadFeed(link: NostrLink) {
     return sub;
   }, [allEvents.length]);
 
-  const store = useRequestBuilder(NoteCollection, sub);
+  const store = useRequestBuilder(sub);
 
   useEffect(() => {
     if (store.data) {

packages/app/src/Feed/TimelineFeed.ts

@@ -1,5 +1,5 @@
 import { unixNow } from "@snort/shared";
-import { EventKind, NoteCollection, RequestBuilder } from "@snort/system";
+import { EventKind, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useCallback, useMemo } from "react";
 
@@ -116,7 +116,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
     return rb?.builder ?? null;
   }, [until, since, options.method, pref, createBuilder]);
 
-  const main = useRequestBuilder(NoteCollection, sub);
+  const main = useRequestBuilder(sub);
 
   const subRealtime = useMemo(() => {
     const rb = createBuilder();
@@ -130,7 +130,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
     return rb?.builder ?? null;
   }, [pref.autoShowLatest, createBuilder]);
 
-  const latest = useRequestBuilder(NoteCollection, subRealtime);
+  const latest = useRequestBuilder(subRealtime);
 
   return {
     main: main.data,

packages/app/src/Feed/ZapsFeed.ts

@@ -1,4 +1,4 @@
-import { EventKind, NostrLink, NoteCollection, parseZap, RequestBuilder } from "@snort/system";
+import { EventKind, NostrLink, parseZap, RequestBuilder } from "@snort/system";
 import { useRequestBuilder } from "@snort/system-react";
 import { useMemo } from "react";
 
@@ -10,7 +10,7 @@ export default function useZapsFeed(link?: NostrLink) {
     return b;
   }, [link]);
 
-  const zapsFeed = useRequestBuilder(NoteCollection, sub);
+  const zapsFeed = useRequestBuilder(sub);
 
   const zaps = useMemo(() => {
     if (zapsFeed.data) {

packages/app/src/Pages/NetworkGraph.tsx

@@ -146,7 +146,7 @@ const NetworkGraph = () => {
         const node = {
           id: UID,
           address: pubkey,
-          profile: system.ProfileLoader.Cache.getFromCache(pubkey),
+          profile: system.profileLoader.cache.getFromCache(pubkey),
           distance,
           inboundCount,
           outboundCount,

packages/app/src/Pages/settings/tools/follows-relay-health.tsx

@@ -23,7 +23,7 @@ export function FollowsRelayHealth({
   const uniqueFollows = dedupe(follows.item);
 
   const hasRelays = useMemo(() => {
-    return uniqueFollows.filter(a => (system.RelayCache.getFromCache(a)?.relays.length ?? 0) > 0);
+    return uniqueFollows.filter(a => (system.relayCache.getFromCache(a)?.relays.length ?? 0) > 0);
   }, [uniqueFollows]);
 
   const missingRelays = useMemo(() => {
@@ -31,7 +31,7 @@ export function FollowsRelayHealth({
   }, [hasRelays]);
 
   const topWriteRelays = useMemo(() => {
-    return pickTopRelays(system.RelayCache, uniqueFollows, 1e31, "write");
+    return pickTopRelays(system.relayCache, uniqueFollows, 1e31, "write");
   }, [uniqueFollows]);
 
   return (

packages/app/src/chat/nip24.ts

@@ -12,7 +12,7 @@ export class Nip24ChatSystem extends ExternalStore<Array<Chat>> implements ChatS
   constructor(cache: GiftWrapCache) {
     super();
     this.#cache = cache;
-    this.#cache.hook(() => this.notifyChange(), "*");
+    this.#cache.on("change", () => this.notifyChange());
   }
 
   subscription() {

packages/app/src/chat/nip28.ts

@@ -19,7 +19,7 @@ import { LoginSession } from "@/Utils/Login";
 
 export class Nip28ChatSystem extends ExternalStore<Array<Chat>> implements ChatSystem {
   #cache: FeedCache<NostrEvent>;
-  #log = debug("NIP-04");
+  #log = debug("NIP-28");
   readonly ChannelKinds = [
     EventKind.PublicChatChannel,
     EventKind.PublicChatMessage,

packages/app/src/system.ts

@@ -30,21 +30,20 @@ System.on("event", (_, ev) => {
   socialGraphInstance.handleEvent(ev);
 });
 
+System.profileCache.on("change", keys => {
+  const changed = removeUndefined(keys.map(a => System.profileCache.getFromCache(a)));
+  changed.forEach(addCachedMetadataToFuzzySearch);
+});
+
 /**
  * Add profile loader fn
  */
 if (CONFIG.httpCache) {
-  System.ProfileLoader.loaderFn = async (keys: Array<string>) => {
+  System.profileLoader.loaderFn = async (keys: Array<string>) => {
     return removeUndefined(await Promise.all(keys.map(a => fetchProfile(a))));
   };
 }
 
-setTimeout(() => {
-  System.UserProfileCache.snapshot().forEach(a => {
-    addCachedMetadataToFuzzySearch(a);
-  });
-}, 2000);
-
 export async function fetchProfile(key: string) {
   try {
     throwIfOffline();

packages/shared/src/feed-cache.ts

@@ -1,6 +1,7 @@
 import debug from "debug";
-import { removeUndefined, unixNowMs, unwrap } from "./utils";
+import { removeUndefined, unixNowMs } from "./utils";
 import { DexieTableLike } from "./dexie-like";
+import EventEmitter from "eventemitter3";
 
 type HookFn = () => void;
 
@@ -9,14 +10,17 @@ export interface KeyedHookFilter {
   fn: HookFn;
 }
 
+export interface FeedCacheEvents {
+  change: (keys: Array<string>) => void;
+}
+
 /**
  * Dexie backed generic hookable store
  */
-export abstract class FeedCache<TCached> {
+export abstract class FeedCache<TCached> extends EventEmitter<FeedCacheEvents> {
-  #name: string;
+  readonly name: string;
-  #hooks: Array<KeyedHookFilter> = [];
   #snapshot: Array<TCached> = [];
-  #changed = true;
+  #log: ReturnType<typeof debug>;
   #hits = 0;
   #miss = 0;
   protected table?: DexieTableLike<TCached>;
@@ -24,21 +28,22 @@ export abstract class FeedCache<TCached> {
   protected cache: Map<string, TCached> = new Map();
 
   constructor(name: string, table?: DexieTableLike<TCached>) {
-    this.#name = name;
+    super();
+    this.name = name;
     this.table = table;
+    this.#log = debug(name);
     setInterval(() => {
-      debug(this.#name)(
+      this.#log(
         "%d loaded, %d on-disk, %d hooks, %d% hit",
         this.cache.size,
         this.onTable.size,
-        this.#hooks.length,
+        this.listenerCount("change"),
         ((this.#hits / (this.#hits + this.#miss)) * 100).toFixed(1),
       );
     }, 30_000);
-  }
+    this.on("change", () => {
-
+      this.#snapshot = this.takeSnapshot();
-  get name() {
+    });
-    return this.#name;
   }
 
   async preload() {
@@ -49,28 +54,25 @@ export abstract class FeedCache<TCached> {
     }
   }
 
-  keysOnTable() {
-    return [...this.onTable];
-  }
-
   hook(fn: HookFn, key: string | undefined) {
-    if (!key) {
+    if (key) {
-      return () => {
+      const handle = (keys: Array<string>) => {
-        //noop
+        if (keys.includes(key)) {
+          fn();
+        }
       };
+      this.on("change", handle);
+      return () => this.off("change", handle);
     }
 
-    this.#hooks.push({
-      key,
-      fn,
-    });
     return () => {
-      const idx = this.#hooks.findIndex(a => a.fn === fn);
+      // noop
-      if (idx >= 0) {
-        this.#hooks.splice(idx, 1);
-      }
     };
   }
+  
+  keysOnTable() {
+    return [...this.onTable];
+  }
 
   getFromCache(key?: string) {
     if (key) {
@@ -89,7 +91,7 @@ export abstract class FeedCache<TCached> {
       const cached = await this.table.get(key);
       if (cached) {
         this.cache.set(this.key(cached), cached);
-        this.notifyChange([key]);
+        this.emit("change", [key]);
         return cached;
       }
     }
@@ -120,7 +122,7 @@ export abstract class FeedCache<TCached> {
         console.error(e);
       }
     }
-    this.notifyChange([k]);
+    this.emit("change", [k]);
   }
 
   async bulkSet(obj: Array<TCached> | Readonly<Array<TCached>>) {
@@ -133,7 +135,10 @@ export abstract class FeedCache<TCached> {
       }
     }
     obj.forEach(v => this.cache.set(this.key(v), v));
-    this.notifyChange(obj.map(a => this.key(a)));
+    this.emit(
+      "change",
+      obj.map(a => this.key(a)),
+    );
   }
 
   /**
@@ -156,7 +161,7 @@ export abstract class FeedCache<TCached> {
       }
       return "no_change";
     })();
-    debug(this.#name)("Updating %s %s %o", k, updateType, m);
+    this.#log("Updating %s %s %o", k, updateType, m);
     if (updateType !== "no_change") {
       const updated = {
         ...existing,
@@ -184,8 +189,11 @@ export abstract class FeedCache<TCached> {
       fromCache.forEach(a => {
         this.cache.set(this.key(a), a);
       });
-      this.notifyChange(fromCache.map(a => this.key(a)));
+      this.emit(
-      debug(this.#name)(`Loaded %d/%d in %d ms`, fromCache.length, keys.length, (unixNowMs() - start).toLocaleString());
+        "change",
+        fromCache.map(a => this.key(a)),
+      );
+      this.#log(`Loaded %d/%d in %d ms`, fromCache.length, keys.length, (unixNowMs() - start).toLocaleString());
       return mapped.filter(a => !a.has).map(a => a.key);
     }
 
@@ -197,23 +205,12 @@ export abstract class FeedCache<TCached> {
     await this.table?.clear();
     this.cache.clear();
     this.onTable.clear();
-    this.#changed = true;
-    this.#hooks.forEach(h => h.fn());
   }
 
   snapshot() {
-    if (this.#changed) {
-      this.#snapshot = this.takeSnapshot();
-      this.#changed = false;
-    }
     return this.#snapshot;
   }
 
-  protected notifyChange(keys: Array<string>) {
-    this.#changed = true;
-    this.#hooks.filter(a => keys.includes(a.key) || a.key === "*").forEach(h => h.fn());
-  }
-
   abstract key(of: TCached): string;
   abstract takeSnapshot(): Array<TCached>;
 }

packages/system-react/src/useEventFeed.ts

@@ -1,5 +1,5 @@
 import { useMemo } from "react";
-import { RequestBuilder, ReplaceableNoteStore, NostrLink, NoteCollection } from "@snort/system";
+import { RequestBuilder, NostrLink } from "@snort/system";
 import { useRequestBuilder } from "./useRequestBuilder";
 
 export function useEventFeed(link: NostrLink) {
@@ -9,7 +9,7 @@ export function useEventFeed(link: NostrLink) {
     return b;
   }, [link]);
 
-  return useRequestBuilder(ReplaceableNoteStore, sub);
+  return useRequestBuilder(sub);
 }
 
 export function useEventsFeed(id: string, links: Array<NostrLink>) {
@@ -19,5 +19,5 @@ export function useEventsFeed(id: string, links: Array<NostrLink>) {
     return b;
   }, [id, links]);
 
-  return useRequestBuilder(NoteCollection, sub);
+  return useRequestBuilder(sub);
 }

packages/system-react/src/useReactions.ts

@@ -30,5 +30,5 @@ export function useReactions(
     return rb.numFilters > 0 ? rb : null;
   }, [ids]);
 
-  return useRequestBuilder(NoteCollection, sub);
+  return useRequestBuilder(sub);
 }

packages/system-react/src/useRequestBuilder.tsx

@@ -6,14 +6,13 @@ import { SnortContext } from "./context";
 /**
  * Send a query to the relays and wait for data
  */
-const useRequestBuilder = <TStore extends NoteStore, TSnapshot = ReturnType<TStore["getSnapshotData"]>>(
+const useRequestBuilder = (
-  type: { new (): TStore },
   rb: RequestBuilder | null,
 ) => {
   const system = useContext(SnortContext);
   const subscribe = (onChanged: () => void) => {
     if (rb) {
-      const q = system.Query<TStore>(type, rb);
+      const q = system.Query(rb);
       q.on("event", onChanged);
       q.uncancel();
       return () => {
@@ -25,14 +24,14 @@ const useRequestBuilder = <TStore extends NoteStore, TSnapshot = ReturnType<TSto
       // noop
     };
   };
-  const getState = (): StoreSnapshot<TSnapshot> => {
+  const getState = () => {
     const q = system.GetQuery(rb?.id ?? "");
     if (q) {
-      return q.snapshot as StoreSnapshot<TSnapshot>;
+      return q.snapshot;
     }
-    return EmptySnapshot as StoreSnapshot<TSnapshot>;
+    return EmptySnapshot;
   };
-  return useSyncExternalStore<StoreSnapshot<TSnapshot>>(
+  return useSyncExternalStore(
     v => subscribe(v),
     () => getState(),
   );

packages/system-react/src/useUserProfile.ts

@@ -10,16 +10,23 @@ export function useUserProfile(pubKey?: HexKey): CachedMetadata | undefined {
   return useSyncExternalStore<CachedMetadata | undefined>(
     h => {
       if (pubKey) {
-        system.ProfileLoader.TrackKeys(pubKey);
+        const handler = (keys: Array<string>) => {
+          if (keys.includes(pubKey)) {
+            h();
+          }
+        };
+        system.profileLoader.cache.on("change", handler);
+        system.profileLoader.TrackKeys(pubKey);
+
+        return () => {
+          system.profileLoader.cache.off("change", handler);
+          system.profileLoader.UntrackKeys(pubKey);
+        };
       }
-      const release = system.ProfileLoader.Cache.hook(h, pubKey);
       return () => {
-        release();
+        // noop
-        if (pubKey) {
-          system.ProfileLoader.UntrackKeys(pubKey);
-        }
       };
     },
-    () => system.ProfileLoader.Cache.getFromCache(pubKey),
+    () => system.profileLoader.cache.getFromCache(pubKey),
   );
 }

packages/system-react/src/useUserSearch.tsx

@@ -5,7 +5,7 @@ import { SnortContext } from "./context";
 
 export function useUserSearch() {
   const system = useContext(SnortContext);
-  const cache = system.ProfileLoader.Cache as UserProfileCache;
+  const cache = system.profileLoader.cache as UserProfileCache;
 
   async function search(input: string): Promise<Array<string>> {
     // try exact match first

packages/system/src/background-loader.ts

@@ -4,7 +4,7 @@ import { SystemInterface, TaggedNostrEvent, RequestBuilder } from ".";
 
 export abstract class BackgroundLoader<T extends { loaded: number; created: number }> {
   #system: SystemInterface;
-  #cache: FeedCache<T>;
+  readonly cache: FeedCache<T>;
   #log = debug(this.name());
 
   /**
@@ -19,14 +19,10 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
 
   constructor(system: SystemInterface, cache: FeedCache<T>) {
     this.#system = system;
-    this.#cache = cache;
+    this.cache = cache;
     this.#FetchMetadata();
   }
 
-  get Cache() {
-    return this.#cache;
-  }
-
   /**
    * Name of this loader service
    */
@@ -74,38 +70,40 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
    * Get object from cache or fetch if missing
    */
   async fetch(key: string) {
-    const existing = this.Cache.get(key);
+    const existing = this.cache.get(key);
     if (existing) {
       return existing;
     } else {
       return await new Promise<T>((resolve, reject) => {
         this.TrackKeys(key);
-        const release = this.Cache.hook(() => {
+        this.cache.on("change", keys => {
-          const existing = this.Cache.getFromCache(key);
+          if (keys.includes(key)) {
-          if (existing) {
+            const existing = this.cache.getFromCache(key);
-            resolve(existing);
+            if (existing) {
-            release();
+              resolve(existing);
-            this.UntrackKeys(key);
+              this.UntrackKeys(key);
+              this.cache.off("change");
+            }
           }
-        }, key);
+        });
       });
     }
   }
 
   async #FetchMetadata() {
     const loading = [...this.#wantsKeys];
-    await this.#cache.buffer(loading);
+    await this.cache.buffer(loading);
 
-    const missing = loading.filter(a => (this.#cache.getFromCache(a)?.loaded ?? 0) < this.getExpireCutoff());
+    const missing = loading.filter(a => (this.cache.getFromCache(a)?.loaded ?? 0) < this.getExpireCutoff());
     if (missing.length > 0) {
       this.#log("Fetching keys: %O", missing);
       try {
         const found = await this.#loadData(missing);
         const noResult = removeUndefined(
-          missing.filter(a => !found.some(b => a === this.#cache.key(b))).map(a => this.makePlaceholder(a)),
+          missing.filter(a => !found.some(b => a === this.cache.key(b))).map(a => this.makePlaceholder(a)),
         );
         if (noResult.length > 0) {
-          await Promise.all(noResult.map(a => this.#cache.update(a)));
+          await Promise.all(noResult.map(a => this.cache.update(a)));
         }
       } catch (e) {
         this.#log("Error: %O", e);
@@ -119,14 +117,14 @@ export abstract class BackgroundLoader<T extends { loaded: number; created: numb
   async #loadData(missing: Array<string>) {
     if (this.loaderFn) {
       const results = await this.loaderFn(missing);
-      await Promise.all(results.map(a => this.#cache.update(a)));
+      await Promise.all(results.map(a => this.cache.update(a)));
       return results;
     } else {
       const v = await this.#system.Fetch(this.buildSub(missing), async e => {
         for (const pe of e) {
           const m = this.onEvent(pe);
           if (m) {
-            await this.#cache.update(m);
+            await this.cache.update(m);
           }
         }
       });

packages/system/src/index.ts

@@ -1,11 +1,13 @@
 import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
 import { RequestBuilder } from "./request-builder";
-import { NoteStore, NoteStoreSnapshotData, StoreSnapshot } from "./note-collection";
+import { NoteCollection, NoteStore, NoteStoreSnapshotData, StoreSnapshot } from "./note-collection";
 import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr";
 import { ProfileLoaderService } from "./profile-cache";
-import { RelayCache } from "./outbox-model";
+import { RelayCache, RelayMetadataLoader } from "./outbox-model";
 import { Optimizer } from "./query-optimizer";
 import { base64 } from "@scure/base";
+import { FeedCache } from "@snort/shared";
+import { ConnectionPool } from "nostr-connection-pool";
 
 export { NostrSystem } from "./nostr-system";
 export { default as EventKind } from "./event-kind";
@@ -49,7 +51,7 @@ export interface QueryLike {
   off: (event: "event", fn?: (evs: Array<TaggedNostrEvent>) => void) => void;
   cancel: () => void;
   uncancel: () => void;
-  get snapshot(): StoreSnapshot<NoteStoreSnapshotData>;
+  get snapshot(): StoreSnapshot<Array<TaggedNostrEvent>>;
 }
 
 export interface SystemInterface {
@@ -76,10 +78,9 @@ export interface SystemInterface {
 
   /**
    * Open a new query to relays
-   * @param type Store type
    * @param req Request to send to relays
    */
-  Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): QueryLike;
+  Query(req: RequestBuilder): QueryLike;
 
   /**
    * Fetch data from nostr relays asynchronously
@@ -123,17 +124,32 @@ export interface SystemInterface {
   /**
    * Profile cache/loader
    */
-  get ProfileLoader(): ProfileLoaderService;
+  get profileLoader(): ProfileLoaderService;
 
   /**
    * Relay cache for "Gossip" model
    */
-  get RelayCache(): RelayCache;
+  get relayCache(): RelayCache;
 
   /**
    * Query optimizer
    */
-  get Optimizer(): Optimizer;
+  get optimizer(): Optimizer;
+
+  /**
+   * Generic cache store for events
+   */
+  get eventsCache(): FeedCache<NostrEvent>;
+
+  /**
+   * Relay loader loads relay metadata for a set of profiles
+   */
+  get relayLoader(): RelayMetadataLoader;
+
+  /**
+   * Main connection pool
+   */
+  get pool(): ConnectionPool;
 }
 
 export interface SystemSnapshot {

packages/system/src/nostr-connection-pool.ts

@@ -24,7 +24,8 @@ export type ConnectionPool = {
   disconnect(address: string): void;
   broadcast(system: SystemInterface, ev: NostrEvent, cb?: (rsp: OkResponse) => void): Promise<OkResponse[]>;
   broadcastTo(address: string, ev: NostrEvent): Promise<OkResponse>;
-} & EventEmitter<NostrConnectionPoolEvents>;
+} & EventEmitter<NostrConnectionPoolEvents> &
+  Iterable<[string, Connection]>;
 
 /**
  * Simple connection pool containing connections to multiple nostr relays

packages/system/src/nostr-query-manager.ts

@@ -1,15 +1,19 @@
 import debug from "debug";
 import EventEmitter from "eventemitter3";
-import { BuiltRawReqFilter, NoteCollection, NoteStore, RequestBuilder, SystemInterface, TaggedNostrEvent } from ".";
+import { BuiltRawReqFilter, RequestBuilder, SystemInterface, TaggedNostrEvent } from ".";
 import { Query, TraceReport } from "./query";
 import { unwrap } from "@snort/shared";
+import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer";
+import { trimFilters } from "./request-trim";
 
 interface NostrQueryManagerEvents {
   change: () => void;
   trace: (report: TraceReport) => void;
-  sendQuery: (q: Query, filter: BuiltRawReqFilter) => void;
 }
 
+/**
+ * Query manager handles sending requests to the nostr network
+ */
 export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
   #log = debug("NostrQueryManager");
 
@@ -23,9 +27,15 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
    */
   #system: SystemInterface;
 
+  /**
+   * Query cache processing layers which can take data from a cache
+   */
+  #queryCacheLayers: Array<FilterCacheLayer> = [];
+
   constructor(system: SystemInterface) {
     super();
     this.#system = system;
+    this.#queryCacheLayers.push(new IdsFilterCacheLayer(system.eventsCache));
 
     setInterval(() => this.#cleanup(), 1_000);
   }
@@ -37,34 +47,21 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
   /**
    * Compute query to send to relays
    */
-  query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): Query {
+  query(req: RequestBuilder): Query {
     const existing = this.#queries.get(req.id);
     if (existing) {
-      // if same instance, just return query
+      if (existing.addRequest(req)) {
-      if (existing.fromInstance === req.instance) {
-        return existing;
-      }
-      const filters = !req.options?.skipDiff ? req.buildDiff(this.#system, existing.filters) : req.build(this.#system);
-      if (filters.length === 0 && !!req.options?.skipDiff) {
-        return existing;
-      } else {
-        for (const subQ of filters) {
-          this.emit("sendQuery", existing, subQ);
-        }
         this.emit("change");
-        return existing;
       }
+      return existing;
     } else {
-      const store = new type();
+      const q = new Query(this.#system, req);
-
-      const filters = req.build(this.#system);
-      const q = new Query(req.id, req.instance, store, req.options?.leaveOpen, req.options?.timeout);
       q.on("trace", r => this.emit("trace", r));
+      q.on("filters", fx => {
+        this.#send(q, fx);
+      });
 
       this.#queries.set(req.id, q);
-      for (const subQ of filters) {
-        this.emit("sendQuery", q, subQ);
-      }
       this.emit("change");
       return q;
     }
@@ -74,10 +71,8 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
    * Async fetch results
    */
   fetch(req: RequestBuilder, cb?: (evs: ReadonlyArray<TaggedNostrEvent>) => void) {
-    const q = this.query(NoteCollection, req);
+    const q = this.query(req);
     return new Promise<Array<TaggedNostrEvent>>(resolve => {
-      let t: ReturnType<typeof setTimeout> | undefined;
-      let tBuf: Array<TaggedNostrEvent> = [];
       if (cb) {
         q.feed.on("event", cb);
       }
@@ -85,7 +80,7 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
         if (!loading) {
           q.feed.off("event");
           q.cancel();
-          resolve(unwrap((q.feed as NoteCollection).snapshot.data));
+          resolve(unwrap(q.snapshot.data));
         }
       });
     });
@@ -97,6 +92,56 @@ export class NostrQueryManager extends EventEmitter<NostrQueryManagerEvents> {
     }
   }
 
+  async #send(q: Query, qSend: BuiltRawReqFilter) {
+    for (const qfl of this.#queryCacheLayers) {
+      qSend = await qfl.processFilter(q, qSend);
+    }
+    for (const f of qSend.filters) {
+      if (f.authors) {
+        this.#system.relayLoader.TrackKeys(f.authors);
+      }
+    }
+
+    // check for empty filters
+    const fNew = trimFilters(qSend.filters);
+    if (fNew.length === 0) {
+      return;
+    }
+    qSend.filters = fNew;
+
+    if (qSend.relay) {
+      this.#log("Sending query to %s %O", qSend.relay, qSend);
+      const s = this.#system.pool.getConnection(qSend.relay);
+      if (s) {
+        const qt = q.sendToRelay(s, qSend);
+        if (qt) {
+          return [qt];
+        }
+      } else {
+        const nc = await this.#system.pool.connect(qSend.relay, { read: true, write: true }, true);
+        if (nc) {
+          const qt = q.sendToRelay(nc, qSend);
+          if (qt) {
+            return [qt];
+          }
+        } else {
+          console.warn("Failed to connect to new relay for:", qSend.relay, q);
+        }
+      }
+    } else {
+      const ret = [];
+      for (const [a, s] of this.#system.pool) {
+        if (!s.Ephemeral) {
+          this.#log("Sending query to %s %O", a, qSend);
+          const qt = q.sendToRelay(s, qSend);
+          if (qt) {
+            ret.push(qt);
+          }
+        }
+      }
+      return ret;
+    }
+  }
   #cleanup() {
     let changed = false;
     for (const [k, v] of this.#queries) {

packages/system/src/nostr-system.ts

@@ -4,9 +4,7 @@ import EventEmitter from "eventemitter3";
 import { FeedCache } from "@snort/shared";
 import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr";
 import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
-import { Query } from "./query";
+import { RequestBuilder } from "./request-builder";
-import { NoteStore } from "./note-collection";
-import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
 import { RelayMetricHandler } from "./relay-metric-handler";
 import {
   CachedMetadata,
@@ -23,12 +21,10 @@ import {
   QueryLike,
 } from ".";
 import { EventsCache } from "./cache/events";
-import { RelayCache, RelayMetadataLoader } from "./outbox-model";
+import { RelayMetadataLoader } from "./outbox-model";
 import { Optimizer, DefaultOptimizer } from "./query-optimizer";
-import { trimFilters } from "./request-trim";
 import { NostrConnectionPool } from "./nostr-connection-pool";
 import { NostrQueryManager } from "./nostr-query-manager";
-import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer";
 
 export interface NostrSystemEvents {
   change: (state: SystemSnapshot) => void;
@@ -52,77 +48,67 @@ export interface NostrsystemProps {
  */
 export class NostrSystem extends EventEmitter<NostrSystemEvents> implements SystemInterface {
   #log = debug("System");
-  #pool = new NostrConnectionPool();
   #queryManager: NostrQueryManager;
 
   /**
    * Storage class for user relay lists
    */
-  #relayCache: FeedCache<UsersRelays>;
+  readonly relayCache: FeedCache<UsersRelays>;
 
   /**
    * Storage class for user profiles
    */
-  #profileCache: FeedCache<CachedMetadata>;
+  readonly profileCache: FeedCache<CachedMetadata>;
 
   /**
    * Storage class for relay metrics (connects/disconnects)
    */
-  #relayMetricsCache: FeedCache<RelayMetrics>;
+  readonly relayMetricsCache: FeedCache<RelayMetrics>;
 
   /**
    * Profile loading service
    */
-  #profileLoader: ProfileLoaderService;
+  readonly profileLoader: ProfileLoaderService;
 
   /**
    * Relay metrics handler cache
    */
-  #relayMetrics: RelayMetricHandler;
+  readonly relayMetricsHandler: RelayMetricHandler;
-
-  /**
-   * General events cache
-   */
-  #eventsCache: FeedCache<NostrEvent>;
 
   /**
    * Optimizer instance, contains optimized functions for processing data
    */
-  #optimizer: Optimizer;
+  readonly optimizer: Optimizer;
+
+  readonly pool = new NostrConnectionPool();
+  readonly eventsCache: FeedCache<NostrEvent>;
+  readonly relayLoader: RelayMetadataLoader;
 
   /**
    * Check event signatures (reccomended)
    */
   checkSigs: boolean;
 
-  #relayLoader: RelayMetadataLoader;
-
-  /**
-   * Query cache processing layers which can take data from a cache
-   */
-  #queryCacheLayers: Array<FilterCacheLayer> = [];
-
   constructor(props: NostrsystemProps) {
     super();
-    this.#relayCache = props.relayCache ?? new UserRelaysCache(props.db?.userRelays);
+    this.relayCache = props.relayCache ?? new UserRelaysCache(props.db?.userRelays);
-    this.#profileCache = props.profileCache ?? new UserProfileCache(props.db?.users);
+    this.profileCache = props.profileCache ?? new UserProfileCache(props.db?.users);
-    this.#relayMetricsCache = props.relayMetrics ?? new RelayMetricCache(props.db?.relayMetrics);
+    this.relayMetricsCache = props.relayMetrics ?? new RelayMetricCache(props.db?.relayMetrics);
-    this.#eventsCache = props.eventsCache ?? new EventsCache(props.db?.events);
+    this.eventsCache = props.eventsCache ?? new EventsCache(props.db?.events);
-    this.#optimizer = props.optimizer ?? DefaultOptimizer;
+    this.optimizer = props.optimizer ?? DefaultOptimizer;
 
-    this.#profileLoader = new ProfileLoaderService(this, this.#profileCache);
+    this.profileLoader = new ProfileLoaderService(this, this.profileCache);
-    this.#relayMetrics = new RelayMetricHandler(this.#relayMetricsCache);
+    this.relayMetricsHandler = new RelayMetricHandler(this.relayMetricsCache);
-    this.#relayLoader = new RelayMetadataLoader(this, this.#relayCache);
+    this.relayLoader = new RelayMetadataLoader(this, this.relayCache);
     this.checkSigs = props.checkSigs ?? true;
 
     this.#queryManager = new NostrQueryManager(this);
-    this.#queryCacheLayers.push(new IdsFilterCacheLayer(this.#eventsCache));
 
     // hook connection pool
-    this.#pool.on("connected", (id, wasReconnect) => {
+    this.pool.on("connected", (id, wasReconnect) => {
-      const c = this.#pool.getConnection(id);
+      const c = this.pool.getConnection(id);
       if (c) {
-        this.#relayMetrics.onConnect(c.Address);
+        this.relayMetricsHandler.onConnect(c.Address);
         if (wasReconnect) {
           for (const [, q] of this.#queryManager) {
             q.connectionRestored(c);
@@ -130,18 +116,18 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
         }
       }
     });
-    this.#pool.on("connectFailed", address => {
+    this.pool.on("connectFailed", address => {
-      this.#relayMetrics.onDisconnect(address, 0);
+      this.relayMetricsHandler.onDisconnect(address, 0);
     });
-    this.#pool.on("event", (_, sub, ev) => {
+    this.pool.on("event", (_, sub, ev) => {
-      ev.relays?.length && this.#relayMetrics.onEvent(ev.relays[0]);
+      ev.relays?.length && this.relayMetricsHandler.onEvent(ev.relays[0]);
 
       if (!EventExt.isValid(ev)) {
         this.#log("Rejecting invalid event %O", ev);
         return;
       }
       if (this.checkSigs) {
-        if (!this.#optimizer.schnorrVerify(ev)) {
+        if (!this.optimizer.schnorrVerify(ev)) {
           this.#log("Invalid sig %O", ev);
           return;
         }
@@ -149,84 +135,68 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
 
       this.emit("event", sub, ev);
     });
-    this.#pool.on("disconnect", (id, code) => {
+    this.pool.on("disconnect", (id, code) => {
-      const c = this.#pool.getConnection(id);
+      const c = this.pool.getConnection(id);
       if (c) {
-        this.#relayMetrics.onDisconnect(c.Address, code);
+        this.relayMetricsHandler.onDisconnect(c.Address, code);
         for (const [, q] of this.#queryManager) {
           q.connectionLost(c.Id);
         }
       }
     });
-    this.#pool.on("eose", (id, sub) => {
+    this.pool.on("eose", (id, sub) => {
-      const c = this.#pool.getConnection(id);
+      const c = this.pool.getConnection(id);
       if (c) {
         for (const [, v] of this.#queryManager) {
           v.eose(sub, c);
         }
       }
     });
-    this.#pool.on("auth", (_, c, r, cb) => this.emit("auth", c, r, cb));
+    this.pool.on("auth", (_, c, r, cb) => this.emit("auth", c, r, cb));
-    this.#pool.on("notice", (addr, msg) => {
+    this.pool.on("notice", (addr, msg) => {
       this.#log("NOTICE: %s %s", addr, msg);
     });
     this.#queryManager.on("change", () => this.emit("change", this.takeSnapshot()));
-    this.#queryManager.on("sendQuery", (q, f) => this.#sendQuery(q, f));
     this.#queryManager.on("trace", t => {
-      this.#relayMetrics.onTraceReport(t);
+      this.relayMetricsHandler.onTraceReport(t);
     });
 
     // internal handler for on-event
     this.on("event", (sub, ev) => {
       for (const [, v] of this.#queryManager) {
         const trace = v.handleEvent(sub, ev);
+        // inject events to cache if query by id
         if (trace && trace.filters.some(a => a.ids)) {
-          this.#eventsCache.set(ev);
+          this.eventsCache.set(ev);
         }
       }
     });
   }
 
-  get ProfileLoader() {
-    return this.#profileLoader;
-  }
-
   get Sockets(): ConnectionStateSnapshot[] {
-    return this.#pool.getState();
+    return this.pool.getState();
-  }
-
-  get RelayCache(): RelayCache {
-    return this.#relayCache;
-  }
-
-  get UserProfileCache(): FeedCache<CachedMetadata> {
-    return this.#profileCache;
-  }
-
-  get Optimizer(): Optimizer {
-    return this.#optimizer;
   }
 
   async Init() {
     const t = [
-      this.#relayCache.preload(),
+      this.relayCache.preload(),
-      this.#profileCache.preload(),
+      this.profileCache.preload(),
-      this.#relayMetricsCache.preload(),
+      this.relayMetricsCache.preload(),
-      this.#eventsCache.preload(),
+      this.eventsCache.preload(),
     ];
     await Promise.all(t);
   }
 
   async ConnectToRelay(address: string, options: RelaySettings) {
-    await this.#pool.connect(address, options, false);
+    await this.pool.connect(address, options, false);
   }
 
   ConnectEphemeralRelay(address: string) {
-    return this.#pool.connect(address, { read: true, write: true }, true);
+    return this.pool.connect(address, { read: true, write: true }, true);
   }
 
   DisconnectRelay(address: string) {
-    this.#pool.disconnect(address);
+    this.pool.disconnect(address);
   }
 
   GetQuery(id: string): QueryLike | undefined {
@@ -237,60 +207,8 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
     return this.#queryManager.fetch(req, cb);
   }
 
-  Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): QueryLike {
+  Query(req: RequestBuilder): QueryLike {
-    return this.#queryManager.query(type, req) as QueryLike;
+    return this.#queryManager.query(req) as QueryLike;
-  }
-
-  async #sendQuery(q: Query, qSend: BuiltRawReqFilter) {
-    for (const qfl of this.#queryCacheLayers) {
-      qSend = await qfl.processFilter(q, qSend);
-    }
-    for (const f of qSend.filters) {
-      if (f.authors) {
-        this.#relayLoader.TrackKeys(f.authors);
-      }
-    }
-
-    // check for empty filters
-    const fNew = trimFilters(qSend.filters);
-    if (fNew.length === 0) {
-      return;
-    }
-    qSend.filters = fNew;
-
-    if (qSend.relay) {
-      this.#log("Sending query to %s %O", qSend.relay, qSend);
-      const s = this.#pool.getConnection(qSend.relay);
-      if (s) {
-        const qt = q.sendToRelay(s, qSend);
-        if (qt) {
-          return [qt];
-        }
-      } else {
-        const nc = await this.ConnectEphemeralRelay(qSend.relay);
-        if (nc) {
-          const qt = q.sendToRelay(nc, qSend);
-          if (qt) {
-            return [qt];
-          }
-        } else {
-          console.warn("Failed to connect to new relay for:", qSend.relay, q);
-        }
-      }
-    } else {
-      const ret = [];
-      for (const [a, s] of this.#pool) {
-        if (!s.Ephemeral) {
-          this.#log("Sending query to %s %O", a, qSend);
-          const qt = q.sendToRelay(s, qSend);
-          if (qt) {
-            ret.push(qt);
-          }
-        }
-      }
-      return ret;
-    }
-    return [];
   }
 
   HandleEvent(ev: TaggedNostrEvent) {
@@ -299,11 +217,11 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
 
   async BroadcastEvent(ev: NostrEvent, cb?: (rsp: OkResponse) => void): Promise<OkResponse[]> {
     this.HandleEvent({ ...ev, relays: [] });
-    return await this.#pool.broadcast(this, ev, cb);
+    return await this.pool.broadcast(this, ev, cb);
   }
 
   async WriteOnceToRelay(address: string, ev: NostrEvent): Promise<OkResponse> {
-    return await this.#pool.broadcastTo(address, ev);
+    return await this.pool.broadcastTo(address, ev);
   }
 
   takeSnapshot(): SystemSnapshot {

packages/system/src/note-collection.ts

@@ -3,7 +3,7 @@ import { EventExt, EventType, TaggedNostrEvent, u256 } from ".";
 import { findTag } from "./utils";
 import EventEmitter from "eventemitter3";
 
-export interface StoreSnapshot<TSnapshot> {
+export interface StoreSnapshot<TSnapshot extends NoteStoreSnapshotData> {
   data: TSnapshot | undefined;
   clear: () => void;
   loading: () => boolean;
@@ -19,7 +19,7 @@ export const EmptySnapshot = {
   add: () => {
     // empty
   },
-} as StoreSnapshot<FlatNoteStore>;
+} as StoreSnapshot<Array<TaggedNostrEvent>>;
 
 export type NoteStoreSnapshotData = Array<TaggedNostrEvent> | TaggedNostrEvent;
 export type NoteStoreHook = () => void;

packages/system/src/outbox-model.ts

@@ -205,7 +205,7 @@ export function pickTopRelays(cache: RelayCache, authors: Array<string>, n: numb
 export async function pickRelaysForReply(ev: NostrEvent, system: SystemInterface, pickN?: number) {
   const recipients = dedupe(ev.tags.filter(a => a[0] === "p").map(a => a[1]));
   await updateRelayLists(recipients, system);
-  const relays = pickTopRelays(system.RelayCache, recipients, pickN ?? DefaultPickNRelays, "read");
+  const relays = pickTopRelays(system.relayCache, recipients, pickN ?? DefaultPickNRelays, "read");
   const ret = removeUndefined(dedupe(relays.map(a => a.relays).flat()));
   logger("Picked %O from authors %O", ret, recipients);
   return ret;
@@ -247,15 +247,15 @@ export function parseRelaysFromKind(ev: NostrEvent) {
 }
 
 export async function updateRelayLists(authors: Array<string>, system: SystemInterface) {
-  await system.RelayCache.buffer(authors);
+  await system.relayCache.buffer(authors);
   const expire = unixNowMs() - RelayListCacheExpire;
-  const expired = authors.filter(a => (system.RelayCache.getFromCache(a)?.loaded ?? 0) < expire);
+  const expired = authors.filter(a => (system.relayCache.getFromCache(a)?.loaded ?? 0) < expire);
   if (expired.length > 0) {
     logger("Updating relays for authors: %O", expired);
     const rb = new RequestBuilder("system-update-relays-for-outbox");
     rb.withFilter().authors(expired).kinds([EventKind.Relays, EventKind.ContactList]);
     const relayLists = await system.Fetch(rb);
-    await system.RelayCache.bulkSet(
+    await system.relayCache.bulkSet(
       removeUndefined(
         relayLists.map(a => {
           const relays = parseRelaysFromKind(a);

packages/system/src/query.ts

@@ -3,9 +3,9 @@ import debug from "debug";
 import EventEmitter from "eventemitter3";
 import { unixNowMs, unwrap } from "@snort/shared";
 
-import { Connection, ReqFilter, Nips, TaggedNostrEvent } from ".";
+import { Connection, ReqFilter, Nips, TaggedNostrEvent, SystemInterface } from ".";
-import { NoteStore } from "./note-collection";
+import { NoteCollection, NoteStore } from "./note-collection";
-import { BuiltRawReqFilter } from "./request-builder";
+import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
 import { eventMatchesFilter } from "./request-matcher";
 
 interface QueryTraceEvents {
@@ -89,23 +89,6 @@ export class QueryTrace extends EventEmitter<QueryTraceEvents> {
   }
 }
 
-export interface QueryBase {
-  /**
-   * Uniquie ID of this query
-   */
-  id: string;
-
-  /**
-   * The query payload (REQ filters)
-   */
-  filters: Array<ReqFilter>;
-
-  /**
-   * List of relays to send this query to
-   */
-  relays?: Array<string>;
-}
-
 export interface TraceReport {
   id: string;
   conn: Connection;
@@ -116,22 +99,28 @@ export interface TraceReport {
 
 interface QueryEvents {
   trace: (report: TraceReport) => void;
+  filters: (req: BuiltRawReqFilter) => void;
   event: (evs: ReadonlyArray<TaggedNostrEvent>) => void;
 }
 
 /**
  * Active or queued query on the system
  */
-export class Query extends EventEmitter<QueryEvents> implements QueryBase {
+export class Query extends EventEmitter<QueryEvents> {
   /**
-   * Uniquie ID of this query
+   * Unique id of this query
    */
-  id: string;
+  readonly id: string;
 
   /**
    * RequestBuilder instance
    */
-  fromInstance: string;
+  requests: Array<RequestBuilder> = [];
+
+  /**
+   * Nostr system interface
+   */
+  #system: SystemInterface;
 
   /**
    * Which relays this query has already been executed on
@@ -156,27 +145,66 @@ export class Query extends EventEmitter<QueryEvents> implements QueryBase {
   /**
    * Feed object which collects events
    */
-  #feed: NoteStore;
+  #feed: NoteCollection;
 
   /**
    * Maximum waiting time for this query
    */
   #timeout: number;
 
+  /**
+   * Milliseconds to wait before sending query (debounce)
+   */
+  #groupingDelay?: number;
+
+  /**
+   * Timer which waits for no-change before emitting filters
+   */
+  #groupTimeout?: ReturnType<typeof setTimeout>;
+
   #log = debug("Query");
 
-  constructor(id: string, instance: string, feed: NoteStore, leaveOpen?: boolean, timeout?: number) {
+  constructor(system: SystemInterface, req: RequestBuilder) {
     super();
-    this.id = id;
+    this.id = uuid();
-    this.#feed = feed;
+    this.requests.push(req);
-    this.fromInstance = instance;
+    this.#system = system;
-    this.#leaveOpen = leaveOpen ?? false;
+    this.#feed = new NoteCollection();
-    this.#timeout = timeout ?? 5_000;
+    this.#leaveOpen = req.options?.leaveOpen ?? false;
+    this.#timeout = req.options?.timeout ?? 5_000;
+    this.#groupingDelay = req.options?.groupingDelay ?? 100;
     this.#checkTraces();
 
     this.feed.on("event", evs => this.emit("event", evs));
   }
 
+  /**
+   * Adds another request to this one
+   */
+  addRequest(req: RequestBuilder) {
+    if (this.#groupTimeout) {
+      clearTimeout(this.#groupTimeout);
+      this.#groupTimeout = undefined;
+    }
+    if (this.requests.some(a => a.instance === req.instance)) {
+      // already exists, nothing to add
+      return false;
+    }
+    if (this.requests.some(a => a.options?.skipDiff !== req.options?.skipDiff)) {
+      throw new Error("Mixing skipDiff option is not supported");
+    }
+    this.requests.push(req);
+
+    if (this.#groupingDelay) {
+      this.#groupTimeout = setTimeout(() => {
+        this.#emitFilters();
+      }, this.#groupingDelay);
+    } else {
+      this.#emitFilters();
+    }
+    return true;
+  }
+
   isOpen() {
     return this.#cancelAt === undefined && this.#leaveOpen;
   }
@@ -232,7 +260,7 @@ export class Query extends EventEmitter<QueryEvents> implements QueryBase {
   /**
    * Insert a new trace as a placeholder
    */
-  insertCompletedTrace(subq: BuiltRawReqFilter, data: Readonly<Array<TaggedNostrEvent>>) {
+  insertCompletedTrace(subq: BuiltRawReqFilter, data: Array<TaggedNostrEvent>) {
     const qt = new QueryTrace(subq.relay, subq.filters, "");
     qt.sentToRelay();
     qt.gotEose();
@@ -288,6 +316,24 @@ export class Query extends EventEmitter<QueryEvents> implements QueryBase {
     return thisProgress;
   }
 
+  #emitFilters() {
+    if (this.requests.every(a => !!a.options?.skipDiff)) {
+      const existing = this.filters;
+      const rb = new RequestBuilder(this.id);
+      this.requests.forEach(a => rb.add(a));
+      const filters = rb.buildDiff(this.#system, existing);
+      filters.forEach(f => this.emit("filters", f));
+      this.requests = [];
+    } else {
+      // send without diff
+      const rb = new RequestBuilder(this.id);
+      this.requests.forEach(a => rb.add(a));
+      const filters = rb.build(this.#system);
+      filters.forEach(f => this.emit("filters", f));
+      this.requests = [];
+    }
+  }
+
   #onProgress() {
     const isFinished = this.progress === 1;
     if (this.feed.loading !== isFinished) {

packages/system/src/relay-metric-handler.ts

@@ -1,7 +1,6 @@
 import { FeedCache, unixNowMs } from "@snort/shared";
-import { Connection } from "connection";
+import { RelayMetrics } from "./cache";
-import { RelayMetrics } from "cache";
+import { TraceReport } from "./query";
-import { TraceReport } from "query";
 
 export class RelayMetricHandler {
   readonly #cache: FeedCache<RelayMetrics>;

packages/system/src/request-builder.ts

@@ -53,6 +53,11 @@ export interface RequestBuilderOptions {
    * Max wait time for this request
    */
   timeout?: number;
+
+  /**
+   * How many milli-seconds to wait to allow grouping
+   */
+  groupingDelay?: number;
 }
 
 /**
@@ -111,7 +116,7 @@ export class RequestBuilder {
   }
 
   build(system: SystemInterface): Array<BuiltRawReqFilter> {
-    const expanded = this.#builders.flatMap(a => a.build(system.RelayCache, this.#options));
+    const expanded = this.#builders.flatMap(a => a.build(system.relayCache, this.#options));
     return this.#groupByRelay(system, expanded);
   }
 
@@ -121,14 +126,14 @@ export class RequestBuilder {
   buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Array<BuiltRawReqFilter> {
     const start = unixNowMs();
 
-    const diff = system.Optimizer.getDiff(prev, this.buildRaw());
+    const diff = system.optimizer.getDiff(prev, this.buildRaw());
     const ts = unixNowMs() - start;
     this.#log("buildDiff %s %d ms +%d", this.id, ts, diff.length);
     if (diff.length > 0) {
-      return splitFlatByWriteRelays(system.RelayCache, diff).map(a => {
+      return splitFlatByWriteRelays(system.relayCache, diff).map(a => {
         return {
           strategy: RequestStrategy.AuthorsRelays,
-          filters: system.Optimizer.flatMerge(a.filters),
+          filters: system.optimizer.flatMerge(a.filters),
           relay: a.relay,
         };
       });
@@ -154,7 +159,7 @@ export class RequestBuilder {
 
     const filtersSquashed = [...relayMerged.values()].map(a => {
       return {
-        filters: system.Optimizer.flatMerge(a.flatMap(b => b.filters.flatMap(c => system.Optimizer.expandFilter(c)))),
+        filters: system.optimizer.flatMerge(a.flatMap(b => b.filters.flatMap(c => system.optimizer.expandFilter(c)))),
         relay: a[0].relay,
         strategy: a[0].strategy,
       } as BuiltRawReqFilter;

packages/system/src/worker/system-worker.ts

@@ -3,11 +3,8 @@ import EventEmitter from "eventemitter3";
 import {
   ConnectionStateSnapshot,
   NostrEvent,
-  NoteStore,
   OkResponse,
   ProfileLoaderService,
-  Optimizer,
-  RelayCache,
   RelaySettings,
   RequestBuilder,
   SystemInterface,
@@ -28,18 +25,19 @@ import { FeedCache } from "@snort/shared";
 import { EventsCache } from "../cache/events";
 import { RelayMetricHandler } from "../relay-metric-handler";
 import debug from "debug";
+import { ConnectionPool } from "nostr-connection-pool";
 
 export class SystemWorker extends EventEmitter<NostrSystemEvents> implements SystemInterface {
   #log = debug("SystemWorker");
   #worker: Worker;
   #commandQueue: Map<string, (v: unknown) => void> = new Map();
-  #relayCache: FeedCache<UsersRelays>;
+  readonly relayCache: FeedCache<UsersRelays>;
-  #profileCache: FeedCache<CachedMetadata>;
+  readonly profileCache: FeedCache<CachedMetadata>;
-  #relayMetricsCache: FeedCache<RelayMetrics>;
+  readonly relayMetricsCache: FeedCache<RelayMetrics>;
-  #profileLoader: ProfileLoaderService;
+  readonly profileLoader: ProfileLoaderService;
-  #relayMetrics: RelayMetricHandler;
+  readonly relayMetricsHandler: RelayMetricHandler;
-  #eventsCache: FeedCache<NostrEvent>;
+  readonly eventsCache: FeedCache<NostrEvent>;
-  #relayLoader: RelayMetadataLoader;
+  readonly relayLoader: RelayMetadataLoader;
 
   get checkSigs() {
     return true;
@@ -49,17 +47,25 @@ export class SystemWorker extends EventEmitter<NostrSystemEvents> implements Sys
     // not used
   }
 
+  get optimizer() {
+    return DefaultOptimizer;
+  }
+
+  get pool() {
+    return {} as ConnectionPool;
+  }
+
   constructor(scriptPath: string, props: NostrsystemProps) {
     super();
 
-    this.#relayCache = props.relayCache ?? new UserRelaysCache(props.db?.userRelays);
+    this.relayCache = props.relayCache ?? new UserRelaysCache(props.db?.userRelays);
-    this.#profileCache = props.profileCache ?? new UserProfileCache(props.db?.users);
+    this.profileCache = props.profileCache ?? new UserProfileCache(props.db?.users);
-    this.#relayMetricsCache = props.relayMetrics ?? new RelayMetricCache(props.db?.relayMetrics);
+    this.relayMetricsCache = props.relayMetrics ?? new RelayMetricCache(props.db?.relayMetrics);
-    this.#eventsCache = props.eventsCache ?? new EventsCache(props.db?.events);
+    this.eventsCache = props.eventsCache ?? new EventsCache(props.db?.events);
 
-    this.#profileLoader = new ProfileLoaderService(this, this.#profileCache);
+    this.profileLoader = new ProfileLoaderService(this, this.profileCache);
-    this.#relayMetrics = new RelayMetricHandler(this.#relayMetricsCache);
+    this.relayMetricsHandler = new RelayMetricHandler(this.relayMetricsCache);
-    this.#relayLoader = new RelayMetadataLoader(this, this.#relayCache);
+    this.relayLoader = new RelayMetadataLoader(this, this.relayCache);
     this.#worker = new Worker(scriptPath, {
       name: "SystemWorker",
       type: "module",
@@ -86,7 +92,7 @@ export class SystemWorker extends EventEmitter<NostrSystemEvents> implements Sys
     return undefined;
   }
 
-  Query<T extends NoteStore>(type: new () => T, req: RequestBuilder): QueryLike {
+  Query(req: RequestBuilder): QueryLike {
     const chan = this.#workerRpc<[RequestBuilder], { id: string; port: MessagePort }>(WorkerCommand.Query, [req]);
     return {
       on: (_: "event", cb) => {
@@ -130,18 +136,6 @@ export class SystemWorker extends EventEmitter<NostrSystemEvents> implements Sys
     throw new Error("Method not implemented.");
   }
 
-  get ProfileLoader(): ProfileLoaderService {
-    return this.#profileLoader;
-  }
-
-  get RelayCache(): RelayCache {
-    return this.#relayCache;
-  }
-
-  get Optimizer(): Optimizer {
-    return DefaultOptimizer;
-  }
-
   #workerRpc<T, R>(type: WorkerCommand, data?: T, timeout = 5_000) {
     const id = uuid();
     const msg = {

packages/system/worker.ts

@@ -1,21 +0,0 @@
-/// <reference lib="webworker" />
-import { UsersRelaysCache } from "../Cache/UserRelayCache";
-import { NostrSystem } from ".";
-declare const self: SharedWorkerGlobalScope;
-
-const RelayCache = new UsersRelaysCache();
-const System = new NostrSystem({
-  get: pk => RelayCache.getFromCache(pk)?.relays,
-});
-
-self.onconnect = e => {
-  const port = e.ports[0];
-
-  port.addEventListener("message", async e1 => {
-    console.debug(e1);
-    const [cmd, ...others] = e1.data;
-    switch (cmd) {
-    }
-  });
-  port.start();
-};