feat: remove buildDiff

packages/app/src/Components/Event/Note/NoteFooter/NoteFooter.tsx

@@ -21,7 +21,7 @@ export default function NoteFooter(props: NoteFooterProps) {
   const link = useMemo(() => NostrLink.fromEvent(ev), [ev.id]);
   const [showReactions, setShowReactions] = useState(false);
 
-  const related = useReactions(`reactions:${link.tagKey}`, link);
+  const related = useReactions("reactions", link);
   const { replies, reactions, zaps, reposts } = useEventReactions(link, related);
   const { positive } = reactions;
 

packages/app/src/Components/Event/Note/ReactionsModal.tsx

@@ -25,7 +25,7 @@ const ReactionsModal = ({ onClose, event, initialTab = 0 }: ReactionsModalProps)
 
   const link = NostrLink.fromEvent(event);
 
-  const related = useReactions(`reactions:${link.tagKey}`, link, undefined, false);
+  const related = useReactions("reactions", link, undefined, false);
   const { reactions, zaps, reposts } = useEventReactions(link, related);
   const { positive, negative } = reactions;
 

packages/system/src/outbox/relay-loader.ts

@@ -27,7 +27,6 @@ export class RelayMetadataLoader extends BackgroundLoader<UsersRelays> {
   protected override buildSub(missing: string[]): RequestBuilder {
     const rb = new RequestBuilder("relay-loader");
     rb.withOptions({
-      skipDiff: true,
       timeout: 10000,
       outboxPickN: 4,
     });

packages/system/src/query-manager.ts

@@ -76,7 +76,7 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
    * Async fetch results
    */
   async fetch(req: RequestBuilder, cb?: (evs: Array<TaggedNostrEvent>) => void) {
-    const filters = req.buildRaw(this.#system);
+    const filters = req.buildRaw();
     const q = this.query(req);
     if (cb) {
       q.on("event", cb);
@@ -101,15 +101,6 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
     for (const qfl of this.#queryCacheLayers) {
       qSend = await qfl.processFilter(q, qSend);
     }
-    if (this.#system.cacheRelay) {
-      // fetch results from cache first, flag qSend for sync
-      const data = await this.#system.cacheRelay.query(["REQ", q.id, ...qSend.filters]);
-      if (data.length > 0) {
-        qSend.syncFrom = data as Array<TaggedNostrEvent>;
-        this.#log("Adding from cache: %O", data);
-        q.feed.add(data.map(a => ({ ...a, relays: [] })));
-      }
-    }
 
     // automated outbox model, load relays for queried authors
     for (const f of qSend.filters) {
@@ -119,19 +110,42 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
     }
 
     // check for empty filters
-    const fNew = trimFilters(qSend.filters);
-    if (fNew.length === 0) {
+    qSend.filters = trimFilters(qSend.filters);
+
+    // fetch results from cache first, flag qSend for sync
+    if (this.#system.cacheRelay) {
+      const data = await this.#system.cacheRelay.query(["REQ", q.id, ...qSend.filters]);
+      if (data.length > 0) {
+        qSend.syncFrom = data as Array<TaggedNostrEvent>;
+        this.#log("Adding from cache: %O", data);
+        q.feed.add(data.map(a => ({ ...a, relays: [] })));
+      }
+    }
+
+    // remove satisfied filters
+    if (qSend.syncFrom && qSend.syncFrom.length > 0) {
+      // only remove the "ids" filters
+      const newFilters = qSend.filters.filter(
+        a => !a.ids || (a.ids && !qSend.syncFrom?.some(b => eventMatchesFilter(b, a))),
+      );
+      if (newFilters.length !== qSend.filters.length) {
+        this.#log("Removing satisfied filters %o %o", newFilters, qSend.filters);
+        qSend.filters = newFilters;
+      }
+    }
+
+    // nothing left to send
+    if (qSend.filters.length === 0) {
       this.#log("Dropping %s %o", q.id, qSend);
       return;
     }
-    qSend.filters = fNew;
 
     if (qSend.relay) {
-      this.#log("Sending query to %s %s %O", qSend.relay, q.id, qSend);
       const nc = await this.#system.pool.connect(qSend.relay, { read: true, write: true }, true);
       if (nc) {
         const qt = q.sendToRelay(nc, qSend);
         if (qt) {
+          this.#log("Sent query %s to %s %s %O", qt.id, qSend.relay, q.id, qSend);
           return [qt];
         } else {
           this.#log("Query not sent to %s: %O", qSend.relay, qSend);
@@ -143,9 +157,9 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
       const ret = [];
       for (const [a, s] of this.#system.pool) {
         if (!s.ephemeral) {
-          this.#log("Sending query to %s %s %O", a, q.id, qSend);
           const qt = q.sendToRelay(s, qSend);
           if (qt) {
+            this.#log("Sent query %s to %s %s %O", qt.id, qSend.relay, q.id, qSend);
             ret.push(qt);
           } else {
             this.#log("Query not sent to %s: %O", a, qSend);

packages/system/src/query.ts

@@ -3,11 +3,10 @@ import debug from "debug";
 import { EventEmitter } from "eventemitter3";
 import { unixNowMs, unwrap } from "@snort/shared";
 
-import { ReqFilter, Nips, TaggedNostrEvent, SystemInterface, ParsedFragment } from ".";
+import { ReqFilter, Nips, TaggedNostrEvent, SystemInterface, ParsedFragment, FlatReqFilter } from ".";
 import { NoteCollection } from "./note-collection";
 import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
 import { eventMatchesFilter } from "./request-matcher";
-import { LRUCache } from "lru-cache";
 import { ConnectionType } from "./connection-pool";
 
 interface QueryTraceEvents {
@@ -103,23 +102,16 @@ export interface QueryEvents {
   done: () => void;
 }
 
-const QueryCache = new LRUCache<string, Array<TaggedNostrEvent>>({
-  ttl: 60_000 * 3,
-  ttlAutopurge: true,
-});
-
 /**
  * Active or queued query on the system
  */
 export class Query extends EventEmitter<QueryEvents> {
-  get id() {
-    return this.request.id;
-  }
+  id: string;
 
   /**
    * RequestBuilder instance
    */
-  request: RequestBuilder;
+  requests: Array<ReqFilter> = [];
 
   /**
    * Nostr system interface
@@ -175,7 +167,7 @@ export class Query extends EventEmitter<QueryEvents> {
 
   constructor(system: SystemInterface, req: RequestBuilder) {
     super();
-    this.request = req;
+    this.id = req.id;
     this.#system = system;
     this.#feed = new NoteCollection();
     this.#leaveOpen = req.options?.leaveOpen ?? false;
@@ -183,11 +175,7 @@ export class Query extends EventEmitter<QueryEvents> {
     this.#groupingDelay = req.options?.groupingDelay ?? 100;
     this.#checkTraces();
 
-    const cached = QueryCache.get(this.request.id);
-    if (cached) {
-      this.#log("Restored %o for %s", cached, this.request.id);
-      this.feed.add(cached);
-    }
+    this.requests.push(...req.buildRaw());
     this.feed.on("event", evs => this.emit("event", evs));
     this.#start();
   }
@@ -196,12 +184,8 @@ export class Query extends EventEmitter<QueryEvents> {
    * Adds another request to this one
    */
   addRequest(req: RequestBuilder) {
-    if (req.instance === this.request.instance) {
-      // same requst, do nothing
-      return;
-    }
     this.#log("Add query %O to %s", req, this.id);
-    this.request.add(req);
+    this.requests.push(...req.buildRaw());
     this.#start();
     return true;
   }
@@ -263,8 +247,6 @@ export class Query extends EventEmitter<QueryEvents> {
     }
     this.#stopCheckTraces();
     this.emit("end");
-    QueryCache.set(this.request.id, this.feed.snapshot);
-    this.#log("Saved %O for %s", this.feed.snapshot, this.request.id);
   }
 
   /**
@@ -345,16 +327,39 @@ export class Query extends EventEmitter<QueryEvents> {
 
   async #emitFilters() {
     this.#log("Starting emit of %s", this.id);
-    const existing = this.filters;
-    if (!(this.request.options?.skipDiff ?? false) && existing.length > 0) {
-      const filters = this.request.buildDiff(this.#system, existing);
-      this.#log("Build %s %O", this.id, filters);
-      filters.forEach(f => this.emit("request", this.id, f));
-    } else {
-      const filters = this.request.build(this.#system);
-      this.#log("Build %s %O", this.id, filters);
-      filters.forEach(f => this.emit("request", this.id, f));
+    let rawFilters = [...this.requests];
+    this.requests = [];
+    if (this.#system.requestRouter) {
+      rawFilters = this.#system.requestRouter.forAllRequest(rawFilters);
     }
+    const expanded = rawFilters.flatMap(a => this.#system.optimizer.expandFilter(a));
+    const fx = this.#groupFlatByRelay(expanded);
+    fx.forEach(a => this.emit("request", this.id, a));
+  }
+
+  #groupFlatByRelay(filters: Array<FlatReqFilter>) {
+    const relayMerged = filters.reduce((acc, v) => {
+      const relay = v.relay ?? "";
+      // delete relay from filter
+      delete v.relay;
+      const existing = acc.get(relay);
+      if (existing) {
+        existing.push(v);
+      } else {
+        acc.set(relay, [v]);
+      }
+      return acc;
+    }, new Map<string, Array<FlatReqFilter>>());
+
+    const ret = [];
+    for (const [k, v] of relayMerged.entries()) {
+      const filters = this.#system.optimizer.flatMerge(v);
+      ret.push({
+        relay: k,
+        filters,
+      } as BuiltRawReqFilter);
+    }
+    return ret;
   }
 
   #stopCheckTraces() {
@@ -419,7 +424,7 @@ export class Query extends EventEmitter<QueryEvents> {
       }
     });
     const eventHandler = (sub: string, ev: TaggedNostrEvent) => {
-      if ((this.request.options?.fillStore ?? true) && qt.id === sub) {
+      if (qt.id === sub) {
         if (qt.filters.some(v => eventMatchesFilter(ev, v))) {
           this.feed.add(ev);
         } else {

packages/system/src/request-builder.ts

@@ -1,9 +1,8 @@
-import debug from "debug";
 import { v4 as uuid } from "uuid";
-import { appendDedupe, dedupe, removeUndefined, sanitizeRelayUrl, unixNowMs, unwrap } from "@snort/shared";
+import { appendDedupe, dedupe, removeUndefined, sanitizeRelayUrl, unwrap } from "@snort/shared";
 
 import EventKind from "./event-kind";
-import { FlatReqFilter, NostrLink, NostrPrefix, SystemInterface, ToNostrEventTag } from ".";
+import { NostrLink, NostrPrefix, ToNostrEventTag } from ".";
 import { ReqFilter, u256, HexKey, TaggedNostrEvent } from "./nostr";
 import { RequestRouter } from "./request-router";
 
@@ -23,11 +22,6 @@ export interface RequestBuilderOptions {
    */
   leaveOpen?: boolean;
 
-  /**
-   * Do not apply diff logic and always use full filters for query
-   */
-  skipDiff?: boolean;
-
   /**
    * Pick N relays per pubkey when using outbox strategy
    */
@@ -42,12 +36,6 @@ export interface RequestBuilderOptions {
    * How many milli-seconds to wait to allow grouping
    */
   groupingDelay?: number;
-
-  /**
-   * If events should be added automatically to the internal NoteCollection
-   * default=true
-   */
-  fillStore?: boolean;
 }
 
 /**
@@ -58,8 +46,6 @@ export class RequestBuilder {
   instance: string;
   #builders: Array<RequestFilterBuilder>;
   #options?: RequestBuilderOptions;
-  #log = debug("RequestBuilder");
-  #rawCached?: Array<ReqFilter>;
 
   constructor(id: string) {
     this.instance = uuid();
@@ -84,20 +70,17 @@ export class RequestBuilder {
    */
   add(other: RequestBuilder) {
     this.#builders.push(...other.#builders);
-    this.#rawCached = undefined;
   }
 
   withFilter() {
     const ret = new RequestFilterBuilder();
     this.#builders.push(ret);
-    this.#rawCached = undefined;
     return ret;
   }
 
   withBareFilter(f: ReqFilter) {
     const ret = new RequestFilterBuilder(f);
     this.#builders.push(ret);
-    this.#rawCached = undefined;
     return ret;
   }
 
@@ -109,67 +92,8 @@ export class RequestBuilder {
     return this;
   }
 
-  buildRaw(system?: SystemInterface): Array<ReqFilter> {
-    if (!this.#rawCached && system) {
-      this.#rawCached = system.optimizer.compress(this.#builders.map(f => f.filter));
-    }
-    return this.#rawCached ?? this.#builders.map(f => f.filter);
-  }
-
-  build(system: SystemInterface): Array<BuiltRawReqFilter> {
-    let rawFilters = this.buildRaw(system);
-    if (system.requestRouter) {
-      rawFilters = system.requestRouter.forAllRequest(rawFilters);
-    }
-    const expanded = rawFilters.flatMap(a => system.optimizer.expandFilter(a));
-    return this.#groupFlatByRelay(system, expanded);
-  }
-
-  /**
-   * Detects a change in request from a previous set of filters
-   */
-  buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Array<BuiltRawReqFilter> {
-    const start = unixNowMs();
-
-    let rawFilters = this.buildRaw(system);
-    if (system.requestRouter) {
-      rawFilters = system.requestRouter.forAllRequest(rawFilters);
-    }
-    const diff = system.optimizer.getDiff(prev, rawFilters);
-    if (diff.length > 0) {
-      const ret = this.#groupFlatByRelay(system, diff);
-      const ts = unixNowMs() - start;
-      if (ts >= 100) {
-        this.#log("slow diff %s %d ms, consider separate query ids, or use skipDiff: %O", this.id, ts, prev);
-      }
-      return ret;
-    }
-    return [];
-  }
-
-  #groupFlatByRelay(system: SystemInterface, filters: Array<FlatReqFilter>) {
-    const relayMerged = filters.reduce((acc, v) => {
-      const relay = v.relay ?? "";
-      // delete relay from filter
-      delete v.relay;
-      const existing = acc.get(relay);
-      if (existing) {
-        existing.push(v);
-      } else {
-        acc.set(relay, [v]);
-      }
-      return acc;
-    }, new Map<string, Array<FlatReqFilter>>());
-
-    const ret = [];
-    for (const [k, v] of relayMerged.entries()) {
-      const filters = system.optimizer.flatMerge(v);
-      ret.push({
-        relay: k,
-        filters,
-      } as BuiltRawReqFilter);
-    }
-    return ret;
+  buildRaw(): Array<ReqFilter> {
+    return this.#builders.map(f => f.filter);
   }
 }