optimize

packages/app/package.json

@@ -48,7 +48,7 @@
   "scripts": {
     "start": "webpack serve",
     "build": "webpack --node-env=production",
-    "test": "jest",
+    "test": "jest --runInBand",
     "intl-extract": "formatjs extract 'src/**/*.ts*' --ignore='**/*.d.ts' --out-file src/lang.json --flatten true",
     "intl-compile": "formatjs compile src/lang.json --out-file src/translations/en.json",
     "format": "prettier --write .",

packages/app/src/Hooks/useRequestBuilder.tsx

@@ -6,35 +6,25 @@ import { System } from "index";
 
 const useRequestBuilder = <TStore extends NoteStore, TSnapshot = ReturnType<TStore["getSnapshotData"]>>(
   type: { new (): TStore },
-  rb: RequestBuilder | null,
-  debounced?: number
+  rb: RequestBuilder | null
 ) => {
   const subscribe = (onChanged: () => void) => {
-    const store = (System.Query<TStore>(type, rb)?.feed as TStore) ?? new type();
-    let t: ReturnType<typeof setTimeout> | undefined;
-    const release = store.hook(() => {
-      if (!t) {
-        t = setTimeout(() => {
-          clearTimeout(t);
-          t = undefined;
-          onChanged();
-        }, debounced ?? 500);
-      }
-    });
-
+    if (rb) {
+      const q = System.Query<TStore>(type, rb);
+      const release = q.feed.hook(onChanged);
+      return () => {
+        q.cancel();
+        release();
+      };
+    }
     return () => {
-      if (rb?.id) {
-        System.GetQuery(rb.id)?.cancel();
-      }
-      release();
+      // noop
     };
   };
   const getState = (): StoreSnapshot<TSnapshot> => {
-    if (rb?.id) {
-      const q = System.GetQuery(rb.id);
-      if (q) {
-        return unwrap(q).feed?.snapshot as StoreSnapshot<TSnapshot>;
-      }
+    const q = System.GetQuery(rb?.id ?? "");
+    if (q) {
+      return unwrap(q).feed?.snapshot as StoreSnapshot<TSnapshot>;
     }
     return EmptySnapshot as StoreSnapshot<TSnapshot>;
   };

packages/app/src/System/NostrSystem.ts

@@ -119,13 +119,11 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
     return this.Queries.get(id);
   }
 
-  Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder | null): Query | undefined {
-    if (!req) return;
-
+  Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): Query {
     const existing = this.Queries.get(req.id);
     if (existing) {
       const filters = req.buildDiff(this.#relayCache, existing.filters);
-      if (filters.length === 0 && !req.options?.skipDiff) {
+      if (filters.length === 0 && !!req.options?.skipDiff) {
         return existing;
       } else {
         for (const subQ of filters) {

packages/app/src/System/NoteCollection.ts

@@ -57,6 +57,7 @@ export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> i
     data: undefined,
   };
   #needsSnapshot = true;
+  #nextNotifyTimer?: ReturnType<typeof setTimeout>;
 
   get snapshot() {
     this.#updateSnapshot();
@@ -106,8 +107,13 @@ export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> i
 
   protected onChange(changes: Readonly<Array<TaggedRawEvent>>): void {
     this.#needsSnapshot = true;
-    for (const hk of this.#hooks) {
-      hk();
+    if (!this.#nextNotifyTimer) {
+      this.#nextNotifyTimer = setTimeout(() => {
+        this.#nextNotifyTimer = undefined;
+        for (const hk of this.#hooks) {
+          hk();
+        }
+      }, 500);
     }
     if (changes.length > 0) {
       for (const hkE of this.#eventHooks) {

packages/app/src/System/Query.test.ts

@@ -101,8 +101,12 @@ describe("query", () => {
 
     expect(q.filters).toEqual([
       {
-        authors: ["a", "b", "c"],
-        kinds: [1, 2],
+        authors: ["a", "b"],
+        kinds: [1],
+      },
+      {
+        authors: ["b", "c"],
+        kinds: [2],
       },
     ]);
   });

packages/app/src/System/Query.ts

@@ -3,9 +3,10 @@ import debug from "debug";
 import { Connection, ReqFilter, Nips, TaggedRawEvent } from "System";
 import { unixNowMs, unwrap } from "SnortUtils";
 import { NoteStore } from "./NoteCollection";
-import { simpleMerge } from "./RequestMerger";
+import { flatMerge, mergeSimilar, simpleMerge } from "./RequestMerger";
 import { eventMatchesFilter } from "./RequestMatcher";
 import { BuiltRawReqFilter } from "./RequestBuilder";
+import { expandFilter } from "./RequestExpander";
 
 /**
  * Tracing for relay query status
@@ -137,6 +138,7 @@ export class Query implements QueryBase {
   #feed: NoteStore;
 
   #log = debug("Query");
+  #allFilters: Array<ReqFilter> = [];
 
   constructor(id: string, feed: NoteStore) {
     this.id = id;
@@ -152,9 +154,11 @@ export class Query implements QueryBase {
     return this.#cancelTimeout;
   }
 
+  /**
+   * Recompute the complete set of compressed filters from all query traces
+   */
   get filters() {
-    const filters = this.#tracing.flatMap(a => a.filters);
-    return [simpleMerge(filters)];
+    return this.#allFilters;
   }
 
   get feed() {
@@ -264,7 +268,13 @@ export class Query implements QueryBase {
       () => this.#onProgress()
     );
     this.#tracing.push(qt);
+    this.#reComputeFilters();
     c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
     return qt;
   }
+  #reComputeFilters() {
+    console.time("reComputeFilters");
+    this.#allFilters = flatMerge(this.#tracing.flatMap(a => a.filters).flatMap(expandFilter));
+    console.timeEnd("reComputeFilters");
+  }
 }

packages/app/src/System/RequestMerger.test.ts

@@ -79,6 +79,7 @@ describe("flatMerge", () => {
       { ids: 0, authors: "b" },
       { kinds: 1 },
       { kinds: 2 },
+      { kinds: 2 },
       { ids: 0, authors: "c" },
       { authors: "c", kinds: 1 },
       { authors: "c", limit: 100 },

packages/app/src/System/RequestMerger.ts

@@ -7,19 +7,35 @@ import { distance } from "./Util";
  */
 const DiscriminatorKeys = ["since", "until", "limit", "search"];
 
-export function canMergeFilters(a: any, b: any): boolean {
+export function canMergeFilters(a: FlatReqFilter, b: FlatReqFilter): boolean {
+  const aObj = a as Record<string, string | number | undefined>;
+  const bObj = b as Record<string, string | number | undefined>;
   for (const key of DiscriminatorKeys) {
-    if (key in a || key in b) {
-      if (a[key] !== b[key]) {
+    if (key in aObj || key in bObj) {
+      if (aObj[key] !== bObj[key]) {
         return false;
       }
     }
   }
+  const keys1 = Object.keys(aObj);
+  const keys2 = Object.keys(bObj);
+  const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
 
-  return true;
+  let distance = 0;
+  for (const key of maxKeys) {
+    if (key in aObj && key in bObj) {
+      if (aObj[key] !== bObj[key]) {
+        distance++;
+      }
+    } else {
+      return false;
+    }
+  }
+  return distance <= 1;
 }
 
 export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
+  console.time("mergeSimilar");
   const ret = [];
 
   while (filters.length > 0) {
@@ -27,13 +43,14 @@ export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
     const mergeSet = [current];
     for (let i = 0; i < filters.length; i++) {
       const f = filters[i];
-      if (mergeSet.every(v => canMergeFilters(v, f) && distance(v, f) === 1)) {
+      if (mergeSet.every(v => canMergeFilters(v, f))) {
         mergeSet.push(filters.splice(i, 1)[0]);
         i--;
       }
     }
     ret.push(simpleMerge(mergeSet));
   }
+  console.timeEnd("mergeSimilar");
   return ret;
 }
 
@@ -96,6 +113,7 @@ export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
  * @returns
  */
 export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
+  console.time("flatMerge");
   let ret: Array<ReqFilter> = [];
 
   // to compute filters which can be merged we need to calucate the distance change between each filter
@@ -130,7 +148,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
     for (let i = 0; i < all.length; i++) {
       const f = all[i];
 
-      if (mergeSet.every(a => canMergeFilters(a, f) && distance(a, f) === 1)) {
+      if (mergeSet.every(a => canMergeFilters(a, f))) {
         mergeSet.push(all.splice(i, 1)[0]);
         i--;
       }
@@ -145,5 +163,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
     }
     ret = n;
   }
+  console.timeEnd("flatMerge");
+  console.debug(ret);
   return ret;
 }