try improve performance

packages/system/src/RequestExpander.ts

@@ -1,18 +1,19 @@
-import { u256, ReqFilter } from "./Nostr";
+import { ReqFilter } from "./Nostr";
 
 export interface FlatReqFilter {
-  ids?: u256;
+  keys: number
-  authors?: u256;
+  ids?: string
-  kinds?: number;
+  authors?: string
-  "#e"?: u256;
+  kinds?: number
-  "#p"?: u256;
+  "#e"?: string
-  "#t"?: string;
+  "#p"?: string
-  "#d"?: string;
+  "#t"?: string
-  "#r"?: string;
+  "#d"?: string
-  search?: string;
+  "#r"?: string
-  since?: number;
+  search?: string
-  until?: number;
+  since?: number
-  limit?: number;
+  until?: number
+  limit?: number
 }
 
 /**
@@ -41,6 +42,7 @@ export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
   }
 
   generateCombinations(0, {
+    keys: keys.length,
     ...Object.fromEntries(props),
   });
 

packages/system/src/RequestMerger.ts

@@ -11,10 +11,8 @@ export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter |
   const aObj = a as Record<string, string | number | undefined>;
   const bObj = b as Record<string, string | number | undefined>;
   for (const key of DiscriminatorKeys) {
-    if (key in aObj || key in bObj) {
+    if (aObj[key] !== bObj[key]) {
-      if (aObj[key] !== bObj[key]) {
+      return false;
-        return false;
-      }
     }
   }
   return distance(a, b) <= 1;
@@ -107,6 +105,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
   function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
     return filters.reduce((acc, a) => {
       Object.entries(a).forEach(([k, v]) => {
+        if (k === "keys") return;
         if (DiscriminatorKeys.includes(k)) {
           acc[k] = v;
         } else {

packages/system/src/Utils.ts

@@ -71,7 +71,8 @@ export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | Req
 }
 
 export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
-  return a.since === b.since
+  return a.keys === b.keys
+    && a.since === b.since
     && a.until === b.until
     && a.limit === b.limit
     && a.search === b.search

packages/system/tests/RequestBuilder.test.ts

@@ -2,6 +2,8 @@ import { RelayCache } from "../src/GossipModel";
 import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder";
 import { describe, expect } from "@jest/globals";
 import { expandFilter } from "../src/RequestExpander";
+import { unixNowMs } from "../src/Utils";
+import { bytesToHex } from "@noble/curves/abstract/utils";
 
 const DummyCache = {
   get: (pk?: string) => {
@@ -162,3 +164,36 @@ describe("RequestBuilder", () => {
     ]);
   });
 });
+
+describe("build diff, large follow list", () => {
+  const f = [];
+  for (let x = 0; x < 500; x++) {
+    const bytes = crypto.getRandomValues(new Uint8Array(32));
+    f.push(bytesToHex(bytes));
+  }
+
+  const rb = new RequestBuilder("test");
+  rb.withFilter().authors(f).kinds([1, 6, 10002, 3, 6969]);
+
+  const start = unixNowMs();
+  const a = rb.build(DummyCache);
+  expect(a).toEqual(f.map(a => {
+    return {
+      strategy: RequestStrategy.AuthorsRelays,
+      relay: `wss://${a}.com/`,
+      filters: [
+        {
+          kinds: [1, 6, 10002, 3, 6969],
+          authors: [a],
+        }
+      ],
+    }
+  }));
+  expect(unixNowMs() - start).toBeLessThan(500);
+
+  const start2 = unixNowMs();
+  const b = rb.buildDiff(DummyCache, rb.buildRaw().flatMap(expandFilter));
+  expect(b).toEqual([]);
+  expect(unixNowMs() - start2).toBeLessThan(200);
+
+})

packages/system/tests/RequestExpander.test.ts

@@ -11,24 +11,24 @@ describe("RequestExpander", () => {
       limit: 10,
     };
     expect(expandFilter(a)).toEqual([
-      { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
-      { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
     ]);
   });
 });

packages/system/tests/RequestMerger.test.ts

@@ -112,38 +112,46 @@ describe('canMerge', () => {
   it("should have 0 distance", () => {
     const a = {
       ids: "a",
+      keys: 1
     };
     const b = {
       ids: "a",
+      keys: 1
     };
     expect(canMergeFilters(a, b)).toEqual(true);
   });
   it("should have 1 distance", () => {
     const a = {
       ids: "a",
+      keys: 1
     };
     const b = {
       ids: "b",
+      keys: 1
     };
     expect(canMergeFilters(a, b)).toEqual(true);
   });
   it("should have 10 distance", () => {
     const a = {
       ids: "a",
+      keys: 1
     };
     const b = {
       ids: "a",
       kinds: 1,
+      keys: 2
     };
     expect(canMergeFilters(a, b)).toEqual(false);
   });
   it("should have 11 distance", () => {
     const a = {
       ids: "a",
+      keys: 1
     };
     const b = {
       ids: "b",
       kinds: 1,
+      keys: 2
     };
     expect(canMergeFilters(a, b)).toEqual(false);
   });
@@ -152,13 +160,13 @@ describe('canMerge', () => {
       since: 1,
       until: 100,
       kinds: [1],
-      authors: ["kieran", "snort", "c", "d", "e"],
+      authors: ["kieran", "snort", "c", "d", "e"]
     };
     const b = {
       since: 1,
       until: 100,
       kinds: [6969],
-      authors: ["kieran", "snort", "c", "d", "e"],
+      authors: ["kieran", "snort", "c", "d", "e"]
     };
     expect(canMergeFilters(a, b)).toEqual(true);
   });
@@ -167,14 +175,14 @@ describe('canMerge', () => {
       since: 1,
       until: 100,
       kinds: [1],
-      authors: ["f", "kieran", "snort", "c", "d"],
+      authors: ["f", "kieran", "snort", "c", "d"]
     };
     const b = {
       since: 1,
       until: 100,
       kinds: [1],
-      authors: ["kieran", "snort", "c", "d", "e"],
+      authors: ["kieran", "snort", "c", "d", "e"]
     };
     expect(canMergeFilters(a, b)).toEqual(true);
   });
-})
+})