From 79011f8ca2d9ebf38f6d0e7555de8d2248a347f4 Mon Sep 17 00:00:00 2001 From: Kieran Date: Tue, 13 Jun 2023 15:56:06 +0100 Subject: [PATCH] try improve performance --- packages/system/src/RequestExpander.ts | 28 ++++++++------- packages/system/src/RequestMerger.ts | 7 ++-- packages/system/src/Utils.ts | 3 +- packages/system/tests/RequestBuilder.test.ts | 35 ++++++++++++++++++ packages/system/tests/RequestExpander.test.ts | 36 +++++++++---------- packages/system/tests/RequestMerger.test.ts | 18 +++++++--- 6 files changed, 86 insertions(+), 41 deletions(-) diff --git a/packages/system/src/RequestExpander.ts b/packages/system/src/RequestExpander.ts index 8aff4238..7b582604 100644 --- a/packages/system/src/RequestExpander.ts +++ b/packages/system/src/RequestExpander.ts @@ -1,18 +1,19 @@ -import { u256, ReqFilter } from "./Nostr"; +import { ReqFilter } from "./Nostr"; export interface FlatReqFilter { - ids?: u256; - authors?: u256; - kinds?: number; - "#e"?: u256; - "#p"?: u256; - "#t"?: string; - "#d"?: string; - "#r"?: string; - search?: string; - since?: number; - until?: number; - limit?: number; + keys: number + ids?: string + authors?: string + kinds?: number + "#e"?: string + "#p"?: string + "#t"?: string + "#d"?: string + "#r"?: string + search?: string + since?: number + until?: number + limit?: number } /** @@ -41,6 +42,7 @@ export function expandFilter(f: ReqFilter): Array { } generateCombinations(0, { + keys: keys.length, ...Object.fromEntries(props), }); diff --git a/packages/system/src/RequestMerger.ts b/packages/system/src/RequestMerger.ts index 7dde780f..b5a6556a 100644 --- a/packages/system/src/RequestMerger.ts +++ b/packages/system/src/RequestMerger.ts @@ -11,10 +11,8 @@ export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | const aObj = a as Record; const bObj = b as Record; for (const key of DiscriminatorKeys) { - if (key in aObj || key in bObj) { - if (aObj[key] !== bObj[key]) { - return false; - } + if (aObj[key] !== bObj[key]) { + return false; } } return distance(a, b) <= 1; @@ -107,6 +105,7 @@ export function flatMerge(all: Array): Array { function mergeFiltersInSet(filters: Array) { return filters.reduce((acc, a) => { Object.entries(a).forEach(([k, v]) => { + if (k === "keys") return; if (DiscriminatorKeys.includes(k)) { acc[k] = v; } else { diff --git a/packages/system/src/Utils.ts b/packages/system/src/Utils.ts index 8eecf8f7..485888f3 100644 --- a/packages/system/src/Utils.ts +++ b/packages/system/src/Utils.ts @@ -71,7 +71,8 @@ export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | Req } export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean { - return a.since === b.since + return a.keys === b.keys + && a.since === b.since && a.until === b.until && a.limit === b.limit && a.search === b.search diff --git a/packages/system/tests/RequestBuilder.test.ts b/packages/system/tests/RequestBuilder.test.ts index 3289900b..62919c36 100644 --- a/packages/system/tests/RequestBuilder.test.ts +++ b/packages/system/tests/RequestBuilder.test.ts @@ -2,6 +2,8 @@ import { RelayCache } from "../src/GossipModel"; import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder"; import { describe, expect } from "@jest/globals"; import { expandFilter } from "../src/RequestExpander"; +import { unixNowMs } from "../src/Utils"; +import { bytesToHex } from "@noble/curves/abstract/utils"; const DummyCache = { get: (pk?: string) => { @@ -162,3 +164,36 @@ describe("RequestBuilder", () => { ]); }); }); + +describe("build diff, large follow list", () => { + const f = []; + for (let x = 0; x < 500; x++) { + const bytes = crypto.getRandomValues(new Uint8Array(32)); + f.push(bytesToHex(bytes)); + } + + const rb = new RequestBuilder("test"); + rb.withFilter().authors(f).kinds([1, 6, 10002, 3, 6969]); + + const start = unixNowMs(); + const a = rb.build(DummyCache); + expect(a).toEqual(f.map(a => { + return { + strategy: RequestStrategy.AuthorsRelays, + relay: `wss://${a}.com/`, + filters: [ + { + kinds: [1, 6, 10002, 3, 6969], + authors: [a], + } + ], + } + })); + expect(unixNowMs() - start).toBeLessThan(500); + + const start2 = unixNowMs(); + const b = rb.buildDiff(DummyCache, rb.buildRaw().flatMap(expandFilter)); + expect(b).toEqual([]); + expect(unixNowMs() - start2).toBeLessThan(200); + +}) \ No newline at end of file diff --git a/packages/system/tests/RequestExpander.test.ts b/packages/system/tests/RequestExpander.test.ts index d87faf7b..fa327146 100644 --- a/packages/system/tests/RequestExpander.test.ts +++ b/packages/system/tests/RequestExpander.test.ts @@ -11,24 +11,24 @@ describe("RequestExpander", () => { limit: 10, }; expect(expandFilter(a)).toEqual([ - { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, - { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, + { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, ]); }); }); diff --git a/packages/system/tests/RequestMerger.test.ts b/packages/system/tests/RequestMerger.test.ts index f9ac464d..f3cefda9 100644 --- a/packages/system/tests/RequestMerger.test.ts +++ b/packages/system/tests/RequestMerger.test.ts @@ -112,38 +112,46 @@ describe('canMerge', () => { it("should have 0 distance", () => { const a = { ids: "a", + keys: 1 }; const b = { ids: "a", + keys: 1 }; expect(canMergeFilters(a, b)).toEqual(true); }); it("should have 1 distance", () => { const a = { ids: "a", + keys: 1 }; const b = { ids: "b", + keys: 1 }; expect(canMergeFilters(a, b)).toEqual(true); }); it("should have 10 distance", () => { const a = { ids: "a", + keys: 1 }; const b = { ids: "a", kinds: 1, + keys: 2 }; expect(canMergeFilters(a, b)).toEqual(false); }); it("should have 11 distance", () => { const a = { ids: "a", + keys: 1 }; const b = { ids: "b", kinds: 1, + keys: 2 }; expect(canMergeFilters(a, b)).toEqual(false); }); @@ -152,13 +160,13 @@ describe('canMerge', () => { since: 1, until: 100, kinds: [1], - authors: ["kieran", "snort", "c", "d", "e"], + authors: ["kieran", "snort", "c", "d", "e"] }; const b = { since: 1, until: 100, kinds: [6969], - authors: ["kieran", "snort", "c", "d", "e"], + authors: ["kieran", "snort", "c", "d", "e"] }; expect(canMergeFilters(a, b)).toEqual(true); }); @@ -167,14 +175,14 @@ describe('canMerge', () => { since: 1, until: 100, kinds: [1], - authors: ["f", "kieran", "snort", "c", "d"], + authors: ["f", "kieran", "snort", "c", "d"] }; const b = { since: 1, until: 100, kinds: [1], - authors: ["kieran", "snort", "c", "d", "e"], + authors: ["kieran", "snort", "c", "d", "e"] }; expect(canMergeFilters(a, b)).toEqual(true); }); -}) +}) \ No newline at end of file