0xshushu/dfklfg
1
0
Fork 0
forked from Kieran/snort
Code Pull Requests Activity

try improve performance

Browse Source
This commit is contained in:
Kieran 2023-06-13 15:56:06 +01:00
parent 7992601f46
commit 79011f8ca2
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
6 changed files with 86 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/system/src/RequestExpander.ts
View File

@ -1,18 +1,19 @@
import { u256, ReqFilter } from "./Nostr";
import { ReqFilter } from "./Nostr";
export interface FlatReqFilter {
ids?: u256;
authors?: u256;
kinds?: number;
"#e"?: u256;
"#p"?: u256;
"#t"?: string;
"#d"?: string;
"#r"?: string;
search?: string;
since?: number;
until?: number;
limit?: number;
keys: number
ids?: string
authors?: string
kinds?: number
"#e"?: string
"#p"?: string
"#t"?: string
"#d"?: string
"#r"?: string
search?: string
since?: number
until?: number
limit?: number
}
/**
@ -41,6 +42,7 @@ export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
}
generateCombinations(0, {
keys: keys.length,
...Object.fromEntries(props),
});

7
packages/system/src/RequestMerger.ts
View File

@ -11,10 +11,8 @@ export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter |
const aObj = a as Record<string, string | number | undefined>;
const bObj = b as Record<string, string | number | undefined>;
for (const key of DiscriminatorKeys) {
if (key in aObj || key in bObj) {
if (aObj[key] !== bObj[key]) {
return false;
}
if (aObj[key] !== bObj[key]) {
return false;
}
}
return distance(a, b) <= 1;
@ -107,6 +105,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
return filters.reduce((acc, a) => {
Object.entries(a).forEach(([k, v]) => {
if (k === "keys") return;
if (DiscriminatorKeys.includes(k)) {
acc[k] = v;
} else {

3
packages/system/src/Utils.ts
View File

@ -71,7 +71,8 @@ export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | Req
}
export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
return a.since === b.since
return a.keys === b.keys
&& a.since === b.since
&& a.until === b.until
&& a.limit === b.limit
&& a.search === b.search

35
packages/system/tests/RequestBuilder.test.ts
View File

@ -2,6 +2,8 @@ import { RelayCache } from "../src/GossipModel";
import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder";
import { describe, expect } from "@jest/globals";
import { expandFilter } from "../src/RequestExpander";
import { unixNowMs } from "../src/Utils";
import { bytesToHex } from "@noble/curves/abstract/utils";
const DummyCache = {
get: (pk?: string) => {
@ -162,3 +164,36 @@ describe("RequestBuilder", () => {
]);
});
});
describe("build diff, large follow list", () => {
const f = [];
for (let x = 0; x < 500; x++) {
const bytes = crypto.getRandomValues(new Uint8Array(32));
f.push(bytesToHex(bytes));
}
const rb = new RequestBuilder("test");
rb.withFilter().authors(f).kinds([1, 6, 10002, 3, 6969]);
const start = unixNowMs();
const a = rb.build(DummyCache);
expect(a).toEqual(f.map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
relay: `wss://${a}.com/`,
filters: [
{
kinds: [1, 6, 10002, 3, 6969],
authors: [a],
}
],
}
}));
expect(unixNowMs() - start).toBeLessThan(500);
const start2 = unixNowMs();
const b = rb.buildDiff(DummyCache, rb.buildRaw().flatMap(expandFilter));
expect(b).toEqual([]);
expect(unixNowMs() - start2).toBeLessThan(200);
})

36
packages/system/tests/RequestExpander.test.ts
View File

@ -11,24 +11,24 @@ describe("RequestExpander", () => {
limit: 10,
};
expect(expandFilter(a)).toEqual([
{ authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
{ authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
{ authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
{ authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
]);
});
});

16
packages/system/tests/RequestMerger.test.ts
View File

@ -112,38 +112,46 @@ describe('canMerge', () => {
it("should have 0 distance", () => {
const a = {
ids: "a",
keys: 1
};
const b = {
ids: "a",
keys: 1
};
expect(canMergeFilters(a, b)).toEqual(true);
});
it("should have 1 distance", () => {
const a = {
ids: "a",
keys: 1
};
const b = {
ids: "b",
keys: 1
};
expect(canMergeFilters(a, b)).toEqual(true);
});
it("should have 10 distance", () => {
const a = {
ids: "a",
keys: 1
};
const b = {
ids: "a",
kinds: 1,
keys: 2
};
expect(canMergeFilters(a, b)).toEqual(false);
});
it("should have 11 distance", () => {
const a = {
ids: "a",
keys: 1
};
const b = {
ids: "b",
kinds: 1,
keys: 2
};
expect(canMergeFilters(a, b)).toEqual(false);
});
@ -152,13 +160,13 @@ describe('canMerge', () => {
since: 1,
until: 100,
kinds: [1],
authors: ["kieran", "snort", "c", "d", "e"],
authors: ["kieran", "snort", "c", "d", "e"]
};
const b = {
since: 1,
until: 100,
kinds: [6969],
authors: ["kieran", "snort", "c", "d", "e"],
authors: ["kieran", "snort", "c", "d", "e"]
};
expect(canMergeFilters(a, b)).toEqual(true);
});
@ -167,13 +175,13 @@ describe('canMerge', () => {
since: 1,
until: 100,
kinds: [1],
authors: ["f", "kieran", "snort", "c", "d"],
authors: ["f", "kieran", "snort", "c", "d"]
};
const b = {
since: 1,
until: 100,
kinds: [1],
authors: ["kieran", "snort", "c", "d", "e"],
authors: ["kieran", "snort", "c", "d", "e"]
};
expect(canMergeFilters(a, b)).toEqual(true);
});