optimize
This commit is contained in:
parent
ae6618f0ed
commit
8e6a1ecbc2
@ -48,7 +48,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "webpack serve",
|
"start": "webpack serve",
|
||||||
"build": "webpack --node-env=production",
|
"build": "webpack --node-env=production",
|
||||||
"test": "jest",
|
"test": "jest --runInBand",
|
||||||
"intl-extract": "formatjs extract 'src/**/*.ts*' --ignore='**/*.d.ts' --out-file src/lang.json --flatten true",
|
"intl-extract": "formatjs extract 'src/**/*.ts*' --ignore='**/*.d.ts' --out-file src/lang.json --flatten true",
|
||||||
"intl-compile": "formatjs compile src/lang.json --out-file src/translations/en.json",
|
"intl-compile": "formatjs compile src/lang.json --out-file src/translations/en.json",
|
||||||
"format": "prettier --write .",
|
"format": "prettier --write .",
|
||||||
|
@ -6,35 +6,25 @@ import { System } from "index";
|
|||||||
|
|
||||||
const useRequestBuilder = <TStore extends NoteStore, TSnapshot = ReturnType<TStore["getSnapshotData"]>>(
|
const useRequestBuilder = <TStore extends NoteStore, TSnapshot = ReturnType<TStore["getSnapshotData"]>>(
|
||||||
type: { new (): TStore },
|
type: { new (): TStore },
|
||||||
rb: RequestBuilder | null,
|
rb: RequestBuilder | null
|
||||||
debounced?: number
|
|
||||||
) => {
|
) => {
|
||||||
const subscribe = (onChanged: () => void) => {
|
const subscribe = (onChanged: () => void) => {
|
||||||
const store = (System.Query<TStore>(type, rb)?.feed as TStore) ?? new type();
|
if (rb) {
|
||||||
let t: ReturnType<typeof setTimeout> | undefined;
|
const q = System.Query<TStore>(type, rb);
|
||||||
const release = store.hook(() => {
|
const release = q.feed.hook(onChanged);
|
||||||
if (!t) {
|
return () => {
|
||||||
t = setTimeout(() => {
|
q.cancel();
|
||||||
clearTimeout(t);
|
release();
|
||||||
t = undefined;
|
};
|
||||||
onChanged();
|
}
|
||||||
}, debounced ?? 500);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (rb?.id) {
|
// noop
|
||||||
System.GetQuery(rb.id)?.cancel();
|
|
||||||
}
|
|
||||||
release();
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
const getState = (): StoreSnapshot<TSnapshot> => {
|
const getState = (): StoreSnapshot<TSnapshot> => {
|
||||||
if (rb?.id) {
|
const q = System.GetQuery(rb?.id ?? "");
|
||||||
const q = System.GetQuery(rb.id);
|
if (q) {
|
||||||
if (q) {
|
return unwrap(q).feed?.snapshot as StoreSnapshot<TSnapshot>;
|
||||||
return unwrap(q).feed?.snapshot as StoreSnapshot<TSnapshot>;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return EmptySnapshot as StoreSnapshot<TSnapshot>;
|
return EmptySnapshot as StoreSnapshot<TSnapshot>;
|
||||||
};
|
};
|
||||||
|
@ -119,13 +119,11 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
|
|||||||
return this.Queries.get(id);
|
return this.Queries.get(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder | null): Query | undefined {
|
Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): Query {
|
||||||
if (!req) return;
|
|
||||||
|
|
||||||
const existing = this.Queries.get(req.id);
|
const existing = this.Queries.get(req.id);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
const filters = req.buildDiff(this.#relayCache, existing.filters);
|
const filters = req.buildDiff(this.#relayCache, existing.filters);
|
||||||
if (filters.length === 0 && !req.options?.skipDiff) {
|
if (filters.length === 0 && !!req.options?.skipDiff) {
|
||||||
return existing;
|
return existing;
|
||||||
} else {
|
} else {
|
||||||
for (const subQ of filters) {
|
for (const subQ of filters) {
|
||||||
|
@ -57,6 +57,7 @@ export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> i
|
|||||||
data: undefined,
|
data: undefined,
|
||||||
};
|
};
|
||||||
#needsSnapshot = true;
|
#needsSnapshot = true;
|
||||||
|
#nextNotifyTimer?: ReturnType<typeof setTimeout>;
|
||||||
|
|
||||||
get snapshot() {
|
get snapshot() {
|
||||||
this.#updateSnapshot();
|
this.#updateSnapshot();
|
||||||
@ -106,8 +107,13 @@ export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> i
|
|||||||
|
|
||||||
protected onChange(changes: Readonly<Array<TaggedRawEvent>>): void {
|
protected onChange(changes: Readonly<Array<TaggedRawEvent>>): void {
|
||||||
this.#needsSnapshot = true;
|
this.#needsSnapshot = true;
|
||||||
for (const hk of this.#hooks) {
|
if (!this.#nextNotifyTimer) {
|
||||||
hk();
|
this.#nextNotifyTimer = setTimeout(() => {
|
||||||
|
this.#nextNotifyTimer = undefined;
|
||||||
|
for (const hk of this.#hooks) {
|
||||||
|
hk();
|
||||||
|
}
|
||||||
|
}, 500);
|
||||||
}
|
}
|
||||||
if (changes.length > 0) {
|
if (changes.length > 0) {
|
||||||
for (const hkE of this.#eventHooks) {
|
for (const hkE of this.#eventHooks) {
|
||||||
|
@ -101,8 +101,12 @@ describe("query", () => {
|
|||||||
|
|
||||||
expect(q.filters).toEqual([
|
expect(q.filters).toEqual([
|
||||||
{
|
{
|
||||||
authors: ["a", "b", "c"],
|
authors: ["a", "b"],
|
||||||
kinds: [1, 2],
|
kinds: [1],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
authors: ["b", "c"],
|
||||||
|
kinds: [2],
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
@ -3,9 +3,10 @@ import debug from "debug";
|
|||||||
import { Connection, ReqFilter, Nips, TaggedRawEvent } from "System";
|
import { Connection, ReqFilter, Nips, TaggedRawEvent } from "System";
|
||||||
import { unixNowMs, unwrap } from "SnortUtils";
|
import { unixNowMs, unwrap } from "SnortUtils";
|
||||||
import { NoteStore } from "./NoteCollection";
|
import { NoteStore } from "./NoteCollection";
|
||||||
import { simpleMerge } from "./RequestMerger";
|
import { flatMerge, mergeSimilar, simpleMerge } from "./RequestMerger";
|
||||||
import { eventMatchesFilter } from "./RequestMatcher";
|
import { eventMatchesFilter } from "./RequestMatcher";
|
||||||
import { BuiltRawReqFilter } from "./RequestBuilder";
|
import { BuiltRawReqFilter } from "./RequestBuilder";
|
||||||
|
import { expandFilter } from "./RequestExpander";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tracing for relay query status
|
* Tracing for relay query status
|
||||||
@ -137,6 +138,7 @@ export class Query implements QueryBase {
|
|||||||
#feed: NoteStore;
|
#feed: NoteStore;
|
||||||
|
|
||||||
#log = debug("Query");
|
#log = debug("Query");
|
||||||
|
#allFilters: Array<ReqFilter> = [];
|
||||||
|
|
||||||
constructor(id: string, feed: NoteStore) {
|
constructor(id: string, feed: NoteStore) {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
@ -152,9 +154,11 @@ export class Query implements QueryBase {
|
|||||||
return this.#cancelTimeout;
|
return this.#cancelTimeout;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recompute the complete set of compressed filters from all query traces
|
||||||
|
*/
|
||||||
get filters() {
|
get filters() {
|
||||||
const filters = this.#tracing.flatMap(a => a.filters);
|
return this.#allFilters;
|
||||||
return [simpleMerge(filters)];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get feed() {
|
get feed() {
|
||||||
@ -264,7 +268,13 @@ export class Query implements QueryBase {
|
|||||||
() => this.#onProgress()
|
() => this.#onProgress()
|
||||||
);
|
);
|
||||||
this.#tracing.push(qt);
|
this.#tracing.push(qt);
|
||||||
|
this.#reComputeFilters();
|
||||||
c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
|
c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
|
||||||
return qt;
|
return qt;
|
||||||
}
|
}
|
||||||
|
#reComputeFilters() {
|
||||||
|
console.time("reComputeFilters");
|
||||||
|
this.#allFilters = flatMerge(this.#tracing.flatMap(a => a.filters).flatMap(expandFilter));
|
||||||
|
console.timeEnd("reComputeFilters");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -79,6 +79,7 @@ describe("flatMerge", () => {
|
|||||||
{ ids: 0, authors: "b" },
|
{ ids: 0, authors: "b" },
|
||||||
{ kinds: 1 },
|
{ kinds: 1 },
|
||||||
{ kinds: 2 },
|
{ kinds: 2 },
|
||||||
|
{ kinds: 2 },
|
||||||
{ ids: 0, authors: "c" },
|
{ ids: 0, authors: "c" },
|
||||||
{ authors: "c", kinds: 1 },
|
{ authors: "c", kinds: 1 },
|
||||||
{ authors: "c", limit: 100 },
|
{ authors: "c", limit: 100 },
|
||||||
|
@ -7,19 +7,35 @@ import { distance } from "./Util";
|
|||||||
*/
|
*/
|
||||||
const DiscriminatorKeys = ["since", "until", "limit", "search"];
|
const DiscriminatorKeys = ["since", "until", "limit", "search"];
|
||||||
|
|
||||||
export function canMergeFilters(a: any, b: any): boolean {
|
export function canMergeFilters(a: FlatReqFilter, b: FlatReqFilter): boolean {
|
||||||
|
const aObj = a as Record<string, string | number | undefined>;
|
||||||
|
const bObj = b as Record<string, string | number | undefined>;
|
||||||
for (const key of DiscriminatorKeys) {
|
for (const key of DiscriminatorKeys) {
|
||||||
if (key in a || key in b) {
|
if (key in aObj || key in bObj) {
|
||||||
if (a[key] !== b[key]) {
|
if (aObj[key] !== bObj[key]) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
const keys1 = Object.keys(aObj);
|
||||||
|
const keys2 = Object.keys(bObj);
|
||||||
|
const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
|
||||||
|
|
||||||
return true;
|
let distance = 0;
|
||||||
|
for (const key of maxKeys) {
|
||||||
|
if (key in aObj && key in bObj) {
|
||||||
|
if (aObj[key] !== bObj[key]) {
|
||||||
|
distance++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return distance <= 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
|
export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
|
||||||
|
console.time("mergeSimilar");
|
||||||
const ret = [];
|
const ret = [];
|
||||||
|
|
||||||
while (filters.length > 0) {
|
while (filters.length > 0) {
|
||||||
@ -27,13 +43,14 @@ export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
|
|||||||
const mergeSet = [current];
|
const mergeSet = [current];
|
||||||
for (let i = 0; i < filters.length; i++) {
|
for (let i = 0; i < filters.length; i++) {
|
||||||
const f = filters[i];
|
const f = filters[i];
|
||||||
if (mergeSet.every(v => canMergeFilters(v, f) && distance(v, f) === 1)) {
|
if (mergeSet.every(v => canMergeFilters(v, f))) {
|
||||||
mergeSet.push(filters.splice(i, 1)[0]);
|
mergeSet.push(filters.splice(i, 1)[0]);
|
||||||
i--;
|
i--;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ret.push(simpleMerge(mergeSet));
|
ret.push(simpleMerge(mergeSet));
|
||||||
}
|
}
|
||||||
|
console.timeEnd("mergeSimilar");
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -96,6 +113,7 @@ export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
|
|||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
||||||
|
console.time("flatMerge");
|
||||||
let ret: Array<ReqFilter> = [];
|
let ret: Array<ReqFilter> = [];
|
||||||
|
|
||||||
// to compute filters which can be merged we need to calucate the distance change between each filter
|
// to compute filters which can be merged we need to calucate the distance change between each filter
|
||||||
@ -130,7 +148,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
|||||||
for (let i = 0; i < all.length; i++) {
|
for (let i = 0; i < all.length; i++) {
|
||||||
const f = all[i];
|
const f = all[i];
|
||||||
|
|
||||||
if (mergeSet.every(a => canMergeFilters(a, f) && distance(a, f) === 1)) {
|
if (mergeSet.every(a => canMergeFilters(a, f))) {
|
||||||
mergeSet.push(all.splice(i, 1)[0]);
|
mergeSet.push(all.splice(i, 1)[0]);
|
||||||
i--;
|
i--;
|
||||||
}
|
}
|
||||||
@ -145,5 +163,7 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
|||||||
}
|
}
|
||||||
ret = n;
|
ret = n;
|
||||||
}
|
}
|
||||||
|
console.timeEnd("flatMerge");
|
||||||
|
console.debug(ret);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user