Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: cache compressed filters

Browse Source
This commit is contained in:
kieran 2024-06-17 14:20:26 +01:00
parent 240c3ba1c3
commit a31c054326
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
2 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
packages/system/src/query.ts
View File

@ -168,6 +168,11 @@ export class Query extends EventEmitter<QueryEvents> {
#log = debug("Query");
/**
* Compressed cached trace filters
*/
#cachedFilters?: Array<ReqFilter>;
constructor(system: SystemInterface, req: RequestBuilder) {
super();
this.request = req;
@ -193,7 +198,6 @@ export class Query extends EventEmitter<QueryEvents> {
addRequest(req: RequestBuilder) {
if (req.instance === this.request.instance) {
// same requst, do nothing
this.#log("Same query %O === %O", req, this.request);
return;
}
this.#log("Add query %O to %s", req, this.id);
@ -214,7 +218,10 @@ export class Query extends EventEmitter<QueryEvents> {
* Recompute the complete set of compressed filters from all query traces
*/
get filters() {
return this.#tracing.flatMap(a => a.filters);
if (this.#system && !this.#cachedFilters) {
this.#cachedFilters = this.#system.optimizer.compress(this.#tracing.flatMap(a => a.filters));
}
return this.#cachedFilters ?? this.#tracing.flatMap(a => a.filters);
}
get feed() {
@ -432,6 +439,7 @@ export class Query extends EventEmitter<QueryEvents> {
c.off("closed", eoseHandler);
});
this.#tracing.push(qt);
this.#cachedFilters = undefined;
if (q.syncFrom !== undefined) {
c.request(["SYNC", qt.id, q.syncFrom, ...qt.filters], () => qt.sentToRelay());

17
packages/system/src/request-builder.ts
View File

@ -59,6 +59,7 @@ export class RequestBuilder {
#builders: Array<RequestFilterBuilder>;
#options?: RequestBuilderOptions;
#log = debug("RequestBuilder");
#rawCached?: Array<ReqFilter>;
constructor(id: string) {
this.instance = uuid();
@ -83,17 +84,20 @@ export class RequestBuilder {
*/
add(other: RequestBuilder) {
this.#builders.push(...other.#builders);
this.#rawCached = undefined;
}
withFilter() {
const ret = new RequestFilterBuilder();
this.#builders.push(ret);
this.#rawCached = undefined;
return ret;
}
withBareFilter(f: ReqFilter) {
const ret = new RequestFilterBuilder(f);
this.#builders.push(ret);
this.#rawCached = undefined;
return ret;
}
@ -105,12 +109,15 @@ export class RequestBuilder {
return this;
}
buildRaw(): Array<ReqFilter> {
return this.#builders.map(f => f.filter);
buildRaw(system?: SystemInterface): Array<ReqFilter> {
if (!this.#rawCached && system) {
this.#rawCached = system.optimizer.compress(this.#builders.map(f => f.filter));
}
return this.#rawCached ?? [];
}
build(system: SystemInterface): Array<BuiltRawReqFilter> {
let rawFilters = this.buildRaw();
let rawFilters = this.buildRaw(system);
if (system.requestRouter) {
rawFilters = system.requestRouter.forAllRequest(rawFilters);
}
@ -124,7 +131,7 @@ export class RequestBuilder {
buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Array<BuiltRawReqFilter> {
const start = unixNowMs();
let rawFilters = this.buildRaw();
let rawFilters = this.buildRaw(system);
if (system.requestRouter) {
rawFilters = system.requestRouter.forAllRequest(rawFilters);
}
@ -133,7 +140,7 @@ export class RequestBuilder {
const ret = this.#groupFlatByRelay(system, diff);
const ts = unixNowMs() - start;
if (ts >= 100) {
this.#log("slow diff %s %d ms, consider separate query ids, or use skipDiff: %O", this.id, ts, ret);
this.#log("slow diff %s %d ms, consider separate query ids, or use skipDiff: %O", this.id, ts, prev);
}
return ret;
}