From 3eb290a594f5c6b538104d27ea09f8469a9a8e36 Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Fri, 5 Jan 2024 12:30:28 +0200 Subject: [PATCH 1/3] move InMemoryDB to system --- packages/app/src/system.ts | 3 --- packages/system/package.json | 2 ++ .../{app/src/Cache => system/src}/InMemoryDB.ts | 9 +-------- packages/system/src/connection.ts | 4 ++-- packages/system/src/nostr-system.ts | 3 +++ yarn.lock | 16 ++++++++++++++++ 6 files changed, 24 insertions(+), 13 deletions(-) rename packages/{app/src/Cache => system/src}/InMemoryDB.ts (97%) diff --git a/packages/app/src/system.ts b/packages/app/src/system.ts index 69b42a63..7ced2bfb 100644 --- a/packages/app/src/system.ts +++ b/packages/app/src/system.ts @@ -15,7 +15,6 @@ import { addEventToFuzzySearch } from "@/Db/FuzzySearch"; import IndexedDBWorker from "@/Db/IndexedDB?worker"; import { LoginStore } from "@/Utils/Login"; import { hasWasm, WasmOptimizer } from "@/Utils/wasm"; -import inMemoryDB from "@/Cache/InMemoryDB"; // move to system or pass alreadyHave fn to system? export const indexedDB = Comlink.wrap(new IndexedDBWorker()); /** @@ -38,7 +37,6 @@ System.on("auth", async (c, r, cb) => { }); System.on("event", (_, ev) => { - inMemoryDB.handleEvent(ev); addEventToFuzzySearch(ev); socialGraphInstance.handleEvent(ev); if (CONFIG.useIndexedDBEvents && socialGraphInstance.getFollowDistance(ev.pubkey) <= 2) { @@ -47,7 +45,6 @@ System.on("event", (_, ev) => { }); System.on("request", (filter: ReqFilter) => { - inMemoryDB.find(filter, e => System.HandleEvent(e)); if (CONFIG.useIndexedDBEvents) { indexedDB.find( filter, diff --git a/packages/system/package.json b/packages/system/package.json index 18d8cf6e..fde412a8 100644 --- a/packages/system/package.json +++ b/packages/system/package.json @@ -21,6 +21,7 @@ "@peculiar/webcrypto": "^1.4.3", "@types/debug": "^4.1.8", "@types/jest": "^29.5.1", + "@types/lokijs": "^1.5.14", "@types/node": "^20.5.9", "@types/uuid": "^9.0.2", "@types/ws": "^8.5.5", @@ -39,6 +40,7 @@ "debug": "^4.3.4", "eventemitter3": "^5.0.1", "isomorphic-ws": "^5.0.0", + "lokijs": "^1.5.12", "uuid": "^9.0.0", "ws": "^8.14.0" } diff --git a/packages/app/src/Cache/InMemoryDB.ts b/packages/system/src/InMemoryDB.ts similarity index 97% rename from packages/app/src/Cache/InMemoryDB.ts rename to packages/system/src/InMemoryDB.ts index cbfbb242..74eb34f5 100644 --- a/packages/app/src/Cache/InMemoryDB.ts +++ b/packages/system/src/InMemoryDB.ts @@ -1,4 +1,4 @@ -import { ID, ReqFilter as Filter, STR, TaggedNostrEvent, UID } from "@snort/system"; +import { ID, ReqFilter as Filter, STR, TaggedNostrEvent, UID } from "."; import loki from "lokijs"; type PackedNostrEvent = { @@ -116,13 +116,6 @@ class InMemoryDB { remove(eventId: string): void { const id = ID(eventId); this.eventsCollection.findAndRemove({ id }); - if (this.idb) { - try { - this.idb.events.where({ id: eventId }).delete(); - } catch (e) { - console.error(e); - } - } } removeOldest(): void { diff --git a/packages/system/src/connection.ts b/packages/system/src/connection.ts index 7d7fa66e..8536690f 100644 --- a/packages/system/src/connection.ts +++ b/packages/system/src/connection.ts @@ -10,7 +10,7 @@ import { NostrEvent, ReqCommand, ReqFilter, TaggedNostrEvent, u256 } from "./nos import { RelayInfo } from "./relay-info"; import EventKind from "./event-kind"; import { getHex64 } from "./utils"; -import inMemoryDB from "@snort/app/src/Cache/InMemoryDB"; +import inMemoryDB from "./InMemoryDB"; /** * Relay settings @@ -204,7 +204,7 @@ export class Connection extends EventEmitter { // skip message processing if we've already seen it const msgId = getHex64(e.data as string, "id"); if (inMemoryDB.has(msgId)) { - console.log('already have'); + console.log("already have"); return; } diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index 61193ebe..b775cdd6 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -26,6 +26,7 @@ import { RelayCache, RelayMetadataLoader } from "./outbox-model"; import { Optimizer, DefaultOptimizer } from "./query-optimizer"; import { trimFilters } from "./request-trim"; import { NostrConnectionPool } from "./nostr-connection-pool"; +import inMemoryDB from "./InMemoryDB"; export interface NostrSystemEvents { change: (state: SystemSnapshot) => void; @@ -167,6 +168,7 @@ export class NostrSystem extends EventEmitter implements Syst // internal handler for on-event this.on("event", (sub, ev) => { + inMemoryDB.handleEvent(ev); for (const [, v] of this.Queries) { v.handleEvent(sub, ev); } @@ -317,6 +319,7 @@ export class NostrSystem extends EventEmitter implements Syst qSend.filters = fNew; fNew.forEach(f => { + inMemoryDB.find(f, e => this.emit("event", "*", e)); this.emit("request", f); }); diff --git a/yarn.lock b/yarn.lock index 8b15a973..e6d83187 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3076,6 +3076,7 @@ __metadata: "@stablelib/xchacha20": ^1.0.1 "@types/debug": ^4.1.8 "@types/jest": ^29.5.1 + "@types/lokijs": ^1.5.14 "@types/node": ^20.5.9 "@types/uuid": ^9.0.2 "@types/ws": ^8.5.5 @@ -3084,6 +3085,7 @@ __metadata: isomorphic-ws: ^5.0.0 jest: ^29.5.0 jest-environment-jsdom: ^29.5.0 + lokijs: ^1.5.12 ts-jest: ^29.1.0 ts-node: ^10.9.1 typescript: ^5.2.2 @@ -3591,6 +3593,13 @@ __metadata: languageName: node linkType: hard +"@types/lokijs@npm:^1.5.14": + version: 1.5.14 + resolution: "@types/lokijs@npm:1.5.14" + checksum: e5601952d19f5d6ace6e2b5d06771de0b4319bcb211f30f094c954e445b335a42e6e2de73561b837868f0b25a18e17333f1e7ff84b5523c0ef324c4650a549d6 + languageName: node + linkType: hard + "@types/magnet-uri@npm:*": version: 5.1.5 resolution: "@types/magnet-uri@npm:5.1.5" @@ -8170,6 +8179,13 @@ __metadata: languageName: node linkType: hard +"lokijs@npm:^1.5.12": + version: 1.5.12 + resolution: "lokijs@npm:1.5.12" + checksum: 5cd45facb890048e83977de1451a7b70e1f9e4cfe41c40d2212dfc160df7919b821072f95b10d692f6932c1080193e0715e8fe18dee332f04da4fda0f08984b6 + languageName: node + linkType: hard + "loose-envify@npm:^1.1.0, loose-envify@npm:^1.4.0": version: 1.4.0 resolution: "loose-envify@npm:1.4.0" From cc753d57084b8da9c3f9d1596028f3a342a27dbe Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Fri, 5 Jan 2024 12:56:46 +0200 Subject: [PATCH 2/3] system.HandleEvent --- packages/system/src/nostr-system.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index b775cdd6..f5d31ea8 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -319,7 +319,7 @@ export class NostrSystem extends EventEmitter implements Syst qSend.filters = fNew; fNew.forEach(f => { - inMemoryDB.find(f, e => this.emit("event", "*", e)); + inMemoryDB.find(f, e => this.HandleEvent(e)); this.emit("request", f); }); From dee9a3de2c5aa71c80ea4ed8ffeb7619b9436fb1 Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Fri, 5 Jan 2024 13:21:18 +0200 Subject: [PATCH 3/3] add filter.not.ids = [alreadyHave] from memory --- packages/app/src/Db/IndexedDB.ts | 10 +++++++++- packages/system/src/nostr-system.ts | 12 +++++++++++- packages/system/src/nostr.ts | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/app/src/Db/IndexedDB.ts b/packages/app/src/Db/IndexedDB.ts index 83509d8e..dbbb7522 100644 --- a/packages/app/src/Db/IndexedDB.ts +++ b/packages/app/src/Db/IndexedDB.ts @@ -165,9 +165,18 @@ class IndexedDB extends Dexie { async find(filter: Filter, callback: (event: TaggedNostrEvent) => void): Promise { if (!filter) return; + const filterString = JSON.stringify(filter); + if (this.readQueue.has(filterString)) { + return; + } + // make sure only 1 argument is passed const cb = e => { this.seenEvents.add(e.id); + if (filter.not?.ids?.includes(e.id)) { + console.log('skipping', e.id); + return; + } callback(e); }; @@ -225,7 +234,6 @@ class IndexedDB extends Dexie { query = query.limit(filter.limit); } // TODO test that the sort is actually working - const filterString = JSON.stringify(filter); this.enqueueRead(filterString, async () => { await query.each(cb); }); diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index f5d31ea8..7a0b3ffa 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -319,7 +319,17 @@ export class NostrSystem extends EventEmitter implements Syst qSend.filters = fNew; fNew.forEach(f => { - inMemoryDB.find(f, e => this.HandleEvent(e)); + const alreadyHave = inMemoryDB.findArray(f).map(e => { + console.log('got from inMemoryDB', e); + this.HandleEvent(e); + return e.id; + }); + f.not = f.not ?? {}; + if (f.not.ids) { + f.not.ids.push(...alreadyHave); + } else { + f.not.ids = alreadyHave; + } this.emit("request", f); }); diff --git a/packages/system/src/nostr.ts b/packages/system/src/nostr.ts index d320381a..f8b187e8 100644 --- a/packages/system/src/nostr.ts +++ b/packages/system/src/nostr.ts @@ -57,6 +57,7 @@ export interface ReqFilter { since?: number; until?: number; limit?: number; + not?: ReqFilter; [key: string]: Array | Array | string | number | undefined; }