diff --git a/package.json b/package.json index 52bdb588..87001fc9 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "dependencies": { "@cloudflare/workers-types": "^4.20230307.0", "@tauri-apps/cli": "^1.2.3", + "comlink": "^4.4.1", "eslint": "^8.48.0", "prettier": "^3.0.3", "typescript": "^5.2.2" diff --git a/packages/app/config/default.json b/packages/app/config/default.json index 498fc11b..fd3f26ba 100644 --- a/packages/app/config/default.json +++ b/packages/app/config/default.json @@ -39,5 +39,6 @@ "wss://relay.snort.social/": { "read": true, "write": true }, "wss://nostr.wine/": { "read": true, "write": false }, "wss://eden.nostr.land/": { "read": true, "write": false } - } + }, + "useIndexedDBEvents": false } diff --git a/packages/app/config/iris.json b/packages/app/config/iris.json index b4e37335..662930ad 100644 --- a/packages/app/config/iris.json +++ b/packages/app/config/iris.json @@ -37,5 +37,6 @@ "wss://eden.nostr.land/": { "read": true, "write": false }, "wss://relay.nostr.band/": { "read": true, "write": true }, "wss://relay.damus.io/": { "read": true, "write": true } - } + }, + "useIndexedDBEvents": true } diff --git a/packages/app/custom.d.ts b/packages/app/custom.d.ts index 89be2e93..1fa4a182 100644 --- a/packages/app/custom.d.ts +++ b/packages/app/custom.d.ts @@ -84,6 +84,7 @@ declare const CONFIG: { eventLinkPrefix: NostrPrefix; profileLinkPrefix: NostrPrefix; defaultRelays: Record; + useIndexedDBEvents: boolean; }; /** diff --git a/packages/app/src/Cache/FollowListCache.ts b/packages/app/src/Cache/FollowListCache.ts index 2c24659d..a4a20454 100644 --- a/packages/app/src/Cache/FollowListCache.ts +++ b/packages/app/src/Cache/FollowListCache.ts @@ -26,7 +26,7 @@ export class FollowListCache extends RefreshFeedCache { loaded: unixNowMs(), }); if (update !== "no_change") { - socialGraphInstance.handleFollowEvent(e); + socialGraphInstance.handleEvent(e); } }), ); @@ -42,6 +42,6 @@ export class FollowListCache extends RefreshFeedCache { override async preload() { await super.preload(); - this.snapshot().forEach(e => socialGraphInstance.handleFollowEvent(e)); + this.snapshot().forEach(e => socialGraphInstance.handleEvent(e)); } } diff --git a/packages/app/src/Cache/IndexedDB.ts b/packages/app/src/Cache/IndexedDB.ts new file mode 100644 index 00000000..5e0b902d --- /dev/null +++ b/packages/app/src/Cache/IndexedDB.ts @@ -0,0 +1,200 @@ +import Dexie, { Table } from "dexie"; +import { TaggedNostrEvent, ReqFilter as Filter } from "@snort/system"; +import * as Comlink from "comlink"; + +type Tag = { + id: string; + eventId: string; + type: string; + value: string; +}; + +type SaveQueueEntry = { event: TaggedNostrEvent; tags: Tag[] }; + +class IndexedDB extends Dexie { + events!: Table; + tags!: Table; + private saveQueue: SaveQueueEntry[] = []; + private seenEvents = new Set(); // LRU set maybe? + private seenFilters = new Set(); + private subscribedEventIds = new Set(); + private subscribedAuthors = new Set(); + private subscribedTags = new Set(); + + constructor() { + super("EventDB"); + + this.version(5).stores({ + events: "id, pubkey, kind, created_at, [pubkey+kind]", + tags: "id, eventId, [type+value]", + }); + + this.startInterval(); + } + + private startInterval() { + const processQueue = async () => { + if (this.saveQueue.length > 0) { + try { + const eventsToSave: TaggedNostrEvent[] = []; + const tagsToSave: Tag[] = []; + for (const item of this.saveQueue) { + eventsToSave.push(item.event); + tagsToSave.push(...item.tags); + } + await this.events.bulkPut(eventsToSave); + await this.tags.bulkPut(tagsToSave); + } catch (e) { + console.error(e); + } finally { + this.saveQueue = []; + } + } + setTimeout(() => processQueue(), 3000); + }; + + setTimeout(() => processQueue(), 3000); + } + + handleEvent(event: TaggedNostrEvent) { + if (this.seenEvents.has(event.id)) { + return; + } + this.seenEvents.add(event.id); + + // maybe we don't want event.kind 3 tags + const tags = + event.kind === 3 + ? [] + : event.tags + ?.filter(tag => { + if (tag[0] === "d") { + return true; + } + if (tag[0] === "e") { + return true; + } + // we're only interested in p tags where we are mentioned + if (tag[0] === "p") { + // && Key.isMine(tag[1])) { + return true; + } + return false; + }) + .map(tag => ({ + id: event.id.slice(0, 16) + "-" + tag[0].slice(0, 16) + "-" + tag[1].slice(0, 16), + eventId: event.id, + type: tag[0], + value: tag[1], + })) || []; + + this.saveQueue.push({ event, tags }); + } + + _throttle(func, limit) { + let inThrottle; + return function (...args) { + if (!inThrottle) { + inThrottle = true; + setTimeout(() => { + inThrottle = false; + func.apply(this, args); + }, limit); + } + }; + } + + subscribeToAuthors = this._throttle(async function (callback: (event: TaggedNostrEvent) => void, limit?: number) { + const authors = [...this.subscribedAuthors]; + this.subscribedAuthors.clear(); + await this.events + .where("pubkey") + .anyOf(authors) + .limit(limit || 1000) + .each(callback); + }, 100); + + subscribeToEventIds = this._throttle(async function (callback: (event: TaggedNostrEvent) => void) { + const ids = [...this.subscribedEventIds]; + this.subscribedEventIds.clear(); + await this.events.where("id").anyOf(ids).each(callback); + }, 100); + + subscribeToTags = this._throttle(async function (callback: (event: TaggedNostrEvent) => void) { + const tagPairs = [...this.subscribedTags].map(tag => tag.split("|")); + this.subscribedTags.clear(); + await this.tags + .where("[type+value]") + .anyOf(tagPairs) + .each(tag => this.subscribedEventIds.add(tag.eventId)); + + await this.subscribeToEventIds(callback); + }, 100); + + async find(filter: Filter, callback: (event: TaggedNostrEvent) => void): Promise { + if (!filter) return; + + // make sure only 1 argument is passed + const cb = e => callback(e); + + if (filter["#p"] && Array.isArray(filter["#p"])) { + for (const eventId of filter["#p"]) { + this.subscribedTags.add("p|" + eventId); + } + + await this.subscribeToTags(cb); + return; + } + + if (filter["#e"] && Array.isArray(filter["#e"])) { + for (const eventId of filter["#e"]) { + this.subscribedTags.add("e|" + eventId); + } + + await this.subscribeToTags(cb); + return; + } + + if (filter["#d"] && Array.isArray(filter["#d"])) { + for (const eventId of filter["#d"]) { + this.subscribedTags.add("d|" + eventId); + } + + await this.subscribeToTags(cb); + return; + } + + if (filter.ids?.length) { + filter.ids.forEach(id => this.subscribedEventIds.add(id)); + await this.subscribeToEventIds(cb); + return; + } + + if (filter.authors?.length) { + filter.authors.forEach(author => this.subscribedAuthors.add(author)); + await this.subscribeToAuthors(cb); + return; + } + + let query = this.events; + if (filter.kinds) { + query = query.where("kind").anyOf(filter.kinds); + } + if (filter.search) { + const regexp = new RegExp(filter.search, "i"); + query = query.filter((event: Event) => event.content?.match(regexp)); + } + if (filter.limit) { + query = query.limit(filter.limit); + } + // TODO test that the sort is actually working + await query.each(e => { + this.seenEvents.add(e.id); + cb(e); + }); + } +} + +const db = new IndexedDB(); + +Comlink.expose(db); diff --git a/packages/app/src/Element/SearchBox.tsx b/packages/app/src/Element/SearchBox.tsx index fd878bd6..e40a8b6f 100644 --- a/packages/app/src/Element/SearchBox.tsx +++ b/packages/app/src/Element/SearchBox.tsx @@ -8,9 +8,9 @@ import { NostrLink, tryParseNostrLink } from "@snort/system"; import { useLocation, useNavigate } from "react-router-dom"; import { unixNow } from "@snort/shared"; import useTimelineFeed, { TimelineFeedOptions, TimelineSubject } from "../Feed/TimelineFeed"; -import { fuzzySearch, FuzzySearchResult } from "@/index"; import ProfileImage from "@/Element/User/ProfileImage"; import { socialGraphInstance } from "@snort/system"; +import fuzzySearch, { FuzzySearchResult } from "@/FuzzySearch"; const MAX_RESULTS = 3; diff --git a/packages/app/src/FuzzySearch.ts b/packages/app/src/FuzzySearch.ts index 9ffe810c..7545223e 100644 --- a/packages/app/src/FuzzySearch.ts +++ b/packages/app/src/FuzzySearch.ts @@ -1,6 +1,4 @@ import Fuse from "fuse.js"; -import { socialGraphInstance } from "@snort/system"; -import { System } from "."; export type FuzzySearchResult = { pubkey: string; @@ -9,7 +7,7 @@ export type FuzzySearchResult = { nip05?: string; }; -export const fuzzySearch = new Fuse([], { +const fuzzySearch = new Fuse([], { keys: ["name", "username", { name: "nip05", weight: 0.5 }], threshold: 0.3, // sortFn here? @@ -17,27 +15,27 @@ export const fuzzySearch = new Fuse([], { const profileTimestamps = new Map(); // is this somewhere in cache? -System.on("event", ev => { - if (ev.kind === 0) { - const existing = profileTimestamps.get(ev.pubkey); - if (existing) { - if (existing > ev.created_at) { - return; - } - fuzzySearch.remove(doc => doc.pubkey === ev.pubkey); - } - profileTimestamps.set(ev.pubkey, ev.created_at); - try { - const data = JSON.parse(ev.content); - if (ev.pubkey && (data.name || data.username || data.nip05)) { - data.pubkey = ev.pubkey; - fuzzySearch.add(data); - } - } catch (e) { - console.error(e); - } +export const addEventToFuzzySearch = ev => { + if (ev.kind !== 0) { + return; } - if (ev.kind === 3) { - socialGraphInstance.handleFollowEvent(ev); + const existing = profileTimestamps.get(ev.pubkey); + if (existing) { + if (existing > ev.created_at) { + return; + } + fuzzySearch.remove(doc => doc.pubkey === ev.pubkey); } -}); + profileTimestamps.set(ev.pubkey, ev.created_at); + try { + const data = JSON.parse(ev.content); + if (ev.pubkey && (data.name || data.username || data.nip05)) { + data.pubkey = ev.pubkey; + fuzzySearch.add(data); + } + } catch (e) { + console.error(e); + } +}; + +export default fuzzySearch; diff --git a/packages/app/src/index.tsx b/packages/app/src/index.tsx index 54b6ee4e..24d9e790 100644 --- a/packages/app/src/index.tsx +++ b/packages/app/src/index.tsx @@ -28,6 +28,7 @@ import { PowWorker, encodeTLVEntries, socialGraphInstance, + TaggedNostrEvent, } from "@snort/system"; import PowWorkerURL from "@snort/system/src/pow-worker.ts?worker&url"; import { SnortContext } from "@snort/system-react"; @@ -62,10 +63,13 @@ import { AboutPage } from "@/Pages/About"; import { OnboardingRoutes } from "@/Pages/onboarding"; import { setupWebLNWalletConfig } from "@/Wallet/WebLN"; import { Wallets } from "@/Wallet"; -import Fuse from "fuse.js"; import NetworkGraph from "@/Pages/NetworkGraph"; import WalletPage from "./Pages/WalletPage"; +import IndexedDBWorker from "./Cache/IndexedDB?worker"; +import * as Comlink from "comlink"; +import { addEventToFuzzySearch } from "@/FuzzySearch"; + declare global { interface Window { plausible?: (tag: string, e?: object) => void; @@ -101,6 +105,8 @@ const hasWasm = "WebAssembly" in globalThis; const DefaultPowWorker = hasWasm ? undefined : new PowWorker(PowWorkerURL); export const GetPowWorker = () => (hasWasm ? new WasmPowWorker() : unwrap(DefaultPowWorker)); +const indexedDB = Comlink.wrap(new IndexedDBWorker()); + /** * Singleton nostr system */ @@ -120,47 +126,25 @@ System.on("auth", async (c, r, cb) => { } }); -export type FuzzySearchResult = { - pubkey: string; - name?: string; - display_name?: string; - nip05?: string; -}; - -export const fuzzySearch = new Fuse([], { - keys: ["name", "display_name", { name: "nip05", weight: 0.5 }], - threshold: 0.3, - // sortFn here? -}); - -const profileTimestamps = new Map(); - -// how to also add entries from ProfileCache? System.on("event", (_, ev) => { - if (ev.kind === 0) { - const existing = profileTimestamps.get(ev.pubkey); - if (existing) { - if (existing > ev.created_at) { - return; - } - fuzzySearch.remove(doc => doc.pubkey === ev.pubkey); - } - profileTimestamps.set(ev.pubkey, ev.created_at); - try { - const data = JSON.parse(ev.content); - if (ev.pubkey && (data.name || data.display_name || data.nip05)) { - data.pubkey = ev.pubkey; - fuzzySearch.add(data); - } - } catch (e) { - console.error(e); - } - } - if (ev.kind === 3) { - socialGraphInstance.handleFollowEvent(ev); + addEventToFuzzySearch(ev); + socialGraphInstance.handleEvent(ev); + if (CONFIG.useIndexedDBEvents) { + indexedDB.handleEvent(ev); } }); +if (CONFIG.useIndexedDBEvents) { + System.on("request", (filter: ReqFilter) => { + indexedDB.find( + filter, + Comlink.proxy((e: TaggedNostrEvent) => { + System.HandleEvent(e); + }), + ); + }); +} + async function fetchProfile(key: string) { try { throwIfOffline(); diff --git a/packages/system/src/SocialGraph/SocialGraph.ts b/packages/system/src/SocialGraph/SocialGraph.ts index 96bccf8b..0c6cfae8 100644 --- a/packages/system/src/SocialGraph/SocialGraph.ts +++ b/packages/system/src/SocialGraph/SocialGraph.ts @@ -47,7 +47,10 @@ export default class SocialGraph { } } - handleFollowEvent(event: NostrEvent) { + handleEvent(event: NostrEvent) { + if (event.kind !== 3) { + return; + } try { const author = ID(event.pubkey); const timestamp = event.created_at; diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index 7f5dea1b..04f07a44 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -2,7 +2,7 @@ import debug from "debug"; import EventEmitter from "eventemitter3"; import { unwrap, FeedCache } from "@snort/shared"; -import { NostrEvent, TaggedNostrEvent } from "./nostr"; +import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr"; import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection"; import { Query } from "./query"; import { NoteCollection, NoteStore } from "./note-collection"; @@ -31,6 +31,7 @@ export interface NostrSystemEvents { change: (state: SystemSnapshot) => void; auth: (challenge: string, relay: string, cb: (ev: NostrEvent) => void) => void; event: (subId: string, ev: TaggedNostrEvent) => void; + request: (filter: ReqFilter) => void; } export interface NostrsystemProps { @@ -316,6 +317,10 @@ export class NostrSystem extends EventEmitter implements Syst } qSend.filters = fNew; + fNew.forEach(f => { + this.emit("request", f); + }); + if (qSend.relay) { this.#log("Sending query to %s %O", qSend.relay, qSend); const s = this.#pool.getConnection(qSend.relay); diff --git a/yarn.lock b/yarn.lock index a87e195f..c348ad48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4898,6 +4898,13 @@ __metadata: languageName: node linkType: hard +"comlink@npm:^4.4.1": + version: 4.4.1 + resolution: "comlink@npm:4.4.1" + checksum: 16d58a8f590087fc45432e31d6c138308dfd4b75b89aec0b7f7bb97ad33d810381bd2b1e608a1fb2cf05979af9cbfcdcaf1715996d5fcf77aeb013b6da3260af + languageName: node + linkType: hard + "commander@npm:^2.20.0": version: 2.20.3 resolution: "commander@npm:2.20.3" @@ -9918,6 +9925,7 @@ __metadata: dependencies: "@cloudflare/workers-types": ^4.20230307.0 "@tauri-apps/cli": ^1.2.3 + comlink: ^4.4.1 eslint: ^8.48.0 prettier: ^3.0.3 typescript: ^5.2.2