add optional idb "relay" worker

package.json

@@ -20,6 +20,7 @@
   "dependencies": {
     "@cloudflare/workers-types": "^4.20230307.0",
     "@tauri-apps/cli": "^1.2.3",
+    "comlink": "^4.4.1",
     "eslint": "^8.48.0",
     "prettier": "^3.0.3",
     "typescript": "^5.2.2"

packages/app/config/default.json

@@ -39,5 +39,6 @@
     "wss://relay.snort.social/": { "read": true, "write": true },
     "wss://nostr.wine/": { "read": true, "write": false },
     "wss://eden.nostr.land/": { "read": true, "write": false }
-  }
+  },
+  "useIndexedDBEvents": false
 }

packages/app/config/iris.json

@@ -37,5 +37,6 @@
     "wss://eden.nostr.land/": { "read": true, "write": false },
     "wss://relay.nostr.band/": { "read": true, "write": true },
     "wss://relay.damus.io/": { "read": true, "write": true }
-  }
+  },
+  "useIndexedDBEvents": true
 }

packages/app/custom.d.ts

@@ -84,6 +84,7 @@ declare const CONFIG: {
   eventLinkPrefix: NostrPrefix;
   profileLinkPrefix: NostrPrefix;
   defaultRelays: Record<string, RelaySettings>;
+  useIndexedDBEvents: boolean;
 };
 
 /**

packages/app/src/Cache/FollowListCache.ts

@@ -26,7 +26,7 @@ export class FollowListCache extends RefreshFeedCache<TaggedNostrEvent> {
           loaded: unixNowMs(),
         });
         if (update !== "no_change") {
-          socialGraphInstance.handleFollowEvent(e);
+          socialGraphInstance.handleEvent(e);
         }
       }),
     );
@@ -42,6 +42,6 @@ export class FollowListCache extends RefreshFeedCache<TaggedNostrEvent> {
 
   override async preload() {
     await super.preload();
-    this.snapshot().forEach(e => socialGraphInstance.handleFollowEvent(e));
+    this.snapshot().forEach(e => socialGraphInstance.handleEvent(e));
   }
 }

packages/app/src/Cache/IndexedDB.ts

@@ -0,0 +1,200 @@
+import Dexie, { Table } from "dexie";
+import { TaggedNostrEvent, ReqFilter as Filter } from "@snort/system";
+import * as Comlink from "comlink";
+
+type Tag = {
+  id: string;
+  eventId: string;
+  type: string;
+  value: string;
+};
+
+type SaveQueueEntry = { event: TaggedNostrEvent; tags: Tag[] };
+
+class IndexedDB extends Dexie {
+  events!: Table<TaggedNostrEvent>;
+  tags!: Table<Tag>;
+  private saveQueue: SaveQueueEntry[] = [];
+  private seenEvents = new Set<string>(); // LRU set maybe?
+  private seenFilters = new Set<string>();
+  private subscribedEventIds = new Set<string>();
+  private subscribedAuthors = new Set<string>();
+  private subscribedTags = new Set<string>();
+
+  constructor() {
+    super("EventDB");
+
+    this.version(5).stores({
+      events: "id, pubkey, kind, created_at, [pubkey+kind]",
+      tags: "id, eventId, [type+value]",
+    });
+
+    this.startInterval();
+  }
+
+  private startInterval() {
+    const processQueue = async () => {
+      if (this.saveQueue.length > 0) {
+        try {
+          const eventsToSave: TaggedNostrEvent[] = [];
+          const tagsToSave: Tag[] = [];
+          for (const item of this.saveQueue) {
+            eventsToSave.push(item.event);
+            tagsToSave.push(...item.tags);
+          }
+          await this.events.bulkPut(eventsToSave);
+          await this.tags.bulkPut(tagsToSave);
+        } catch (e) {
+          console.error(e);
+        } finally {
+          this.saveQueue = [];
+        }
+      }
+      setTimeout(() => processQueue(), 3000);
+    };
+
+    setTimeout(() => processQueue(), 3000);
+  }
+
+  handleEvent(event: TaggedNostrEvent) {
+    if (this.seenEvents.has(event.id)) {
+      return;
+    }
+    this.seenEvents.add(event.id);
+
+    // maybe we don't want event.kind 3 tags
+    const tags =
+      event.kind === 3
+        ? []
+        : event.tags
+            ?.filter(tag => {
+              if (tag[0] === "d") {
+                return true;
+              }
+              if (tag[0] === "e") {
+                return true;
+              }
+              // we're only interested in p tags where we are mentioned
+              if (tag[0] === "p") {
+                // && Key.isMine(tag[1])) {
+                return true;
+              }
+              return false;
+            })
+            .map(tag => ({
+              id: event.id.slice(0, 16) + "-" + tag[0].slice(0, 16) + "-" + tag[1].slice(0, 16),
+              eventId: event.id,
+              type: tag[0],
+              value: tag[1],
+            })) || [];
+
+    this.saveQueue.push({ event, tags });
+  }
+
+  _throttle(func, limit) {
+    let inThrottle;
+    return function (...args) {
+      if (!inThrottle) {
+        inThrottle = true;
+        setTimeout(() => {
+          inThrottle = false;
+          func.apply(this, args);
+        }, limit);
+      }
+    };
+  }
+
+  subscribeToAuthors = this._throttle(async function (callback: (event: TaggedNostrEvent) => void, limit?: number) {
+    const authors = [...this.subscribedAuthors];
+    this.subscribedAuthors.clear();
+    await this.events
+      .where("pubkey")
+      .anyOf(authors)
+      .limit(limit || 1000)
+      .each(callback);
+  }, 100);
+
+  subscribeToEventIds = this._throttle(async function (callback: (event: TaggedNostrEvent) => void) {
+    const ids = [...this.subscribedEventIds];
+    this.subscribedEventIds.clear();
+    await this.events.where("id").anyOf(ids).each(callback);
+  }, 100);
+
+  subscribeToTags = this._throttle(async function (callback: (event: TaggedNostrEvent) => void) {
+    const tagPairs = [...this.subscribedTags].map(tag => tag.split("|"));
+    this.subscribedTags.clear();
+    await this.tags
+      .where("[type+value]")
+      .anyOf(tagPairs)
+      .each(tag => this.subscribedEventIds.add(tag.eventId));
+
+    await this.subscribeToEventIds(callback);
+  }, 100);
+
+  async find(filter: Filter, callback: (event: TaggedNostrEvent) => void): Promise<void> {
+    if (!filter) return;
+
+    // make sure only 1 argument is passed
+    const cb = e => callback(e);
+
+    if (filter["#p"] && Array.isArray(filter["#p"])) {
+      for (const eventId of filter["#p"]) {
+        this.subscribedTags.add("p|" + eventId);
+      }
+
+      await this.subscribeToTags(cb);
+      return;
+    }
+
+    if (filter["#e"] && Array.isArray(filter["#e"])) {
+      for (const eventId of filter["#e"]) {
+        this.subscribedTags.add("e|" + eventId);
+      }
+
+      await this.subscribeToTags(cb);
+      return;
+    }
+
+    if (filter["#d"] && Array.isArray(filter["#d"])) {
+      for (const eventId of filter["#d"]) {
+        this.subscribedTags.add("d|" + eventId);
+      }
+
+      await this.subscribeToTags(cb);
+      return;
+    }
+
+    if (filter.ids?.length) {
+      filter.ids.forEach(id => this.subscribedEventIds.add(id));
+      await this.subscribeToEventIds(cb);
+      return;
+    }
+
+    if (filter.authors?.length) {
+      filter.authors.forEach(author => this.subscribedAuthors.add(author));
+      await this.subscribeToAuthors(cb);
+      return;
+    }
+
+    let query = this.events;
+    if (filter.kinds) {
+      query = query.where("kind").anyOf(filter.kinds);
+    }
+    if (filter.search) {
+      const regexp = new RegExp(filter.search, "i");
+      query = query.filter((event: Event) => event.content?.match(regexp));
+    }
+    if (filter.limit) {
+      query = query.limit(filter.limit);
+    }
+    // TODO test that the sort is actually working
+    await query.each(e => {
+      this.seenEvents.add(e.id);
+      cb(e);
+    });
+  }
+}
+
+const db = new IndexedDB();
+
+Comlink.expose(db);

packages/app/src/Element/SearchBox.tsx

@@ -8,9 +8,9 @@ import { NostrLink, tryParseNostrLink } from "@snort/system";
 import { useLocation, useNavigate } from "react-router-dom";
 import { unixNow } from "@snort/shared";
 import useTimelineFeed, { TimelineFeedOptions, TimelineSubject } from "../Feed/TimelineFeed";
-import { fuzzySearch, FuzzySearchResult } from "@/index";
 import ProfileImage from "@/Element/User/ProfileImage";
 import { socialGraphInstance } from "@snort/system";
+import fuzzySearch, { FuzzySearchResult } from "@/FuzzySearch";
 
 const MAX_RESULTS = 3;
 

packages/app/src/FuzzySearch.ts

@@ -1,6 +1,4 @@
 import Fuse from "fuse.js";
-import { socialGraphInstance } from "@snort/system";
-import { System } from ".";
 
 export type FuzzySearchResult = {
   pubkey: string;
@@ -9,7 +7,7 @@ export type FuzzySearchResult = {
   nip05?: string;
 };
 
-export const fuzzySearch = new Fuse<FuzzySearchResult>([], {
+const fuzzySearch = new Fuse<FuzzySearchResult>([], {
   keys: ["name", "username", { name: "nip05", weight: 0.5 }],
   threshold: 0.3,
   // sortFn here?
@@ -17,27 +15,27 @@ export const fuzzySearch = new Fuse<FuzzySearchResult>([], {
 
 const profileTimestamps = new Map<string, number>(); // is this somewhere in cache?
 
-System.on("event", ev => {
-  if (ev.kind === 0) {
-    const existing = profileTimestamps.get(ev.pubkey);
-    if (existing) {
-      if (existing > ev.created_at) {
-        return;
-      }
-      fuzzySearch.remove(doc => doc.pubkey === ev.pubkey);
-    }
-    profileTimestamps.set(ev.pubkey, ev.created_at);
-    try {
-      const data = JSON.parse(ev.content);
-      if (ev.pubkey && (data.name || data.username || data.nip05)) {
-        data.pubkey = ev.pubkey;
-        fuzzySearch.add(data);
-      }
-    } catch (e) {
-      console.error(e);
-    }
+export const addEventToFuzzySearch = ev => {
+  if (ev.kind !== 0) {
+    return;
   }
-  if (ev.kind === 3) {
-    socialGraphInstance.handleFollowEvent(ev);
+  const existing = profileTimestamps.get(ev.pubkey);
+  if (existing) {
+    if (existing > ev.created_at) {
+      return;
+    }
+    fuzzySearch.remove(doc => doc.pubkey === ev.pubkey);
   }
-});
+  profileTimestamps.set(ev.pubkey, ev.created_at);
+  try {
+    const data = JSON.parse(ev.content);
+    if (ev.pubkey && (data.name || data.username || data.nip05)) {
+      data.pubkey = ev.pubkey;
+      fuzzySearch.add(data);
+    }
+  } catch (e) {
+    console.error(e);
+  }
+};
+
+export default fuzzySearch;

packages/app/src/index.tsx

@@ -28,6 +28,7 @@ import {
   PowWorker,
   encodeTLVEntries,
   socialGraphInstance,
+  TaggedNostrEvent,
 } from "@snort/system";
 import PowWorkerURL from "@snort/system/src/pow-worker.ts?worker&url";
 import { SnortContext } from "@snort/system-react";
@@ -62,10 +63,13 @@ import { AboutPage } from "@/Pages/About";
 import { OnboardingRoutes } from "@/Pages/onboarding";
 import { setupWebLNWalletConfig } from "@/Wallet/WebLN";
 import { Wallets } from "@/Wallet";
-import Fuse from "fuse.js";
 import NetworkGraph from "@/Pages/NetworkGraph";
 import WalletPage from "./Pages/WalletPage";
 
+import IndexedDBWorker from "./Cache/IndexedDB?worker";
+import * as Comlink from "comlink";
+import { addEventToFuzzySearch } from "@/FuzzySearch";
+
 declare global {
   interface Window {
     plausible?: (tag: string, e?: object) => void;
@@ -101,6 +105,8 @@ const hasWasm = "WebAssembly" in globalThis;
 const DefaultPowWorker = hasWasm ? undefined : new PowWorker(PowWorkerURL);
 export const GetPowWorker = () => (hasWasm ? new WasmPowWorker() : unwrap(DefaultPowWorker));
 
+const indexedDB = Comlink.wrap(new IndexedDBWorker());
+
 /**
  * Singleton nostr system
  */
@@ -120,47 +126,25 @@ System.on("auth", async (c, r, cb) => {
   }
 });
 
-export type FuzzySearchResult = {
-  pubkey: string;
-  name?: string;
-  display_name?: string;
-  nip05?: string;
-};
-
-export const fuzzySearch = new Fuse<FuzzySearchResult>([], {
-  keys: ["name", "display_name", { name: "nip05", weight: 0.5 }],
-  threshold: 0.3,
-  // sortFn here?
-});
-
-const profileTimestamps = new Map<string, number>();
-
-// how to also add entries from ProfileCache?
 System.on("event", (_, ev) => {
-  if (ev.kind === 0) {
-    const existing = profileTimestamps.get(ev.pubkey);
-    if (existing) {
-      if (existing > ev.created_at) {
-        return;
-      }
-      fuzzySearch.remove(doc => doc.pubkey === ev.pubkey);
-    }
-    profileTimestamps.set(ev.pubkey, ev.created_at);
-    try {
-      const data = JSON.parse(ev.content);
-      if (ev.pubkey && (data.name || data.display_name || data.nip05)) {
-        data.pubkey = ev.pubkey;
-        fuzzySearch.add(data);
-      }
-    } catch (e) {
-      console.error(e);
-    }
-  }
-  if (ev.kind === 3) {
-    socialGraphInstance.handleFollowEvent(ev);
+  addEventToFuzzySearch(ev);
+  socialGraphInstance.handleEvent(ev);
+  if (CONFIG.useIndexedDBEvents) {
+    indexedDB.handleEvent(ev);
   }
 });
 
+if (CONFIG.useIndexedDBEvents) {
+  System.on("request", (filter: ReqFilter) => {
+    indexedDB.find(
+      filter,
+      Comlink.proxy((e: TaggedNostrEvent) => {
+        System.HandleEvent(e);
+      }),
+    );
+  });
+}
+
 async function fetchProfile(key: string) {
   try {
     throwIfOffline();

packages/system/src/SocialGraph/SocialGraph.ts

@@ -47,7 +47,10 @@ export default class SocialGraph {
     }
   }
 
-  handleFollowEvent(event: NostrEvent) {
+  handleEvent(event: NostrEvent) {
+    if (event.kind !== 3) {
+      return;
+    }
     try {
       const author = ID(event.pubkey);
       const timestamp = event.created_at;

packages/system/src/nostr-system.ts

@@ -2,7 +2,7 @@ import debug from "debug";
 import EventEmitter from "eventemitter3";
 
 import { unwrap, FeedCache } from "@snort/shared";
-import { NostrEvent, TaggedNostrEvent } from "./nostr";
+import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr";
 import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
 import { Query } from "./query";
 import { NoteCollection, NoteStore } from "./note-collection";
@@ -31,6 +31,7 @@ export interface NostrSystemEvents {
   change: (state: SystemSnapshot) => void;
   auth: (challenge: string, relay: string, cb: (ev: NostrEvent) => void) => void;
   event: (subId: string, ev: TaggedNostrEvent) => void;
+  request: (filter: ReqFilter) => void;
 }
 
 export interface NostrsystemProps {
@@ -316,6 +317,10 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
     }
     qSend.filters = fNew;
 
+    fNew.forEach(f => {
+      this.emit("request", f);
+    });
+
     if (qSend.relay) {
       this.#log("Sending query to %s %O", qSend.relay, qSend);
       const s = this.#pool.getConnection(qSend.relay);

yarn.lock

@@ -4898,6 +4898,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"comlink@npm:^4.4.1":
+  version: 4.4.1
+  resolution: "comlink@npm:4.4.1"
+  checksum: 16d58a8f590087fc45432e31d6c138308dfd4b75b89aec0b7f7bb97ad33d810381bd2b1e608a1fb2cf05979af9cbfcdcaf1715996d5fcf77aeb013b6da3260af
+  languageName: node
+  linkType: hard
+
 "commander@npm:^2.20.0":
   version: 2.20.3
   resolution: "commander@npm:2.20.3"
@@ -9918,6 +9925,7 @@ __metadata:
   dependencies:
     "@cloudflare/workers-types": ^4.20230307.0
     "@tauri-apps/cli": ^1.2.3
+    comlink: ^4.4.1
     eslint: ^8.48.0
     prettier: ^3.0.3
     typescript: ^5.2.2