feat: worker-relay delete

fix: worker-relay insert replacable events duplicate

packages/app/package.json

@@ -4,11 +4,11 @@
   "dependencies": {
     "@cashu/cashu-ts": "^1.0.0-rc.3",
     "@here/maps-api-for-javascript": "^1.50.0",
-    "@noble/curves": "^1.0.0",
-    "@noble/hashes": "^1.3.3",
-    "@scure/base": "^1.1.1",
-    "@scure/bip32": "^1.3.0",
-    "@scure/bip39": "^1.1.1",
+    "@noble/curves": "^1.4.0",
+    "@noble/hashes": "^1.4.0",
+    "@scure/base": "^1.1.6",
+    "@scure/bip32": "^1.4.0",
+    "@scure/bip39": "^1.3.0",
     "@snort/shared": "workspace:*",
     "@snort/system": "workspace:*",
     "@snort/system-react": "workspace:*",
@@ -16,7 +16,7 @@
     "@snort/system-web": "workspace:*",
     "@snort/wallet": "workspace:*",
     "@snort/worker-relay": "workspace:*",
-    "@szhsin/react-menu": "^3.3.1",
+    "@szhsin/react-menu": "^3.5.3",
     "@uidotdev/usehooks": "^2.4.1",
     "@void-cat/api": "^1.0.12",
     "classnames": "^2.3.2",
@@ -102,7 +102,7 @@
     "@typescript-eslint/eslint-plugin": "^6.1.0",
     "@typescript-eslint/parser": "^6.1.0",
     "@vitejs/plugin-react": "^4.2.0",
-    "@webbtc/webln-types": "^2.1.0",
+    "@webbtc/webln-types": "^3.0.0",
     "@webscopeio/react-textarea-autocomplete": "^4.9.2",
     "@welldone-software/why-did-you-render": "^8.0.1",
     "autoprefixer": "^10.4.16",

packages/app/src/Pages/CacheDebug.tsx

@@ -0,0 +1,59 @@
+import { NostrEvent, TaggedNostrEvent } from "@snort/system";
+import { SnortContext } from "@snort/system-react";
+import { useContext, useState } from "react";
+
+import AsyncButton from "@/Components/Button/AsyncButton";
+
+export function DebugPage() {
+  const system = useContext(SnortContext);
+  const [filter, setFilter] = useState("");
+  const [event, setEvent] = useState("");
+  const [results, setResult] = useState<Array<TaggedNostrEvent>>([]);
+
+  async function search() {
+    if (filter && system.cacheRelay) {
+      const r = await system.cacheRelay.query(["REQ", "test", JSON.parse(filter)]);
+      setResult(r.map(a => ({ ...a, relays: [] })));
+    }
+  }
+
+  async function insert() {
+    if (event && system.cacheRelay) {
+      const r = await system.cacheRelay.event(JSON.parse(event) as NostrEvent);
+      setResult([
+        {
+          content: JSON.stringify(r),
+        } as unknown as TaggedNostrEvent,
+      ]);
+    }
+  }
+
+  async function removeEvents() {
+    if (filter && system.cacheRelay) {
+      const r = await system.cacheRelay.delete(["REQ", "delete-events", JSON.parse(filter)]);
+      setResult(r.map(a => ({ id: a }) as TaggedNostrEvent));
+    }
+  }
+  return (
+    <div className="flex flex-col gap-2">
+      <h3>Cache Query</h3>
+      <textarea value={filter} onChange={e => setFilter(e.target.value)} placeholder="nostr filter" />
+      <AsyncButton onClick={() => search()}>Query</AsyncButton>
+      <AsyncButton onClick={() => removeEvents()} className="!bg-red-500">
+        Delete
+      </AsyncButton>
+
+      <h3>Manual Insert</h3>
+      <textarea value={event} onChange={e => setEvent(e.target.value)} placeholder="nostr event" />
+      <AsyncButton onClick={() => insert()}>Insert</AsyncButton>
+      <div className="p-4 overflow-hidden">
+        <h4>Results: {results.length}</h4>
+        {results?.map(a => (
+          <pre key={a.id} className="text-mono text-xs text-pretty">
+            {JSON.stringify(a, undefined, 2)}
+          </pre>
+        ))}
+      </div>
+    </div>
+  );
+}

packages/app/src/Pages/settings/Cache.tsx

@@ -1,6 +1,7 @@
 import { FeedCache } from "@snort/shared";
 import { ReactNode, useEffect, useState, useSyncExternalStore } from "react";
 import { FormattedMessage, FormattedNumber } from "react-intl";
+import { useNavigate } from "react-router-dom";
 
 import { GiftsCache, Relay, RelayMetrics } from "@/Cache";
 import AsyncButton from "@/Components/Button/AsyncButton";
@@ -32,7 +33,6 @@ function CacheDetails<T>({ cache, name }: { cache: FeedCache<T>; name: ReactNode
         <small>
           <FormattedMessage
             defaultMessage="{count} ({count2} in memory)"
-            id="geppt8"
             values={{
               count: <FormattedNumber value={cache.keysOnTable().length} />,
               count2: <FormattedNumber value={snapshot.length} />,
@@ -53,6 +53,7 @@ function RelayCacheStats() {
   const [counts, setCounts] = useState<Record<string, number>>({});
   const [myEvents, setMyEvents] = useState<number>(0);
   const login = useLogin();
+  const navigate = useNavigate();
 
   useEffect(() => {
     Relay.summary().then(setCounts);
@@ -69,7 +70,6 @@ function RelayCacheStats() {
           <p>
             <FormattedMessage
               defaultMessage="My events: {n}"
-              id="lEnclp"
               values={{
                 n: <FormattedNumber value={myEvents} />,
               }}
@@ -124,6 +124,9 @@ function RelayCacheStats() {
           }}>
           <FormattedMessage defaultMessage="Dump" />
         </AsyncButton>
+        <AsyncButton onClick={() => navigate("/cache-debug")}>
+          <FormattedMessage defaultMessage="Debug" />
+        </AsyncButton>
       </div>
     </div>
   );

packages/app/src/index.tsx

@@ -13,6 +13,7 @@ import { IntlProvider } from "@/Components/IntlProvider/IntlProvider";
 import { db } from "@/Db";
 import { addCachedMetadataToFuzzySearch } from "@/Db/FuzzySearch";
 import { AboutPage } from "@/Pages/About";
+import { DebugPage } from "@/Pages/CacheDebug";
 import { SnortDeckLayout } from "@/Pages/Deck/DeckLayout";
 import DonatePage from "@/Pages/Donate/DonatePage";
 import ErrorPage from "@/Pages/ErrorPage";
@@ -74,6 +75,10 @@ async function initSite() {
 let didInit = false;
 const mainRoutes = [
   ...RootRoutes,
+  {
+    path: "/cache-debug",
+    element: <DebugPage />,
+  },
   {
     path: "/help",
     element: <HelpPage />,

packages/shared/package.json

@@ -13,9 +13,9 @@
     "build": "rm -rf dist && tsc"
   },
   "dependencies": {
-    "@noble/curves": "^1.2.0",
-    "@noble/hashes": "^1.3.2",
-    "@scure/base": "^1.1.2",
+    "@noble/curves": "^1.4.0",
+    "@noble/hashes": "^1.4.0",
+    "@scure/base": "^1.1.6",
     "debug": "^4.3.4",
     "eventemitter3": "^5.0.1",
     "light-bolt11-decoder": "^3.0.0"

packages/system/package.json

@@ -19,7 +19,7 @@
   ],
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@peculiar/webcrypto": "^1.4.3",
+    "@peculiar/webcrypto": "^1.4.6",
     "@types/debug": "^4.1.8",
     "@types/jest": "^29.5.11",
     "@types/lokijs": "^1.5.14",
@@ -33,10 +33,10 @@
     "typescript": "^5.2.2"
   },
   "dependencies": {
-    "@noble/curves": "^1.2.0",
-    "@noble/hashes": "^1.3.2",
-    "@nostr-dev-kit/ndk": "^2.7.1",
-    "@scure/base": "^1.1.2",
+    "@noble/curves": "^1.4.0",
+    "@noble/hashes": "^1.4.0",
+    "@nostr-dev-kit/ndk": "^2.8.2",
+    "@scure/base": "^1.1.6",
     "@snort/shared": "^1.0.15",
     "@stablelib/xchacha20": "^1.0.1",
     "debug": "^4.3.4",

packages/system/src/cache-relay.ts

@@ -14,4 +14,9 @@ export interface CacheRelay {
    * Read event from cache relay
    */
   query(req: ReqCommand): Promise<Array<NostrEvent>>;
+
+  /**
+   * Delete events by filter
+   */
+  delete(req: ReqCommand): Promise<Array<string>>;
 }

packages/worker-relay/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snort/worker-relay",
-  "version": "1.0.10",
+  "version": "1.1.0",
   "description": "A nostr relay in a service worker",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -18,7 +18,7 @@
     "dist"
   ],
   "dependencies": {
-    "@sqlite.org/sqlite-wasm": "^3.45.1-build1",
+    "@sqlite.org/sqlite-wasm": "^3.45.3-build3",
     "eventemitter3": "^5.0.1",
     "uuid": "^9.0.1"
   },

packages/worker-relay/src/interface.ts

@@ -63,6 +63,11 @@ export class WorkerRelayInterface {
     return await this.#workerRpc<ReqCommand, number>("count", req);
   }
 
+  async delete(req: ReqCommand) {
+    console.debug("DELETE", req);
+    return await this.#workerRpc<ReqCommand, Array<string>>("delete", req);
+  }
+
   async summary() {
     return await this.#workerRpc<void, Record<string, number>>("summary");
   }

packages/worker-relay/src/memory-relay.ts

@@ -80,6 +80,13 @@ export class InMemoryRelay extends EventEmitter<RelayHandlerEvents> implements R
     return ret;
   }
 
+  delete(filter: ReqFilter) {
+    const forDelete = this.req("ids-for-delete", { ...filter, ids_only: true }) as Array<string>;
+    forDelete.forEach(a => this.#events.delete(a));
+
+    return forDelete;
+  }
+
   setEventMetadata(_id: string, _meta: EventMetadata) {
     return;
   }

packages/worker-relay/src/sqlite-relay.ts

@@ -99,6 +99,7 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
   }
 
   #deleteById(db: Database, ids: Array<string>) {
+    if (ids.length === 0) return;
     db.exec(`delete from events where id in (${this.#repeatParams(ids.length)})`, {
       bind: ids,
     });
@@ -126,6 +127,9 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
           this.#deleteById(db, toDelete);
         }
         return false;
+      } else {
+        // delete older versions
+        this.#deleteById(db, oldEvents);
       }
     }
     if (ev.kind >= 30_000 && ev.kind < 40_000) {
@@ -142,6 +146,9 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
           this.#deleteById(db, toDelete);
         }
         return false;
+      } else {
+        // delete older versions
+        this.#deleteById(db, oldEvents);
       }
     }
     db.exec("insert or ignore into events(id, pubkey, created, kind, json) values(?,?,?,?,?)", {
@@ -196,6 +203,32 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
     return results;
   }
 
+  /**
+   * Delete events by nostr filter
+   */
+  delete(req: ReqFilter) {
+    this.#log(`Starting delete of ${JSON.stringify(req)}`);
+    const start = unixNowMs();
+    const for_delete = this.req("ids-for-delete", { ...req, ids_only: true }) as Array<string>;
+
+    const grouped = for_delete.reduce(
+      (acc, v, i) => {
+        const batch = (i / 1000).toFixed(0);
+        acc[batch] ??= [];
+        acc[batch].push(v);
+        return acc;
+      },
+      {} as Record<string, Array<string>>,
+    );
+    this.#log(`Starting delete of ${Object.keys(grouped).length} batches`);
+    Object.entries(grouped).forEach(([batch, ids]) => {
+      this.#deleteById(this.db!, ids);
+    });
+    const time = unixNowMs() - start;
+    this.#log(`Delete ${for_delete.length} events took ${time.toLocaleString()}ms`);
+    return for_delete;
+  }
+
   /**
    * Get a summary about events table
    */
@@ -231,7 +264,7 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
     return new Uint8Array();
   }
 
-  #buildQuery(req: ReqFilter, count = false): [string, Array<any>] {
+  #buildQuery(req: ReqFilter, count = false, remove = false): [string, Array<any>] {
     const conditions: Array<string> = [];
     const params: Array<any> = [];
 
@@ -241,7 +274,11 @@ export class SqliteRelay extends EventEmitter<RelayHandlerEvents> implements Rel
     } else if (req.ids_only === true) {
       resultType = "id";
     }
-    let sql = `select ${resultType} from events`;
+    let operation = `select ${resultType}`;
+    if (remove) {
+      operation = "delete";
+    }
+    let sql = `${operation} from events`;
     const tags = Object.entries(req).filter(([k]) => k.startsWith("#"));
     let tx = 0;
     for (const [key, values] of tags) {

packages/worker-relay/src/types.ts

@@ -12,7 +12,8 @@ export type WorkerMessageCommand =
   | "emit-event"
   | "forYouFeed"
   | "setEventMetadata"
-  | "debug";
+  | "debug"
+  | "delete";
 
 export interface WorkerMessage<T> {
   id: string;
@@ -70,6 +71,7 @@ export interface RelayHandler extends EventEmitter<RelayHandlerEvents> {
   count(req: ReqFilter): number;
   summary(): Record<string, number>;
   dump(): Promise<Uint8Array>;
+  delete(req: ReqFilter): Array<string>;
   setEventMetadata(id: string, meta: EventMetadata): void;
 }
 

packages/worker-relay/src/worker.ts

@@ -4,7 +4,16 @@ import { SqliteRelay } from "./sqlite-relay";
 import { InMemoryRelay } from "./memory-relay";
 import { setLogging } from "./debug";
 import { WorkQueueItem, barrierQueue, processWorkQueue } from "./queue";
-import { NostrEvent, RelayHandler, ReqCommand, ReqFilter, WorkerMessage, unixNowMs, EventMetadata } from "./types";
+import {
+  NostrEvent,
+  RelayHandler,
+  ReqCommand,
+  ReqFilter,
+  WorkerMessage,
+  unixNowMs,
+  EventMetadata,
+  OkResponse,
+} from "./types";
 import { getForYouFeed } from "./forYouFeed";
 
 let relay: RelayHandler | undefined;
@@ -86,8 +95,13 @@ const handleMsg = async (port: MessagePort | DedicatedWorkerGlobalScope, ev: Mes
         break;
       }
       case "event": {
-        eventWriteQueue.push(msg.args as NostrEvent);
-        reply(msg.id, true);
+        const ev = msg.args as NostrEvent;
+        eventWriteQueue.push(ev);
+        reply(msg.id, {
+          ok: true,
+          id: ev.id,
+          relay: "",
+        } as OkResponse);
         break;
       }
       case "close": {
@@ -130,6 +144,20 @@ const handleMsg = async (port: MessagePort | DedicatedWorkerGlobalScope, ev: Mes
         });
         break;
       }
+      case "delete": {
+        console.debug("DELETE", msg.args);
+        await barrierQueue(cmdQueue, async () => {
+          const req = msg.args as ReqCommand;
+          let results = [];
+          const filters = req.slice(2) as Array<ReqFilter>;
+          for (const r of filters) {
+            const c = relay!.delete(r);
+            results.push(...c);
+          }
+          reply(msg.id, results);
+        });
+        break;
+      }
       case "summary": {
         await barrierQueue(cmdQueue, async () => {
           const res = relay!.summary();