From 21e88b06cb050912ca5e23a190063968a2db8274 Mon Sep 17 00:00:00 2001 From: kieran Date: Mon, 16 Sep 2024 10:55:15 +0100 Subject: [PATCH] feat: track event seen on relays fix: dump/clear commands --- .../Components/Event/Note/ReactionsModal.tsx | 8 +- packages/app/src/Components/Feed/Timeline.tsx | 6 +- packages/app/src/Hooks/useWoT.ts | 11 +++ packages/app/src/Pages/settings/Cache.tsx | 47 +++++++---- packages/system/src/cache-relay.ts | 6 +- packages/system/src/query-manager.ts | 2 +- packages/worker-relay/package.json | 2 +- packages/worker-relay/src/interface.ts | 4 + packages/worker-relay/src/memory-relay.ts | 8 +- .../worker-relay/src/sqlite/migrations.ts | 10 +++ .../worker-relay/src/sqlite/sqlite-relay.ts | 82 ++++++++++++++----- packages/worker-relay/src/types.ts | 4 +- packages/worker-relay/src/worker.ts | 5 ++ 13 files changed, 141 insertions(+), 54 deletions(-) create mode 100644 packages/app/src/Hooks/useWoT.ts diff --git a/packages/app/src/Components/Event/Note/ReactionsModal.tsx b/packages/app/src/Components/Event/Note/ReactionsModal.tsx index ab621243..962c8753 100644 --- a/packages/app/src/Components/Event/Note/ReactionsModal.tsx +++ b/packages/app/src/Components/Event/Note/ReactionsModal.tsx @@ -1,6 +1,6 @@ import "./ReactionsModal.css"; -import { NostrLink, socialGraphInstance, TaggedNostrEvent } from "@snort/system"; +import { NostrLink, TaggedNostrEvent } from "@snort/system"; import { useEventReactions, useReactions } from "@snort/system-react"; import { useMemo, useState } from "react"; import { FormattedMessage, MessageDescriptor, useIntl } from "react-intl"; @@ -11,6 +11,7 @@ import Modal from "@/Components/Modal/Modal"; import TabSelectors, { Tab } from "@/Components/TabSelectors/TabSelectors"; import ProfileImage from "@/Components/User/ProfileImage"; import ZapAmount from "@/Components/zap-amount"; +import useWoT from "@/Hooks/useWoT"; import messages from "../../messages"; @@ -29,10 +30,7 @@ const ReactionsModal = ({ onClose, event, initialTab = 0 }: ReactionsModalProps) const { reactions, zaps, reposts } = useEventReactions(link, related); const { positive, negative } = reactions; - const sortEvents = (events: Array) => - events.sort( - (a, b) => socialGraphInstance.getFollowDistance(a.pubkey) - socialGraphInstance.getFollowDistance(b.pubkey), - ); + const { sortEvents } = useWoT(); const likes = useMemo(() => sortEvents([...positive]), [positive]); const dislikes = useMemo(() => sortEvents([...negative]), [negative]); diff --git a/packages/app/src/Components/Feed/Timeline.tsx b/packages/app/src/Components/Feed/Timeline.tsx index e98b2405..1a035691 100644 --- a/packages/app/src/Components/Feed/Timeline.tsx +++ b/packages/app/src/Components/Feed/Timeline.tsx @@ -1,7 +1,7 @@ import "./Timeline.css"; import { unixNow } from "@snort/shared"; -import { socialGraphInstance, TaggedNostrEvent } from "@snort/system"; +import { TaggedNostrEvent } from "@snort/system"; import { useCallback, useMemo, useState } from "react"; import { DisplayAs, DisplayAsSelector } from "@/Components/Feed/DisplayAsSelector"; @@ -9,6 +9,7 @@ import { TimelineRenderer } from "@/Components/Feed/TimelineRenderer"; import useTimelineFeed, { TimelineFeed, TimelineSubject } from "@/Feed/TimelineFeed"; import useHistoryState from "@/Hooks/useHistoryState"; import useLogin from "@/Hooks/useLogin"; +import useWoT from "@/Hooks/useWoT"; import { dedupeByPubkey } from "@/Utils"; export interface TimelineProps { @@ -41,6 +42,7 @@ const Timeline = (props: TimelineProps) => { const feed: TimelineFeed = useTimelineFeed(props.subject, feedOptions); const displayAsInitial = props.displayAs ?? login.feedDisplayAs ?? "list"; const [displayAs, setDisplayAs] = useState(displayAsInitial); + const wot = useWoT(); const filterPosts = useCallback( (nts: readonly TaggedNostrEvent[]) => { @@ -48,7 +50,7 @@ const Timeline = (props: TimelineProps) => { if (props.followDistance === undefined) { return true; } - const followDistance = socialGraphInstance.getFollowDistance(a.pubkey); + const followDistance = wot.followDistance(a.pubkey); return followDistance === props.followDistance; }; return nts diff --git a/packages/app/src/Hooks/useWoT.ts b/packages/app/src/Hooks/useWoT.ts new file mode 100644 index 00000000..82f42c35 --- /dev/null +++ b/packages/app/src/Hooks/useWoT.ts @@ -0,0 +1,11 @@ +import { TaggedNostrEvent } from "@snort/system"; +import { socialGraphInstance } from "@snort/system/dist/SocialGraph/SocialGraph"; + +export default function useWoT() { + const sg = socialGraphInstance; + return { + sortEvents: (events: Array) => + events.sort((a, b) => sg.getFollowDistance(a.pubkey) - sg.getFollowDistance(b.pubkey)), + followDistance: sg.getFollowDistance, + }; +} diff --git a/packages/app/src/Pages/settings/Cache.tsx b/packages/app/src/Pages/settings/Cache.tsx index 83060ef3..580c3e17 100644 --- a/packages/app/src/Pages/settings/Cache.tsx +++ b/packages/app/src/Pages/settings/Cache.tsx @@ -123,24 +123,35 @@ function RelayCacheStats() {
- {}}> - - - { - const data = new Uint8Array(); - const url = URL.createObjectURL( - new File([data], "snort.db", { - type: "application/octet-stream", - }), - ); - const a = document.createElement("a"); - a.href = url; - a.download = "snort.db"; - a.click(); - }}> - - + {Relay instanceof WorkerRelayInterface && ( + <> + { + if (Relay instanceof WorkerRelayInterface) { + await Relay.wipe(); + } + }}> + + + { + const data = Relay instanceof WorkerRelayInterface ? await Relay.dump() : undefined; + if (data) { + const url = URL.createObjectURL( + new File([data], "snort.db", { + type: "application/octet-stream", + }), + ); + const a = document.createElement("a"); + a.href = url; + a.download = "snort.db"; + a.click(); + } + }}> + + + + )} navigate("/cache-debug")}> diff --git a/packages/system/src/cache-relay.ts b/packages/system/src/cache-relay.ts index 1b222d6c..536bc655 100644 --- a/packages/system/src/cache-relay.ts +++ b/packages/system/src/cache-relay.ts @@ -1,4 +1,4 @@ -import { NostrEvent, OkResponse, ReqCommand } from "./nostr"; +import { OkResponse, ReqCommand, TaggedNostrEvent } from "./nostr"; /** * A cache relay is an always available local (local network / browser worker) relay @@ -8,12 +8,12 @@ export interface CacheRelay { /** * Write event to cache relay */ - event(ev: NostrEvent): Promise; + event(ev: TaggedNostrEvent): Promise; /** * Read event from cache relay */ - query(req: ReqCommand): Promise>; + query(req: ReqCommand): Promise>; /** * Delete events by filter diff --git a/packages/system/src/query-manager.ts b/packages/system/src/query-manager.ts index a373ccb0..f9021fa0 100644 --- a/packages/system/src/query-manager.ts +++ b/packages/system/src/query-manager.ts @@ -107,8 +107,8 @@ export class QueryManager extends EventEmitter { // fetch results from cache first, flag qSend for sync if (this.#system.cacheRelay) { const data = await this.#system.cacheRelay.query(["REQ", q.id, ...filters]); + syncFrom = data; if (data.length > 0) { - syncFrom = data.map(a => ({ ...a, relays: [] })); this.#log("Adding from cache %s %O", q.id, data); q.feed.add(syncFrom); } diff --git a/packages/worker-relay/package.json b/packages/worker-relay/package.json index 6494d478..c1e61d26 100644 --- a/packages/worker-relay/package.json +++ b/packages/worker-relay/package.json @@ -1,6 +1,6 @@ { "name": "@snort/worker-relay", - "version": "1.2.0", + "version": "1.3.0", "description": "A nostr relay in a service worker", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker-relay/src/interface.ts b/packages/worker-relay/src/interface.ts index 07a13f2d..a1e2d676 100644 --- a/packages/worker-relay/src/interface.ts +++ b/packages/worker-relay/src/interface.ts @@ -79,6 +79,10 @@ export class WorkerRelayInterface { return await this.#workerRpc("dumpDb"); } + async wipe() { + return await this.#workerRpc("wipe"); + } + async forYouFeed(pubkey: string) { return await this.#workerRpc>("forYouFeed", pubkey); } diff --git a/packages/worker-relay/src/memory-relay.ts b/packages/worker-relay/src/memory-relay.ts index f273e213..533b070a 100644 --- a/packages/worker-relay/src/memory-relay.ts +++ b/packages/worker-relay/src/memory-relay.ts @@ -34,13 +34,19 @@ export class InMemoryRelay extends EventEmitter implements R } dump(): Promise { - return Promise.resolve(new Uint8Array()); + const enc = new TextEncoder(); + return Promise.resolve(enc.encode(JSON.stringify(this.#events.values()))); } close(): void { // nothing } + wipe() { + this.#events = new Map(); + return Promise.resolve(); + } + event(ev: NostrEvent) { if (this.#events.has(ev.id)) return false; this.#events.set(ev.id, ev); diff --git a/packages/worker-relay/src/sqlite/migrations.ts b/packages/worker-relay/src/sqlite/migrations.ts index ea3b412c..8ba3e33b 100644 --- a/packages/worker-relay/src/sqlite/migrations.ts +++ b/packages/worker-relay/src/sqlite/migrations.ts @@ -10,6 +10,7 @@ const migrations = [ { version: 3, script: migrate_v3 }, { version: 4, script: migrate_v4 }, { version: 5, script: migrate_v5 }, + { version: 6, script: migrate_v6 }, ]; async function migrate(relay: SqliteRelay) { @@ -103,4 +104,13 @@ async function migrate_v5(relay: SqliteRelay) { }); } +async function migrate_v6(relay: SqliteRelay) { + relay.db?.transaction(db => { + db.exec("ALTER TABLE events ADD COLUMN relays TEXT"); + db.exec("insert into __migration values(6, ?)", { + bind: [new Date().getTime() / 1000], + }); + }); +} + export default migrate; diff --git a/packages/worker-relay/src/sqlite/sqlite-relay.ts b/packages/worker-relay/src/sqlite/sqlite-relay.ts index ee84759f..40ba81f8 100644 --- a/packages/worker-relay/src/sqlite/sqlite-relay.ts +++ b/packages/worker-relay/src/sqlite/sqlite-relay.ts @@ -1,4 +1,4 @@ -import sqlite3InitModule, { Database, Sqlite3Static } from "@sqlite.org/sqlite-wasm"; +import sqlite3InitModule, { Database, SAHPoolUtil, Sqlite3Static } from "@sqlite.org/sqlite-wasm"; import { EventEmitter } from "eventemitter3"; import { EventMetadata, NostrEvent, RelayHandler, RelayHandlerEvents, ReqFilter, unixNowMs } from "../types"; import migrate from "./migrations"; @@ -12,6 +12,7 @@ export class SqliteRelay extends EventEmitter implements Rel #sqlite?: Sqlite3Static; #log = (msg: string, ...args: Array) => debugLog("SqliteRelay", msg, ...args); db?: Database; + #pool?: SAHPoolUtil; #seenInserts = new Set(); /** @@ -45,8 +46,8 @@ export class SqliteRelay extends EventEmitter implements Rel if (!this.#sqlite) throw new Error("Must call init first"); if (this.db) return; - const pool = await this.#sqlite.installOpfsSAHPoolVfs({}); - this.db = new pool.OpfsSAHPoolDb(path); + this.#pool = await this.#sqlite.installOpfsSAHPoolVfs({}); + this.db = new this.#pool.OpfsSAHPoolDb(path); this.#log(`Opened ${this.db.filename}`); /*this.db.exec( `PRAGMA cache_size=${32 * 1024 @@ -54,6 +55,19 @@ export class SqliteRelay extends EventEmitter implements Rel );*/ } + /** + * Delete all data + */ + async wipe() { + if (this.#pool && this.db) { + const dbName = this.db.filename; + this.close(); + await this.#pool.wipeFiles(); + await this.#open(dbName); + await migrate(this); + } + } + close() { this.db?.close(); this.db = undefined; @@ -157,8 +171,15 @@ export class SqliteRelay extends EventEmitter implements Rel this.#deleteById(db, oldEvents); } } - db.exec("insert or ignore into events(id, pubkey, created, kind, json) values(?,?,?,?,?)", { - bind: [ev.id, ev.pubkey, ev.created_at, ev.kind, JSON.stringify(ev)], + + // remove relays from event json + const evInsert = { + ...ev, + } as NostrEvent; + delete evInsert["relays"]; + + db.exec("insert or ignore into events(id, pubkey, created, kind, json, relays) values(?,?,?,?,?,?)", { + bind: [ev.id, ev.pubkey, ev.created_at, ev.kind, JSON.stringify(evInsert), (ev.relays ?? []).join(",")], }); const insertedEvents = db.changes(); if (insertedEvents > 0) { @@ -169,12 +190,33 @@ export class SqliteRelay extends EventEmitter implements Rel } this.insertIntoSearchIndex(db, ev); } else { + this.#updateRelays(db, ev); return 0; } this.#seenInserts.add(ev.id); return insertedEvents; } + /** + * Append relays + */ + #updateRelays(db: Database, ev: NostrEvent) { + const relays = db.selectArrays("select relays from events where id = ?", [ev.id]); + const oldRelays = new Set((relays?.at(0)?.at(0) as string | null)?.split(",") ?? []); + let hasNew = false; + for (const r of ev.relays ?? []) { + if (!oldRelays.has(r)) { + oldRelays.add(r); + hasNew = true; + } + } + if (hasNew) { + db.exec("update events set relays = ? where id = ?", { + bind: [[...oldRelays].join(","), ev.id], + }); + } + } + /** * Query relay by nostr filter */ @@ -188,7 +230,11 @@ export class SqliteRelay extends EventEmitter implements Rel if (req.ids_only === true) { return a[0] as string; } - return JSON.parse(a[0] as string) as NostrEvent; + const ev = JSON.parse(a[0] as string) as NostrEvent; + return { + ...ev, + relays: (a[1] as string | null)?.split(","), + }; }) ?? []; const time = unixNowMs() - start; this.#log(`Query ${id} results took ${time.toLocaleString()}ms`); @@ -252,22 +298,14 @@ export class SqliteRelay extends EventEmitter implements Rel */ async dump() { const filePath = String(this.db?.filename ?? ""); - try { - this.db?.close(); - this.db = undefined; - const dir = await navigator.storage.getDirectory(); - // @ts-expect-error - for await (const [name, file] of dir) { - if (`/${name}` === filePath) { - const fh = await (file as FileSystemFileHandle).getFile(); - const ret = new Uint8Array(await fh.arrayBuffer()); - return ret; - } + if (this.db && this.#pool) { + try { + return await this.#pool.exportFile(`/${filePath}`); + } catch (e) { + console.error(e); + } finally { + await this.#open(filePath); } - } catch (e) { - console.error(e); - } finally { - await this.#open(filePath); } return new Uint8Array(); } @@ -276,7 +314,7 @@ export class SqliteRelay extends EventEmitter implements Rel const conditions: Array = []; const params: Array = []; - let resultType = "json"; + let resultType = "json,relays"; if (count) { resultType = "count(json)"; } else if (req.ids_only === true) { diff --git a/packages/worker-relay/src/types.ts b/packages/worker-relay/src/types.ts index ae03980d..b5a0a64e 100644 --- a/packages/worker-relay/src/types.ts +++ b/packages/worker-relay/src/types.ts @@ -13,7 +13,8 @@ export type WorkerMessageCommand = | "forYouFeed" | "setEventMetadata" | "debug" - | "delete"; + | "delete" + | "wipe"; export interface WorkerMessage { id: string; @@ -73,6 +74,7 @@ export interface RelayHandler extends EventEmitter { dump(): Promise; delete(req: ReqFilter): Array; setEventMetadata(id: string, meta: EventMetadata): void; + wipe(): Promise; } export interface RelayHandlerEvents { diff --git a/packages/worker-relay/src/worker.ts b/packages/worker-relay/src/worker.ts index b73798ee..7caaa5f0 100644 --- a/packages/worker-relay/src/worker.ts +++ b/packages/worker-relay/src/worker.ts @@ -154,6 +154,11 @@ const handleMsg = async (port: MessagePort | DedicatedWorkerGlobalScope, ev: Mes reply(msg.id, res); break; } + case "wipe": { + await relay!.wipe(); + reply(msg.id, true); + break; + } case "forYouFeed": { const res = await getForYouFeed(relay!, msg.args as string); reply(msg.id, res);