diff --git a/packages/app/src/Cache/ProfileWorkerCache.ts b/packages/app/src/Cache/ProfileWorkerCache.ts index 88d86a6d..43ddf69a 100644 --- a/packages/app/src/Cache/ProfileWorkerCache.ts +++ b/packages/app/src/Cache/ProfileWorkerCache.ts @@ -77,7 +77,6 @@ export class ProfileCacheRelayWorker extends EventEmitter implement "change", mapped.map(a => this.key(a)), ); - console.debug("ProfileCacheRelayWorker", keys, results); return mapped; } diff --git a/packages/app/src/Hooks/useSubscribe.ts b/packages/app/src/Hooks/useSubscribe.ts deleted file mode 100644 index 66d6d4d4..00000000 --- a/packages/app/src/Hooks/useSubscribe.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { ReqFilter, RequestBuilder, TaggedNostrEvent } from "@snort/system"; -import inMemoryDB from "@snort/system/src/InMemoryDB"; -import { useRequestBuilder } from "@snort/system-react"; -import { useEffect, useMemo, useState } from "react"; - -import { System } from "@/system"; - -export default function useSubscribe(id: string, filter: ReqFilter): TaggedNostrEvent[] { - const getEvents = () => inMemoryDB.findArray(filter); - const [events, setEvents] = useState(getEvents()); - const rb = useMemo(() => { - const rb = new RequestBuilder(id); - rb.withBareFilter(filter); - return rb; - }, [id, filter]); - useRequestBuilder(rb); - - useEffect(() => { - const cb = (subId: string) => { - if (subId === id) { - setEvents(getEvents()); - } - }; - System.on("event", cb); - return () => { - System.off("event", cb); - }; - }, [id, filter]); - - return events as Array; -} diff --git a/packages/app/src/system.ts b/packages/app/src/system.ts index ae54aeec..9a033e0a 100644 --- a/packages/app/src/system.ts +++ b/packages/app/src/system.ts @@ -1,6 +1,5 @@ import { removeUndefined, throwIfOffline } from "@snort/shared"; import { mapEventToProfile, NostrEvent, NostrSystem, socialGraphInstance } from "@snort/system"; -import inMemoryDB from "@snort/system/src/InMemoryDB"; import { EventsCache, Relay, RelayMetrics, SystemDb, UserCache, UserRelays } from "@/Cache"; import { addEventToFuzzySearch } from "@/Db/FuzzySearch"; @@ -32,7 +31,6 @@ System.on("event", (_, ev) => { Relay.event(ev); EventsCache.discover(ev); UserCache.discover(ev); - inMemoryDB.handleEvent(ev); socialGraphInstance.handleEvent(ev); addEventToFuzzySearch(ev); }); diff --git a/packages/system/src/InMemoryDB.ts b/packages/system/src/InMemoryDB.ts deleted file mode 100644 index 74d4352b..00000000 --- a/packages/system/src/InMemoryDB.ts +++ /dev/null @@ -1,214 +0,0 @@ -import { ID, ReqFilter as Filter, STR, TaggedNostrEvent, UID } from "."; -import loki from "lokijs"; -import debug from "debug"; - -type PackedNostrEvent = { - id: UID; - pubkey: number; - kind: number; - tags: Array[]; - flatTags: string[]; - sig: string; - created_at: number; - content?: string; - relays: string[]; - saved_at: number; -}; - -const DEFAULT_MAX_SIZE = 5000; - -class InMemoryDB { - private loki = new loki("EventDB"); - private eventsCollection: Collection; - private maxSize: number; - - constructor(maxSize = DEFAULT_MAX_SIZE) { - this.maxSize = maxSize; - this.eventsCollection = this.loki.addCollection("events", { - unique: ["id"], - indices: ["pubkey", "kind", "flatTags", "created_at", "saved_at"], - }); - this.startRemoveOldestInterval(); - } - - private startRemoveOldestInterval() { - const removeOldest = () => { - this.removeOldest(); - setTimeout(() => removeOldest(), 3000); - }; - setTimeout(() => removeOldest(), 3000); - } - - #log = debug("InMemoryDB"); - - get(id: string): TaggedNostrEvent | undefined { - const event = this.eventsCollection.by("id", ID(id)); // throw if db not ready yet? - if (event) { - return this.unpack(event); - } - } - - has(id: string): boolean { - return !!this.eventsCollection.by("id", ID(id)); - } - - // map to internal UIDs to save memory - private pack(event: TaggedNostrEvent): PackedNostrEvent { - return { - id: ID(event.id), - pubkey: ID(event.pubkey), - sig: event.sig, - kind: event.kind, - tags: event.tags.map(tag => { - if (["e", "p"].includes(tag[0]) && typeof tag[1] === "string") { - return [tag[0], ID(tag[1] as string), ...tag.slice(2)]; - } else { - return tag; - } - }), - flatTags: event.tags.filter(tag => ["e", "p", "d"].includes(tag[0])).map(tag => `${tag[0]}_${ID(tag[1])}`), - created_at: event.created_at, - content: event.content, - relays: event.relays, - saved_at: Date.now(), - }; - } - - private unpack(packedEvent: PackedNostrEvent): TaggedNostrEvent { - return { - id: STR(packedEvent.id), - pubkey: STR(packedEvent.pubkey), - sig: packedEvent.sig, - kind: packedEvent.kind, - tags: packedEvent.tags.map(tag => { - if (["e", "p"].includes(tag[0] as string) && typeof tag[1] === "number") { - return [tag[0], STR(tag[1] as number), ...tag.slice(2)]; - } else { - return tag; - } - }), - created_at: packedEvent.created_at, - content: packedEvent.content, - relays: packedEvent.relays, - }; - } - - handleEvent(event: TaggedNostrEvent): boolean { - if (!event || !event.id || !event.created_at) { - throw new Error("Invalid event"); - } - - const id = ID(event.id); - if (this.eventsCollection.by("id", id)) { - return false; // this prevents updating event.relays? - } - - const packed = this.pack(event); - - // we might want to limit the kinds of events we save, e.g. no kind 0, 3 or only 1, 6 - - try { - this.eventsCollection.insert(packed); - } catch (e) { - return false; - } - - return true; - } - - remove(eventId: string): void { - const id = ID(eventId); - this.eventsCollection.findAndRemove({ id }); - } - - removeOldest(): void { - const count = this.eventsCollection.count(); - this.#log("InMemoryDB: count", count, this.maxSize); - if (count > this.maxSize) { - this.#log("InMemoryDB: removing oldest events", count - this.maxSize); - this.eventsCollection - .chain() - .simplesort("saved_at") - .limit(count - this.maxSize) - .remove(); - } - } - - count(filter: Filter): number { - return this.findArray(filter).length; - } - - find(filter: Filter, callback: (event: TaggedNostrEvent) => void): void { - this.findArray(filter).forEach(event => { - callback(event); - }); - } - - findArray(filter: Filter): TaggedNostrEvent[] { - const query = this.constructQuery(filter); - - const searchRegex = filter.search ? new RegExp(filter.search, "i") : undefined; - let chain = this.eventsCollection - .chain() - .find(query) - .where((e: PackedNostrEvent) => { - if (searchRegex && !e.content?.match(searchRegex)) { - return false; - } - return true; - }) - .simplesort("created_at", true); - - if (filter.limit) { - chain = chain.limit(filter.limit); - } - - return chain.data().map(e => this.unpack(e)); - } - - findAndRemove(filter: Filter) { - const query = this.constructQuery(filter); - this.eventsCollection.findAndRemove(query); - } - - private constructQuery(filter: Filter): LokiQuery { - const query: LokiQuery = {}; - - if (filter.ids) { - query.id = { $in: filter.ids.map(ID) }; - } else { - if (filter.authors) { - query.pubkey = { $in: filter.authors.map(ID) }; - } - if (filter.kinds) { - query.kind = { $in: filter.kinds }; - } - if (filter["#e"]) { - query.flatTags = { $contains: "e_" + filter["#e"]!.map(ID) }; - } else if (filter["#p"]) { - query.flatTags = { $contains: "p_" + filter["#p"]!.map(ID) }; - } else if (filter["#d"]) { - query.flatTags = { $contains: "d_" + filter["#d"]!.map(ID) }; - } - if (filter.since && filter.until) { - query.created_at = { $between: [filter.since, filter.until] }; - } - if (filter.since) { - query.created_at = { $gte: filter.since }; - } - if (filter.until) { - query.created_at = { $lte: filter.until }; - } - } - - return query; - } - - findOne(filter: Filter): TaggedNostrEvent | undefined { - return this.findArray(filter)[0]; - } -} - -export { InMemoryDB }; - -export default new InMemoryDB(); diff --git a/packages/worker-relay/src/memory-relay.ts b/packages/worker-relay/src/memory-relay.ts index 98dc7b27..9272b654 100644 --- a/packages/worker-relay/src/memory-relay.ts +++ b/packages/worker-relay/src/memory-relay.ts @@ -1,12 +1,13 @@ import EventEmitter from "eventemitter3"; import { NostrEvent, RelayHandler, RelayHandlerEvents, ReqFilter, eventMatchesFilter } from "./types"; +import debug from "debug"; /** * A very simple dumb fallback relay using a flat table */ export class InMemoryRelay extends EventEmitter implements RelayHandler { #events: Map = new Map(); - #log = (...args: any[]) => console.debug(...args); + #log = debug("InMemoryRelay"); init(path: string): Promise { this.#log("Using in-memory relay"); diff --git a/packages/worker-relay/src/sqlite-relay.ts b/packages/worker-relay/src/sqlite-relay.ts index 5d21e179..0e0da2a7 100644 --- a/packages/worker-relay/src/sqlite-relay.ts +++ b/packages/worker-relay/src/sqlite-relay.ts @@ -1,10 +1,11 @@ import sqlite3InitModule, { Database, Sqlite3Static } from "@sqlite.org/sqlite-wasm"; import { EventEmitter } from "eventemitter3"; import { NostrEvent, RelayHandler, RelayHandlerEvents, ReqFilter, unixNowMs } from "./types"; +import debug from "debug"; export class SqliteRelay extends EventEmitter implements RelayHandler { #sqlite?: Sqlite3Static; - #log = (...args: any[]) => console.debug(...args); + #log = debug("SqliteRelay"); #db?: Database; #seenInserts = new Set(); diff --git a/packages/worker-relay/src/worker.ts b/packages/worker-relay/src/worker.ts index 669f8c8a..a1a18d1a 100644 --- a/packages/worker-relay/src/worker.ts +++ b/packages/worker-relay/src/worker.ts @@ -35,7 +35,7 @@ async function insertBatch() { if (relay) { while (eventWriteQueue.length > 0) { if (unixNowMs() - start >= timeLimit) { - console.debug("Yield insert, queue length: ", eventWriteQueue.length, ", cmds: ", cmdQueue.length); + //console.debug("Yield insert, queue length: ", eventWriteQueue.length, ", cmds: ", cmdQueue.length); break; } const batch = eventWriteQueue.splice(0, 10);