From 35ec58377c51d5fa5bdeac6c07100e5bae91740c Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Sat, 13 Jan 2024 22:44:16 +0200 Subject: [PATCH] for you feed etc --- packages/app/config/default.json | 3 +- packages/app/config/iris.json | 8 +- packages/app/custom.d.ts | 1 + .../app/src/Components/Feed/RootTabItems.tsx | 12 + packages/app/src/Components/Feed/RootTabs.tsx | 2 +- packages/app/src/Db/IndexedDB.ts | 245 ------------------ packages/app/src/Db/getForYouFeed.ts | 107 ++++++++ packages/app/src/Pages/Root/DefaultTab.tsx | 2 +- packages/app/src/Pages/Root/ForYouTab.tsx | 98 +++++++ packages/app/src/Pages/Root/GlobalTab.tsx | 16 +- packages/app/src/Pages/Root/RootTabRoutes.tsx | 5 + packages/app/src/Pages/SearchPage.tsx | 14 +- packages/app/src/Pages/TopicsPage.tsx | 2 +- .../app/src/Pages/settings/Preferences.tsx | 5 + packages/app/src/Utils/Login/Preferences.ts | 4 +- packages/app/src/lang.json | 3 + packages/app/src/translations/en.json | 1 + packages/system/src/InMemoryDB.ts | 210 +++++++++++++++ 18 files changed, 469 insertions(+), 269 deletions(-) delete mode 100644 packages/app/src/Db/IndexedDB.ts create mode 100644 packages/app/src/Db/getForYouFeed.ts create mode 100644 packages/app/src/Pages/Root/ForYouTab.tsx create mode 100644 packages/system/src/InMemoryDB.ts diff --git a/packages/app/config/default.json b/packages/app/config/default.json index fb5e09e4..ce563437 100644 --- a/packages/app/config/default.json +++ b/packages/app/config/default.json @@ -18,7 +18,8 @@ "notificationGraph": true, "communityLeaders": true, "nostrAddress": true, - "pushNotifications": true + "pushNotifications": true, + "forYouFeed": false }, "signUp": { "moderation": true, diff --git a/packages/app/config/iris.json b/packages/app/config/iris.json index a0a388ef..cdeaba49 100644 --- a/packages/app/config/iris.json +++ b/packages/app/config/iris.json @@ -16,7 +16,8 @@ "deck": true, "zapPool": true, "notificationGraph": false, - "communityLeaders": true + "communityLeaders": true, + "forYouFeed": true }, "defaultPreferences": { "hideMutedNotes": true @@ -45,7 +46,10 @@ "wss://relay.nostr.band/": { "read": true, "write": true }, "wss://relay.damus.io/": { "read": true, "write": true } }, - "chatChannels": [{ "type": "telegram", "value": "https://t.me/irismessenger" }], + "chatChannels": [ + { "type": "telegram", "value": "https://t.me/irismessenger" }, + { "type": "nip28", "value": "23286a4602ada10cc10200553bff62a110e8dc0eacddf73277395a89ddf26a09" } + ], "alby": { "clientId": "5rYcHDrlDb", "clientSecret": "QAI3QmgiaPH3BfTMzzFd" diff --git a/packages/app/custom.d.ts b/packages/app/custom.d.ts index 85f08691..57790c24 100644 --- a/packages/app/custom.d.ts +++ b/packages/app/custom.d.ts @@ -61,6 +61,7 @@ declare const CONFIG: { communityLeaders: boolean; nostrAddress: boolean; pushNotifications: boolean; + forYouFeed: boolean; }; defaultPreferences: { hideMutedNotes: boolean; diff --git a/packages/app/src/Components/Feed/RootTabItems.tsx b/packages/app/src/Components/Feed/RootTabItems.tsx index 4cbf9ae2..bab50a66 100644 --- a/packages/app/src/Components/Feed/RootTabItems.tsx +++ b/packages/app/src/Components/Feed/RootTabItems.tsx @@ -5,6 +5,7 @@ import Icon from "@/Components/Icons/Icon"; import { Newest } from "@/Utils/Login"; export type RootTab = + | "for-you" | "following" | "followed-by-friends" | "conversations" @@ -16,6 +17,17 @@ export type RootTab = export function rootTabItems(base: string, pubKey: string | undefined, tags: Newest>) { const menuItems = [ + { + tab: "for-you", + path: `${base}/for-you`, + show: Boolean(pubKey) && CONFIG.features.forYouFeed, + element: ( + <> + + + + ), + }, { tab: "following", path: `${base}/notes`, diff --git a/packages/app/src/Components/Feed/RootTabs.tsx b/packages/app/src/Components/Feed/RootTabs.tsx index 62850495..687ffaba 100644 --- a/packages/app/src/Components/Feed/RootTabs.tsx +++ b/packages/app/src/Components/Feed/RootTabs.tsx @@ -25,7 +25,7 @@ export function RootTabs({ base = "/" }: { base: string }) { const defaultTab = pubKey ? preferences.defaultRootTab ?? `${base}/notes` : `${base}/trending/notes`; const initialPathname = location.pathname === "/" ? defaultTab : location.pathname; - const initialRootType = menuItems.find(a => a.path === initialPathname)?.tab || "following"; + const initialRootType = menuItems.find(a => a.path === initialPathname)?.tab || "for-you"; const [rootType, setRootType] = useState(initialRootType); diff --git a/packages/app/src/Db/IndexedDB.ts b/packages/app/src/Db/IndexedDB.ts deleted file mode 100644 index 0a4735e6..00000000 --- a/packages/app/src/Db/IndexedDB.ts +++ /dev/null @@ -1,245 +0,0 @@ -import LRUSet from "@snort/shared/src/LRUSet"; -import { ReqFilter as Filter, TaggedNostrEvent } from "@snort/system"; -import * as Comlink from "comlink"; -import Dexie, { Table } from "dexie"; - -type Tag = { - id: string; - eventId: string; - type: string; - value: string; -}; - -type SaveQueueEntry = { event: TaggedNostrEvent; tags: Tag[] }; -type Task = () => Promise; - -class IndexedDB extends Dexie { - events!: Table; - tags!: Table; - private saveQueue: SaveQueueEntry[] = []; - private subscribedEventIds = new Set(); - private subscribedAuthors = new Set(); - private subscribedTags = new Set(); - private subscribedAuthorsAndKinds = new Set(); - private readQueue: Map = new Map(); - private isProcessingQueue = false; - private seenEvents = new LRUSet(2000); - - constructor() { - super("EventDB"); - - this.version(6).stores({ - // TODO use multientry index for *tags - events: "++id, pubkey, kind, created_at, [pubkey+kind]", - tags: "&[type+value+eventId], [type+value], eventId", - }); - - this.startInterval(); - } - - private startInterval() { - const processQueue = async () => { - if (this.saveQueue.length > 0) { - try { - const eventsToSave: TaggedNostrEvent[] = []; - const tagsToSave: Tag[] = []; - for (const item of this.saveQueue) { - eventsToSave.push(item.event); - tagsToSave.push(...item.tags); - } - await this.events.bulkPut(eventsToSave); - await this.tags.bulkPut(tagsToSave); - } catch (e) { - console.error(e); - } finally { - this.saveQueue = []; - } - } - setTimeout(() => processQueue(), 3000); - }; - - setTimeout(() => processQueue(), 3000); - } - - handleEvent(event: TaggedNostrEvent) { - if (this.seenEvents.has(event.id)) { - return; - } - this.seenEvents.add(event.id); - - // maybe we don't want event.kind 3 tags - const tags = - event.kind === 3 - ? [] - : event.tags - ?.filter(tag => { - if (tag[0] === "d") { - return true; - } - if (tag[0] === "e") { - return true; - } - // we're only interested in p tags where we are mentioned - /* - if (tag[0] === "p") { - Key.isMine(tag[1])) { // TODO - return true; - }*/ - return false; - }) - .map(tag => ({ - eventId: event.id, - type: tag[0], - value: tag[1], - })) || []; - - this.saveQueue.push({ event, tags }); - } - - private async startReadQueue() { - if (!this.isProcessingQueue && this.readQueue.size > 0) { - this.isProcessingQueue = true; - for (const [key, task] of this.readQueue.entries()) { - this.readQueue.delete(key); // allow to re-queue right away - console.log("starting idb task", key); - console.time(key); - await task(); - console.timeEnd(key); - } - this.isProcessingQueue = false; - } - } - - private enqueueRead(key: string, task: () => Promise) { - this.readQueue.set(key, task); - this.startReadQueue(); - } - - getByAuthors = async (callback: (event: TaggedNostrEvent) => void, limit?: number) => { - this.enqueueRead("getByAuthors", async () => { - const authors = [...this.subscribedAuthors]; - this.subscribedAuthors.clear(); - - await this.events - .where("pubkey") - .anyOf(authors) - .limit(limit || 1000) - .each(callback); - }); - }; - - getByEventIds = async (callback: (event: TaggedNostrEvent) => void) => { - this.enqueueRead("getByEventIds", async () => { - const ids = [...this.subscribedEventIds]; - this.subscribedEventIds.clear(); - await this.events.where("id").anyOf(ids).each(callback); - }); - }; - - getByTags = async (callback: (event: TaggedNostrEvent) => void) => { - this.enqueueRead("getByTags", async () => { - const tagPairs = [...this.subscribedTags].map(tag => tag.split("|")); - this.subscribedTags.clear(); - - await this.tags - .where("[type+value]") - .anyOf(tagPairs) - .each(tag => this.subscribedEventIds.add(tag.eventId)); - - await this.getByEventIds(callback); - }); - }; - - getByAuthorsAndKinds = async (callback: (event: TaggedNostrEvent) => void) => { - this.enqueueRead("authorsAndKinds", async () => { - const authorsAndKinds = [...this.subscribedAuthorsAndKinds]; - this.subscribedAuthorsAndKinds.clear(); - const pairs = authorsAndKinds.map(pair => { - const [author, kind] = pair.split("|"); - return [author, parseInt(kind)]; - }); - await this.events.where("[pubkey+kind]").anyOf(pairs).each(callback); - }); - }; - - async find(filter: Filter, callback: (event: TaggedNostrEvent) => void): Promise { - if (!filter) return; - - const filterString = JSON.stringify(filter); - if (this.readQueue.has(filterString)) { - return; - } - - // make sure only 1 argument is passed - const cb = e => { - this.seenEvents.add(e.id); - if (filter.not?.ids?.includes(e.id)) { - console.log("skipping", e.id); - return; - } - callback(e); - }; - - let hasTags = false; - for (const key in filter) { - if (key.startsWith("#")) { - hasTags = true; - const tagName = key.slice(1); // Remove the hash to get the tag name - const values = filter[key]; - if (Array.isArray(values)) { - for (const value of values) { - this.subscribedTags.add(tagName + "|" + value); - } - } - } - } - - if (hasTags) { - await this.getByTags(cb); - return; - } - - if (filter.ids?.length) { - filter.ids.forEach(id => this.subscribedEventIds.add(id)); - await this.getByEventIds(cb); - return; - } - - if (filter.authors?.length && filter.kinds?.length) { - const permutations = filter.authors.flatMap(author => filter.kinds!.map(kind => author + "|" + kind)); - permutations.forEach(permutation => this.subscribedAuthorsAndKinds.add(permutation)); - await this.getByAuthorsAndKinds(cb); - return; - } - - if (filter.authors?.length) { - filter.authors.forEach(author => this.subscribedAuthors.add(author)); - await this.getByAuthors(cb); - return; - } - - let query = this.events; - if (filter.kinds) { - query = query.where("kind").anyOf(filter.kinds); - } - if (filter.search) { - const term = filter.search.replace(" sort:popular", ""); - if (term === "") { - return; - } - const regexp = new RegExp(term, "i"); - query = query.filter((event: Event) => event.content?.match(regexp)); - } - if (filter.limit) { - query = query.limit(filter.limit); - } - // TODO test that the sort is actually working - this.enqueueRead(filterString, async () => { - await query.each(cb); - }); - } -} - -const db = new IndexedDB(); - -Comlink.expose(db); diff --git a/packages/app/src/Db/getForYouFeed.ts b/packages/app/src/Db/getForYouFeed.ts new file mode 100644 index 00000000..d09f5d86 --- /dev/null +++ b/packages/app/src/Db/getForYouFeed.ts @@ -0,0 +1,107 @@ +import { NostrEvent, parseZap } from "@snort/system"; + +import { Relay } from "@/Cache"; + +export async function getForYouFeed(pubkey: string): Promise { + console.time("For You feed generation time"); + + console.log("pubkey", pubkey); + + // Get events reacted to by me + const myReactedEvents = await getMyReactedEventIds(pubkey); + console.log("my reacted events", myReactedEvents); + + // Get others who reacted to the same events as me + const othersWhoReacted = await getOthersWhoReacted(myReactedEvents, pubkey); + // this tends to be small when the user has just logged in, we should maybe subscribe for more from relays + console.log("others who reacted", othersWhoReacted); + + // Get event ids reacted to by those others + const reactedByOthers = await getEventIdsReactedByOthers(othersWhoReacted); + console.log("reacted by others", reactedByOthers); + + // Get events reacted to by others that I haven't reacted to + const idsToFetch = Array.from(reactedByOthers).filter(id => !myReactedEvents.has(id)); + console.log("ids to fetch", idsToFetch); + + // Get full events in sorted order + const feed = await getFeedEvents(idsToFetch); + console.log("feed.length", feed.length); + + console.timeEnd("For You feed generation time"); + return feed; +} + +async function getMyReactedEventIds(pubkey: string) { + const myReactedEventIds = new Set(); + const myEvents = await Relay.query([ + "REQ", + "getMyReactedEventIds", + { + authors: [pubkey], + kinds: [1, 6, 7, 9735], + }, + ]); + myEvents.forEach(ev => { + const targetEventId = ev.kind === 9735 ? parseZap(ev).event?.id : ev.tags.find(tag => tag[0] === "e")?.[1]; + if (targetEventId) { + myReactedEventIds.add(targetEventId); + } + }); + + return myReactedEventIds; +} + +async function getOthersWhoReacted(myReactedEventIds: Set, myPubkey: string) { + const othersWhoReacted = new Set(); + + const otherReactions = await Relay.query([ + "REQ", + "getOthersWhoReacted", + { + "#e": Array.from(myReactedEventIds), + }, + ]); + + otherReactions.forEach(reaction => { + if (reaction.pubkey !== myPubkey) { + othersWhoReacted.add(reaction.pubkey); + } + }); + + return [...othersWhoReacted]; +} + +async function getEventIdsReactedByOthers(othersWhoReacted: string[]) { + const eventIdsReactedByOthers = new Set(); + + const events = await Relay.query([ + "REQ", + "getEventIdsReactedByOthers", + { + authors: othersWhoReacted, + }, + ]); + + events.forEach(event => { + event.tags.forEach(tag => { + if (tag[0] === "e") eventIdsReactedByOthers.add(tag[1]); + }); + }); + + return [...eventIdsReactedByOthers]; +} + +async function getFeedEvents(ids: string[]) { + return (await Relay.query([ + "REQ", + "getFeedEvents", + { + ids, + kinds: [1], + }, + ])).filter((ev) => { + // no replies + return !ev.tags.some((tag) => tag[0] === "e"); + }).sort((a, b) => b.created_at - a.created_at); +} diff --git a/packages/app/src/Pages/Root/DefaultTab.tsx b/packages/app/src/Pages/Root/DefaultTab.tsx index cb8cfb23..2cda6b0c 100644 --- a/packages/app/src/Pages/Root/DefaultTab.tsx +++ b/packages/app/src/Pages/Root/DefaultTab.tsx @@ -6,7 +6,7 @@ export const DefaultTab = () => { preferences: s.appData.item.preferences, publicKey: s.publicKey, })); - const tab = publicKey ? preferences.defaultRootTab ?? `notes` : `trending/notes`; + const tab = publicKey ? preferences.defaultRootTab ?? `for-you` : `trending/notes`; const elm = RootTabRoutes.find(a => a.path === tab)?.element; return elm; }; diff --git a/packages/app/src/Pages/Root/ForYouTab.tsx b/packages/app/src/Pages/Root/ForYouTab.tsx new file mode 100644 index 00000000..ba11f090 --- /dev/null +++ b/packages/app/src/Pages/Root/ForYouTab.tsx @@ -0,0 +1,98 @@ +import { TaggedNostrEvent } from "@snort/system"; +import { memo, useEffect, useMemo, useState } from "react"; +import { FormattedMessage } from "react-intl"; +import { Link } from "react-router-dom"; + +import { DisplayAs, DisplayAsSelector } from "@/Components/Feed/DisplayAsSelector"; +import { TimelineRenderer } from "@/Components/Feed/TimelineRenderer"; +import { TaskList } from "@/Components/Tasks/TaskList"; +import { getForYouFeed } from "@/Db/getForYouFeed"; +import useLogin from "@/Hooks/useLogin"; +import messages from "@/Pages/messages"; +import { System } from "@/system"; + +const FollowsHint = () => { + const { publicKey: pubKey, follows } = useLogin(); + if (follows.item?.length === 0 && pubKey) { + return ( + + + + ), + }} + /> + ); + } + return null; +}; + +let forYouFeed = { + events: [] as TaggedNostrEvent[], + created_at: 0, +}; + +let getForYouFeedPromise: Promise | null = null; + +export const ForYouTab = memo(function ForYouTab() { + const [notes, setNotes] = useState(forYouFeed.events); + const { feedDisplayAs } = useLogin(); + const displayAsInitial = feedDisplayAs ?? "list"; + const [displayAs, setDisplayAs] = useState(displayAsInitial); + const { publicKey } = useLogin(); + + const getFeed = () => { + if (!publicKey) { + return []; + } + if (!getForYouFeedPromise) { + getForYouFeedPromise = getForYouFeed(publicKey); + } + getForYouFeedPromise!.then(notes => { + console.log("for you feed", notes); + if (notes.length < 10) { + setTimeout(() => { + getForYouFeedPromise = null; + getForYouFeed(); + }, 1000); + } + forYouFeed = { + events: notes, + created_at: Date.now(), + }; + setNotes(notes); + notes.forEach(note => { + queueMicrotask(() => { + System.HandleEvent(note); + }); + }); + }); + }; + + useEffect(() => { + if (forYouFeed.events.length < 10 || Date.now() - forYouFeed.created_at > 1000 * 60 * 1) { + getFeed(); + } + }, []); + + const frags = useMemo(() => { + return [ + { + events: notes, + refTime: Date.now(), + }, + ]; + }, [notes]); + + return ( + <> + setDisplayAs(a)} /> + + + + + ); +}); diff --git a/packages/app/src/Pages/Root/GlobalTab.tsx b/packages/app/src/Pages/Root/GlobalTab.tsx index f73be6f4..8c7a8e02 100644 --- a/packages/app/src/Pages/Root/GlobalTab.tsx +++ b/packages/app/src/Pages/Root/GlobalTab.tsx @@ -4,7 +4,6 @@ import { useContext, useEffect, useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; import Timeline from "@/Components/Feed/Timeline"; -import { TimelineSubject } from "@/Feed/TimelineFeed"; import useHistoryState from "@/Hooks/useHistoryState"; import useLogin from "@/Hooks/useLogin"; import { debounce, getRelayName, sha256 } from "@/Utils"; @@ -81,14 +80,13 @@ export const GlobalTab = () => { }, [relays, relay]); const subject = useMemo( - () => - ({ - type: "global", - items: [], - relay: [relay?.url], - discriminator: `all-${sha256(relay?.url ?? "")}`, - }) as TimelineSubject, - [relay?.url], + () => ({ + type: "global", + items: [], + relay: [relay.url], + discriminator: `all-${sha256(relay.url)}`, + }), + [relay.url], ); return ( diff --git a/packages/app/src/Pages/Root/RootTabRoutes.tsx b/packages/app/src/Pages/Root/RootTabRoutes.tsx index 7bb183c8..b8c20759 100644 --- a/packages/app/src/Pages/Root/RootTabRoutes.tsx +++ b/packages/app/src/Pages/Root/RootTabRoutes.tsx @@ -6,6 +6,7 @@ import HashTagsPage from "@/Pages/HashTagsPage"; import { ConversationsTab } from "@/Pages/Root/ConversationsTab"; import { DefaultTab } from "@/Pages/Root/DefaultTab"; import { FollowedByFriendsTab } from "@/Pages/Root/FollowedByFriendsTab"; +import { ForYouTab } from "@/Pages/Root/ForYouTab"; import { GlobalTab } from "@/Pages/Root/GlobalTab"; import { NotesTab } from "@/Pages/Root/NotesTab"; import { TagsTab } from "@/Pages/Root/TagsTab"; @@ -16,6 +17,10 @@ export const RootTabRoutes = [ path: "", element: , }, + { + path: "for-you", + element: , + }, { path: "global", element: , diff --git a/packages/app/src/Pages/SearchPage.tsx b/packages/app/src/Pages/SearchPage.tsx index 089e80b9..30d609b9 100644 --- a/packages/app/src/Pages/SearchPage.tsx +++ b/packages/app/src/Pages/SearchPage.tsx @@ -7,7 +7,6 @@ import TabSelectors, { Tab } from "@/Components/TabSelectors/TabSelectors"; import TrendingNotes from "@/Components/Trending/TrendingPosts"; import TrendingUsers from "@/Components/Trending/TrendingUsers"; import FollowListBase from "@/Components/User/FollowListBase"; -import { TimelineSubject } from "@/Feed/TimelineFeed"; import useProfileSearch from "@/Hooks/useProfileSearch"; import { debounce } from "@/Utils"; @@ -54,13 +53,14 @@ const SearchPage = () => { return debounce(500, () => setKeyword(search)); }, [search]); - const searchTimeline = useMemo(() => { - return { + const subject = useMemo( + () => ({ type: "post_keyword", + items: [keyword + (sortPopular ? " sort:popular" : "")], discriminator: keyword, - items: [keyword], - } as TimelineSubject; - }, [keyword]); + }), + [keyword, sortPopular], + ); function tabContent() { if (tab.value === PROFILES) { @@ -74,7 +74,7 @@ const SearchPage = () => { return ( <> {sortOptions()} - + ); } diff --git a/packages/app/src/Pages/TopicsPage.tsx b/packages/app/src/Pages/TopicsPage.tsx index 1ef2e8d0..5ed56d04 100644 --- a/packages/app/src/Pages/TopicsPage.tsx +++ b/packages/app/src/Pages/TopicsPage.tsx @@ -14,5 +14,5 @@ export function TopicsPage() { [tags, pubKey], ); - return ; + return ; } diff --git a/packages/app/src/Pages/settings/Preferences.tsx b/packages/app/src/Pages/settings/Preferences.tsx index 9f13d6a8..81856c56 100644 --- a/packages/app/src/Pages/settings/Preferences.tsx +++ b/packages/app/src/Pages/settings/Preferences.tsx @@ -84,6 +84,11 @@ const PreferencesPage = () => { defaultRootTab: e.target.value, } as UserPreferences) }> + {CONFIG.features.forYouFeed && ( + + )} diff --git a/packages/app/src/Utils/Login/Preferences.ts b/packages/app/src/Utils/Login/Preferences.ts index 3cf3f3a8..dc8a24da 100644 --- a/packages/app/src/Utils/Login/Preferences.ts +++ b/packages/app/src/Utils/Login/Preferences.ts @@ -55,7 +55,7 @@ export interface UserPreferences { /** * Default page to select on load */ - defaultRootTab: "notes" | "conversations" | "global"; + defaultRootTab: "for-you" | "notes" | "conversations" | "global"; /** * Default zap amount @@ -113,7 +113,7 @@ export const DefaultPreferences = { autoShowLatest: false, fileUploader: "void.cat", imgProxyConfig: DefaultImgProxy, - defaultRootTab: "notes", + defaultRootTab: CONFIG.features.forYouFeed ? "for-you" : "notes", defaultZapAmount: 50, autoZap: false, telemetry: true, diff --git a/packages/app/src/lang.json b/packages/app/src/lang.json index 7320eb9d..36b1d9a6 100644 --- a/packages/app/src/lang.json +++ b/packages/app/src/lang.json @@ -1706,6 +1706,9 @@ "x82IOl": { "defaultMessage": "Mute" }, + "xEjBS7": { + "defaultMessage": "For you" + }, "xIcAOU": { "defaultMessage": "Votes by {type}" }, diff --git a/packages/app/src/translations/en.json b/packages/app/src/translations/en.json index 378fb520..c8302b0f 100644 --- a/packages/app/src/translations/en.json +++ b/packages/app/src/translations/en.json @@ -563,6 +563,7 @@ "wtLjP6": "Copy ID", "x/Fx2P": "Fund the services that you use by splitting a portion of all your zaps into a pool of funds!", "x82IOl": "Mute", + "xEjBS7": "For you", "xIcAOU": "Votes by {type}", "xIoGG9": "Go to", "xQtL3v": "Unlock", diff --git a/packages/system/src/InMemoryDB.ts b/packages/system/src/InMemoryDB.ts new file mode 100644 index 00000000..6c8a3902 --- /dev/null +++ b/packages/system/src/InMemoryDB.ts @@ -0,0 +1,210 @@ +import { ID, ReqFilter as Filter, STR, TaggedNostrEvent, UID } from "."; +import loki from "lokijs"; +import debug from "debug"; + +type PackedNostrEvent = { + id: UID; + pubkey: number; + kind: number; + tags: Array[]; + flatTags: string[]; + sig: string; + created_at: number; + content?: string; + relays: string[]; + saved_at: number; +}; + +const DEFAULT_MAX_SIZE = 5000; + +class InMemoryDB { + private loki = new loki("EventDB"); + private eventsCollection: Collection; + private maxSize: number; + + constructor(maxSize = DEFAULT_MAX_SIZE) { + this.maxSize = maxSize; + this.eventsCollection = this.loki.addCollection("events", { + unique: ["id"], + indices: ["pubkey", "kind", "flatTags", "created_at", "saved_at"], + }); + this.startRemoveOldestInterval(); + } + + private startRemoveOldestInterval() { + const removeOldest = () => { + this.removeOldest(); + setTimeout(() => removeOldest(), 3000); + }; + setTimeout(() => removeOldest(), 3000); + } + + #log = debug("InMemoryDB"); + + get(id: string): TaggedNostrEvent | undefined { + const event = this.eventsCollection.by("id", ID(id)); // throw if db not ready yet? + if (event) { + return this.unpack(event); + } + } + + has(id: string): boolean { + return !!this.eventsCollection.by("id", ID(id)); + } + + // map to internal UIDs to save memory + private pack(event: TaggedNostrEvent): PackedNostrEvent { + return { + id: ID(event.id), + pubkey: ID(event.pubkey), + sig: event.sig, + kind: event.kind, + tags: event.tags.map(tag => { + if (["e", "p"].includes(tag[0]) && typeof tag[1] === "string") { + return [tag[0], ID(tag[1] as string), ...tag.slice(2)]; + } else { + return tag; + } + }), + flatTags: event.tags.filter(tag => ["e", "p", "d"].includes(tag[0])).map(tag => `${tag[0]}_${ID(tag[1])}`), + created_at: event.created_at, + content: event.content, + relays: event.relays, + saved_at: Date.now(), + }; + } + + private unpack(packedEvent: PackedNostrEvent): TaggedNostrEvent { + return { + id: STR(packedEvent.id), + pubkey: STR(packedEvent.pubkey), + sig: packedEvent.sig, + kind: packedEvent.kind, + tags: packedEvent.tags.map(tag => { + if (["e", "p"].includes(tag[0] as string) && typeof tag[1] === "number") { + return [tag[0], STR(tag[1] as number), ...tag.slice(2)]; + } else { + return tag; + } + }), + created_at: packedEvent.created_at, + content: packedEvent.content, + relays: packedEvent.relays, + }; + } + + handleEvent(event: TaggedNostrEvent): boolean { + if (!event || !event.id || !event.created_at) { + throw new Error("Invalid event"); + } + + const id = ID(event.id); + if (this.eventsCollection.by("id", id)) { + return false; // this prevents updating event.relays? + } + + const packed = this.pack(event); + + // we might want to limit the kinds of events we save, e.g. no kind 0, 3 or only 1, 6 + + try { + this.eventsCollection.insert(packed); + } catch (e) { + return false; + } + + return true; + } + + remove(eventId: string): void { + const id = ID(eventId); + this.eventsCollection.findAndRemove({ id }); + } + + removeOldest(): void { + const count = this.eventsCollection.count(); + this.#log("InMemoryDB: count", count, this.maxSize); + if (count > this.maxSize) { + this.#log("InMemoryDB: removing oldest events", count - this.maxSize); + this.eventsCollection + .chain() + .simplesort("saved_at") + .limit(count - this.maxSize) + .remove(); + } + } + + find(filter: Filter, callback: (event: TaggedNostrEvent) => void): void { + this.findArray(filter).forEach(event => { + callback(event); + }); + } + + findArray(filter: Filter): TaggedNostrEvent[] { + const query = this.constructQuery(filter); + + const searchRegex = filter.search ? new RegExp(filter.search, "i") : undefined; + let chain = this.eventsCollection + .chain() + .find(query) + .where((e: PackedNostrEvent) => { + if (searchRegex && !e.content?.match(searchRegex)) { + return false; + } + return true; + }) + .simplesort("created_at", true); + + if (filter.limit) { + chain = chain.limit(filter.limit); + } + + return chain.data().map(e => this.unpack(e)); + } + + findAndRemove(filter: Filter) { + const query = this.constructQuery(filter); + this.eventsCollection.findAndRemove(query); + } + + private constructQuery(filter: Filter): LokiQuery { + const query: LokiQuery = {}; + + if (filter.ids) { + query.id = { $in: filter.ids.map(ID) }; + } else { + if (filter.authors) { + query.pubkey = { $in: filter.authors.map(ID) }; + } + if (filter.kinds) { + query.kind = { $in: filter.kinds }; + } + if (filter["#e"]) { + query.flatTags = { $contains: "e_" + filter["#e"]!.map(ID) }; + } else if (filter["#p"]) { + query.flatTags = { $contains: "p_" + filter["#p"]!.map(ID) }; + } else if (filter["#d"]) { + query.flatTags = { $contains: "d_" + filter["#d"]!.map(ID) }; + } + if (filter.since && filter.until) { + query.created_at = { $between: [filter.since, filter.until] }; + } + if (filter.since) { + query.created_at = { $gte: filter.since }; + } + if (filter.until) { + query.created_at = { $lte: filter.until }; + } + } + + return query; + } + + findOne(filter: Filter): TaggedNostrEvent | undefined { + return this.findArray(filter)[0]; + } +} + +export { InMemoryDB }; + +export default new InMemoryDB();