From 48407d91987c9f114747fb111f653a113a9e5a3e Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 1 Feb 2023 13:22:11 +0000 Subject: [PATCH 1/3] feed cache --- src/Db/index.ts | 22 ++++++++--- src/Feed/EventPublisher.ts | 20 +++++----- src/Feed/FollowersFeed.ts | 2 +- src/Feed/FollowsFeed.ts | 2 +- src/Feed/LoginFeed.ts | 8 ++-- src/Feed/ProfileFeed.ts | 4 +- src/Feed/Subscription.ts | 79 +++++++++++++++++++++++++++++-------- src/Feed/ThreadFeed.ts | 2 +- src/Feed/TimelineFeed.ts | 11 +++--- src/Pages/HashTagsPage.tsx | 2 +- src/Pages/Notifications.tsx | 2 +- src/Pages/ProfilePage.tsx | 4 +- src/Pages/Root.tsx | 2 +- src/Pages/SearchPage.tsx | 2 +- src/State/Users.ts | 22 +++++------ src/State/Users/Db.ts | 27 +++++++------ src/State/Users/Hooks.ts | 44 ++++++++++----------- 17 files changed, 158 insertions(+), 97 deletions(-) diff --git a/src/Db/index.ts b/src/Db/index.ts index 8834c96d..46ec27cf 100644 --- a/src/Db/index.ts +++ b/src/Db/index.ts @@ -1,23 +1,35 @@ import Dexie, { Table } from "dexie"; +import { TaggedRawEvent, u256 } from "Nostr"; import { MetadataCache } from "State/Users"; import { hexToBech32 } from "Util"; export const NAME = 'snortDB' -export const VERSION = 2 +export const VERSION = 3 + +export interface SubCache { + id: string, + ids: u256[], + until?: number, + since?: number, +} const STORES = { - users: '++pubkey, name, display_name, picture, nip05, npub' + users: '++pubkey, name, display_name, picture, nip05, npub', + events: '++id, pubkey, created_at', + feeds: '++id' } export class SnortDB extends Dexie { users!: Table; + events!: Table; + feeds!: Table; constructor() { super(NAME); - this.version(VERSION).stores(STORES).upgrade(tx => { - return tx.table("users").toCollection().modify(user => { + this.version(VERSION).stores(STORES).upgrade(async tx => { + await tx.table("users").toCollection().modify(user => { user.npub = hexToBech32("npub", user.pubkey) - }) + }); }); } } diff --git a/src/Feed/EventPublisher.ts b/src/Feed/EventPublisher.ts index 9d46ae4b..2f3b5285 100644 --- a/src/Feed/EventPublisher.ts +++ b/src/Feed/EventPublisher.ts @@ -78,8 +78,8 @@ export default function useEventPublisher() { } return { - nip42Auth: async (challenge: string, relay:string) => { - if(pubKey) { + nip42Auth: async (challenge: string, relay: string) => { + if (pubKey) { const ev = NEvent.ForPubKey(pubKey); ev.Kind = EventKind.Auth; ev.Content = ""; @@ -112,17 +112,17 @@ export default function useEventPublisher() { ev.Kind = EventKind.Lists; ev.Tags.push(new Tag(["d", Lists.Muted], ev.Tags.length)) keys.forEach(p => { - ev.Tags.push(new Tag(["p", p], ev.Tags.length)) + ev.Tags.push(new Tag(["p", p], ev.Tags.length)) }) let content = "" if (priv.length > 0) { - const ps = priv.map(p => ["p", p]) - const plaintext = JSON.stringify(ps) - if (hasNip07 && !privKey) { - content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext)); - } else if (privKey) { - content = await ev.EncryptData(plaintext, pubKey, privKey) - } + const ps = priv.map(p => ["p", p]) + const plaintext = JSON.stringify(ps) + if (hasNip07 && !privKey) { + content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext)); + } else if (privKey) { + content = await ev.EncryptData(plaintext, pubKey, privKey) + } } ev.Content = content; return await signEvent(ev); diff --git a/src/Feed/FollowersFeed.ts b/src/Feed/FollowersFeed.ts index 7e0f65eb..5c97e487 100644 --- a/src/Feed/FollowersFeed.ts +++ b/src/Feed/FollowersFeed.ts @@ -7,7 +7,7 @@ import useSubscription from "Feed/Subscription"; export default function useFollowersFeed(pubkey: HexKey) { const sub = useMemo(() => { let x = new Subscriptions(); - x.Id = "followers"; + x.Id = `followers:${pubkey.slice(0, 12)}`; x.Kinds = new Set([EventKind.ContactList]); x.PTags = new Set([pubkey]); diff --git a/src/Feed/FollowsFeed.ts b/src/Feed/FollowsFeed.ts index b1f4c883..28004000 100644 --- a/src/Feed/FollowsFeed.ts +++ b/src/Feed/FollowsFeed.ts @@ -7,7 +7,7 @@ import useSubscription, { NoteStore } from "Feed/Subscription"; export default function useFollowsFeed(pubkey: HexKey) { const sub = useMemo(() => { let x = new Subscriptions(); - x.Id = "follows"; + x.Id = `follows:${pubkey.slice(0, 12)}`; x.Kinds = new Set([EventKind.ContactList]); x.Authors = new Set([pubkey]); diff --git a/src/Feed/LoginFeed.ts b/src/Feed/LoginFeed.ts index 847dbc61..f3b67792 100644 --- a/src/Feed/LoginFeed.ts +++ b/src/Feed/LoginFeed.ts @@ -77,10 +77,10 @@ export default function useLoginFeed() { return dms; }, [pubKey]); - const metadataFeed = useSubscription(subMetadata, { leaveOpen: true }); - const notificationFeed = useSubscription(subNotification, { leaveOpen: true }); - const dmsFeed = useSubscription(subDms, { leaveOpen: true }); - const mutedFeed = useSubscription(subMuted, { leaveOpen: true }); + const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true }); + const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true }); + const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true }); + const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true }); useEffect(() => { let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList); diff --git a/src/Feed/ProfileFeed.ts b/src/Feed/ProfileFeed.ts index c8669410..4c1d5a01 100644 --- a/src/Feed/ProfileFeed.ts +++ b/src/Feed/ProfileFeed.ts @@ -1,6 +1,4 @@ -import { useLiveQuery } from "dexie-react-hooks"; -import { useEffect, useMemo } from "react"; -import { RootState } from "State/Store"; +import { useEffect } from "react"; import { MetadataCache } from "State/Users"; import { useKey, useKeys } from "State/Users/Hooks"; import { HexKey } from "Nostr"; diff --git a/src/Feed/Subscription.ts b/src/Feed/Subscription.ts index 54797dad..73a967cd 100644 --- a/src/Feed/Subscription.ts +++ b/src/Feed/Subscription.ts @@ -3,6 +3,7 @@ import { System } from "Nostr/System"; import { TaggedRawEvent } from "Nostr"; import { Subscriptions } from "Nostr/Subscriptions"; import { debounce } from "Util"; +import { db } from "Db"; export type NoteStore = { notes: Array, @@ -10,7 +11,8 @@ export type NoteStore = { }; export type UseSubscriptionOptions = { - leaveOpen: boolean + leaveOpen: boolean, + cache: boolean } interface ReducerArg { @@ -77,33 +79,49 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use const [state, dispatch] = useReducer(notesReducer, initStore); const [debounceOutput, setDebounceOutput] = useState(0); const [subDebounce, setSubDebounced] = useState(); + const useCache = useMemo(() => options?.cache === true, [options]); useEffect(() => { if (sub) { return debounce(DebounceMs, () => { - dispatch({ - type: "END", - end: false - }); setSubDebounced(sub); }); } }, [sub, options]); useEffect(() => { - if (sub) { - sub.OnEvent = (e) => { + if (subDebounce) { + dispatch({ + type: "END", + end: false + }); + + if (useCache) { + // preload notes from db + PreloadNotes(subDebounce.Id) + .then(ev => { + dispatch({ + type: "EVENT", + ev: ev + }); + }) + .catch(console.warn); + } + subDebounce.OnEvent = (e) => { dispatch({ type: "EVENT", ev: e }); + if (useCache) { + db.events.put(e); + } }; - sub.OnEnd = (c) => { + subDebounce.OnEnd = (c) => { if (!(options?.leaveOpen ?? false)) { - c.RemoveSubscription(sub.Id); - if (sub.IsFinished()) { - System.RemoveSubscription(sub.Id); + c.RemoveSubscription(subDebounce.Id); + if (subDebounce.IsFinished()) { + System.RemoveSubscription(subDebounce.Id); } } dispatch({ @@ -112,14 +130,23 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use }); }; - console.debug("Adding sub: ", sub.ToObject()); - System.AddSubscription(sub); + console.debug("Adding sub: ", subDebounce.ToObject()); + System.AddSubscription(subDebounce); return () => { - console.debug("Removing sub: ", sub.ToObject()); - System.RemoveSubscription(sub.Id); + console.debug("Removing sub: ", subDebounce.ToObject()); + System.RemoveSubscription(subDebounce.Id); }; } - }, [subDebounce]); + }, [subDebounce, useCache]); + + useEffect(() => { + if (subDebounce && useCache) { + return debounce(500, () => { + TrackNotesInFeed(subDebounce.Id, state.notes) + .catch(console.warn); + }); + } + }, [state, useCache]); useEffect(() => { return debounce(DebounceMs, () => { @@ -140,4 +167,24 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use }); } } +} + +/** + * Lookup cached copy of feed + */ +const PreloadNotes = async (id: string): Promise => { + const feed = await db.feeds.get(id); + if (feed) { + const events = await db.events.bulkGet(feed.ids); + return events.filter(a => a !== undefined).map(a => a!); + } + return []; +} + +const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => { + const existing = await db.feeds.get(id); + const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)])); + const since = notes.reduce((acc, v) => acc > v.created_at ? v.created_at : acc, +Infinity); + const until = notes.reduce((acc, v) => acc < v.created_at ? v.created_at : acc, -Infinity); + await db.feeds.put({ id, ids, since, until }); } \ No newline at end of file diff --git a/src/Feed/ThreadFeed.ts b/src/Feed/ThreadFeed.ts index aef2c024..25d52d1a 100644 --- a/src/Feed/ThreadFeed.ts +++ b/src/Feed/ThreadFeed.ts @@ -38,7 +38,7 @@ export default function useThreadFeed(id: u256) { return thisSub; }, [trackingEvents, pref, id]); - const main = useSubscription(sub, { leaveOpen: true }); + const main = useSubscription(sub, { leaveOpen: true, cache: true }); useEffect(() => { if (main.store) { diff --git a/src/Feed/TimelineFeed.ts b/src/Feed/TimelineFeed.ts index 0fc9c67a..0e3d4b5a 100644 --- a/src/Feed/TimelineFeed.ts +++ b/src/Feed/TimelineFeed.ts @@ -14,6 +14,7 @@ export interface TimelineFeedOptions { export interface TimelineSubject { type: "pubkey" | "hashtag" | "global" | "ptag" | "keyword", + discriminator: string, items: string[] } @@ -32,7 +33,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel } let sub = new Subscriptions(); - sub.Id = `timeline:${subject.type}`; + sub.Id = `timeline:${subject.type}:${subject.discriminator}`; sub.Kinds = new Set([EventKind.TextNote, EventKind.Repost]); switch (subject.type) { case "pubkey": { @@ -54,7 +55,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel } } return sub; - }, [subject.type, subject.items]); + }, [subject.type, subject.items, subject.discriminator]); const sub = useMemo(() => { let sub = createSub(); @@ -86,7 +87,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel return sub; }, [until, since, options.method, pref, createSub]); - const main = useSubscription(sub, { leaveOpen: true }); + const main = useSubscription(sub, { leaveOpen: true, cache: true }); const subRealtime = useMemo(() => { let subLatest = createSub(); @@ -98,7 +99,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel return subLatest; }, [pref, createSub]); - const latest = useSubscription(subRealtime, { leaveOpen: true }); + const latest = useSubscription(subRealtime, { leaveOpen: true, cache: false }); const subNext = useMemo(() => { let sub: Subscriptions | undefined; @@ -111,7 +112,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel return sub ?? null; }, [trackingEvents, pref, subject.type]); - const others = useSubscription(subNext, { leaveOpen: true }); + const others = useSubscription(subNext, { leaveOpen: true, cache: true }); const subParents = useMemo(() => { if (trackingParentEvents.length > 0) { diff --git a/src/Pages/HashTagsPage.tsx b/src/Pages/HashTagsPage.tsx index 4390c60c..7f5400a3 100644 --- a/src/Pages/HashTagsPage.tsx +++ b/src/Pages/HashTagsPage.tsx @@ -8,7 +8,7 @@ const HashTagsPage = () => { return ( <>

#{tag}

- + ) } diff --git a/src/Pages/Notifications.tsx b/src/Pages/Notifications.tsx index 0b063492..e9019bf8 100644 --- a/src/Pages/Notifications.tsx +++ b/src/Pages/Notifications.tsx @@ -17,7 +17,7 @@ export default function NotificationsPage() { return ( <> {pubkey ? - + : null} ) diff --git a/src/Pages/ProfilePage.tsx b/src/Pages/ProfilePage.tsx index d6d6b564..5496e1e2 100644 --- a/src/Pages/ProfilePage.tsx +++ b/src/Pages/ProfilePage.tsx @@ -105,7 +105,7 @@ export default function ProfilePage() { function tabContent() { switch (tab) { case ProfileTab.Notes: - return ; + return ; case ProfileTab.Follows: { if (isMe) { return ( @@ -195,7 +195,7 @@ export default function ProfilePage() { return ( <>
- {user?.banner && } + {user?.banner && }
{avatar()} {userDetails()} diff --git a/src/Pages/Root.tsx b/src/Pages/Root.tsx index 9e89694c..aeef0a03 100644 --- a/src/Pages/Root.tsx +++ b/src/Pages/Root.tsx @@ -29,7 +29,7 @@ export default function RootPage() { } const isGlobal = loggedOut || tab === RootTab.Global; - const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [] } : { type: "pubkey", items: follows }; + const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [], discriminator: "all" } : { type: "pubkey", items: follows, discriminator: "follows" }; return ( <> {pubKey ? <> diff --git a/src/Pages/SearchPage.tsx b/src/Pages/SearchPage.tsx index 7e51e1f5..28ba25b1 100644 --- a/src/Pages/SearchPage.tsx +++ b/src/Pages/SearchPage.tsx @@ -43,7 +43,7 @@ const SearchPage = () => {
setSearch(e.target.value)} />
- {keyword && } + {keyword && } ) } diff --git a/src/State/Users.ts b/src/State/Users.ts index 9f92183a..b8ff079b 100644 --- a/src/State/Users.ts +++ b/src/State/Users.ts @@ -35,20 +35,20 @@ export function mapEventToProfile(ev: TaggedRawEvent) { ...data } as MetadataCache; } catch (e) { - console.error("Failed to parse JSON", ev, e); + console.error("Failed to parse JSON", ev, e); } } export interface UsersDb { - isAvailable(): Promise - query(str: string): Promise - find(key: HexKey): Promise - add(user: MetadataCache): Promise - put(user: MetadataCache): Promise - bulkAdd(users: MetadataCache[]): Promise - bulkGet(keys: HexKey[]): Promise - bulkPut(users: MetadataCache[]): Promise - update(key: HexKey, fields: Record): Promise + isAvailable(): Promise + query(str: string): Promise + find(key: HexKey): Promise + add(user: MetadataCache): Promise + put(user: MetadataCache): Promise + bulkAdd(users: MetadataCache[]): Promise + bulkGet(keys: HexKey[]): Promise + bulkPut(users: MetadataCache[]): Promise + update(key: HexKey, fields: Record): Promise } export interface UsersStore { @@ -65,7 +65,7 @@ const UsersSlice = createSlice({ initialState: InitState, reducers: { setUsers(state, action: PayloadAction>) { - state.users = action.payload + state.users = action.payload } } }); diff --git a/src/State/Users/Db.ts b/src/State/Users/Db.ts index a2e5bf57..8983a414 100644 --- a/src/State/Users/Db.ts +++ b/src/State/Users/Db.ts @@ -5,6 +5,8 @@ import { UsersDb, MetadataCache, setUsers } from "State/Users"; import store from "State/Store"; class IndexedDb implements UsersDb { + ready: boolean = false; + isAvailable() { if ("indexedDB" in window) { return new Promise((resolve) => { @@ -26,12 +28,12 @@ class IndexedDb implements UsersDb { query(q: string) { return idb.users - .where("npub").startsWithIgnoreCase(q) - .or("name").startsWithIgnoreCase(q) - .or("display_name").startsWithIgnoreCase(q) - .or("nip05").startsWithIgnoreCase(q) - .limit(5) - .toArray() + .where("npub").startsWithIgnoreCase(q) + .or("name").startsWithIgnoreCase(q) + .or("display_name").startsWithIgnoreCase(q) + .or("nip05").startsWithIgnoreCase(q) + .limit(5) + .toArray() } bulkGet(keys: HexKey[]) { @@ -41,7 +43,7 @@ class IndexedDb implements UsersDb { add(user: MetadataCache) { return idb.users.add(user) } - + put(user: MetadataCache) { return idb.users.put(user) } @@ -88,21 +90,21 @@ class ReduxUsersDb implements UsersDb { async add(user: MetadataCache) { const state = store.getState() const { users } = state.users - store.dispatch(setUsers({...users, [user.pubkey]: user })) + store.dispatch(setUsers({ ...users, [user.pubkey]: user })) } async put(user: MetadataCache) { const state = store.getState() const { users } = state.users - store.dispatch(setUsers({...users, [user.pubkey]: user })) + store.dispatch(setUsers({ ...users, [user.pubkey]: user })) } async bulkAdd(newUserProfiles: MetadataCache[]) { const state = store.getState() const { users } = state.users const newUsers = newUserProfiles.reduce(groupByPubkey, {}) - store.dispatch(setUsers({...users, ...newUsers })) + store.dispatch(setUsers({ ...users, ...newUsers })) } async bulkGet(keys: HexKey[]) { @@ -118,8 +120,8 @@ class ReduxUsersDb implements UsersDb { const state = store.getState() const { users } = state.users const current = users[key] - const updated = {...current, ...fields } - store.dispatch(setUsers({...users, [key]: updated })) + const updated = { ...current, ...fields } + store.dispatch(setUsers({ ...users, [key]: updated })) } async bulkPut(newUsers: MetadataCache[]) { @@ -138,6 +140,7 @@ let db: UsersDb = inMemoryDb indexedDb.isAvailable().then((available) => { if (available) { console.debug('Using Indexed DB') + indexedDb.ready = true; db = indexedDb; } else { console.debug('Using in-memory DB') diff --git a/src/State/Users/Hooks.ts b/src/State/Users/Hooks.ts index 95d94e63..345daff1 100644 --- a/src/State/Users/Hooks.ts +++ b/src/State/Users/Hooks.ts @@ -10,10 +10,10 @@ export function useQuery(query: string, limit: number = 5) { const allUsers = useLiveQuery( () => db.query(query) - .catch((err) => { - console.error(err) - return inMemoryDb.query(query) - }), + .catch((err) => { + console.error(err) + return inMemoryDb.query(query) + }), [query], ) @@ -26,14 +26,14 @@ export function useKey(pubKey: HexKey) { const defaultUser = users[pubKey] const user = useLiveQuery(async () => { - if (pubKey) { - try { - return await db.find(pubKey); - } catch (error) { - console.error(error) - return defaultUser - } - } + if (pubKey) { + try { + return await db.find(pubKey); + } catch (error) { + console.error(error) + return defaultUser + } + } }, [pubKey, defaultUser]); return user @@ -42,17 +42,17 @@ export function useKey(pubKey: HexKey) { export function useKeys(pubKeys: HexKey[]): Map { const db = getDb() const dbUsers = useLiveQuery(async () => { - if (pubKeys) { - try { - const ret = await db.bulkGet(pubKeys); - return new Map(ret.map(a => [a.pubkey, a])) - } catch (error) { - console.error(error) - const ret = await inMemoryDb.bulkGet(pubKeys); - return new Map(ret.map(a => [a.pubkey, a])) - } + if (pubKeys) { + try { + const ret = await db.bulkGet(pubKeys); + return new Map(ret.map(a => [a.pubkey, a])) + } catch (error) { + console.error(error) + const ret = await inMemoryDb.bulkGet(pubKeys); + return new Map(ret.map(a => [a.pubkey, a])) } - return new Map() + } + return new Map() }, [pubKeys]); return dbUsers! From 59ff5de651c77f95629f1d8d4511e3f2558370bd Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 1 Feb 2023 19:48:35 +0000 Subject: [PATCH 2/3] useDb hook --- src/Feed/LoginFeed.ts | 236 +++++++++++++++++++-------------------- src/Nostr/System.ts | 90 ++++++++------- src/Notifications.ts | 6 +- src/Pages/Layout.tsx | 17 ++- src/State/Login.ts | 10 +- src/State/Users/Db.ts | 29 ++--- src/State/Users/Hooks.ts | 27 ++--- 7 files changed, 210 insertions(+), 205 deletions(-) diff --git a/src/Feed/LoginFeed.ts b/src/Feed/LoginFeed.ts index f3b67792..6d391485 100644 --- a/src/Feed/LoginFeed.ts +++ b/src/Feed/LoginFeed.ts @@ -8,151 +8,149 @@ import Event from "Nostr/Event"; import { Subscriptions } from "Nostr/Subscriptions"; import { addDirectMessage, setFollows, setRelays, setMuted, setBlocked, sendNotification } from "State/Login"; import { RootState } from "State/Store"; -import { mapEventToProfile, MetadataCache } from "State/Users"; -import { getDb } from "State/Users/Db"; +import { mapEventToProfile, MetadataCache } from "State/Users"; +import { useDb } from "State/Users/Db"; import useSubscription from "Feed/Subscription"; -import { getDisplayName } from "Element/ProfileImage"; import { barierNip07 } from "Feed/EventPublisher"; import { getMutedKeys, getNewest } from "Feed/MuteList"; -import { MentionRegex } from "Const"; import useModeration from "Hooks/useModeration"; /** * Managed loading data for the current logged in user */ export default function useLoginFeed() { - const dispatch = useDispatch(); - const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login); - const { isMuted } = useModeration(); + const dispatch = useDispatch(); + const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login); + const { isMuted } = useModeration(); + const db = useDb(); - const subMetadata = useMemo(() => { - if (!pubKey) return null; + const subMetadata = useMemo(() => { + if (!pubKey) return null; - let sub = new Subscriptions(); - sub.Id = `login:meta`; - sub.Authors = new Set([pubKey]); - sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]); - sub.Limit = 2 + let sub = new Subscriptions(); + sub.Id = `login:meta`; + sub.Authors = new Set([pubKey]); + sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]); + sub.Limit = 2 - return sub; - }, [pubKey]); + return sub; + }, [pubKey]); - const subNotification = useMemo(() => { - if (!pubKey) return null; + const subNotification = useMemo(() => { + if (!pubKey) return null; - let sub = new Subscriptions(); - sub.Id = "login:notifications"; - sub.Kinds = new Set([EventKind.TextNote]); - sub.PTags = new Set([pubKey]); - sub.Limit = 1; - return sub; - }, [pubKey]); + let sub = new Subscriptions(); + sub.Id = "login:notifications"; + sub.Kinds = new Set([EventKind.TextNote]); + sub.PTags = new Set([pubKey]); + sub.Limit = 1; + return sub; + }, [pubKey]); - const subMuted = useMemo(() => { - if (!pubKey) return null; + const subMuted = useMemo(() => { + if (!pubKey) return null; - let sub = new Subscriptions(); - sub.Id = "login:muted"; - sub.Kinds = new Set([EventKind.Lists]); - sub.Authors = new Set([pubKey]); - sub.DTags = new Set([Lists.Muted]); - sub.Limit = 1; + let sub = new Subscriptions(); + sub.Id = "login:muted"; + sub.Kinds = new Set([EventKind.Lists]); + sub.Authors = new Set([pubKey]); + sub.DTags = new Set([Lists.Muted]); + sub.Limit = 1; - return sub; - }, [pubKey]); + return sub; + }, [pubKey]); - const subDms = useMemo(() => { - if (!pubKey) return null; + const subDms = useMemo(() => { + if (!pubKey) return null; - let dms = new Subscriptions(); - dms.Id = "login:dms"; - dms.Kinds = new Set([EventKind.DirectMessage]); - dms.PTags = new Set([pubKey]); + let dms = new Subscriptions(); + dms.Id = "login:dms"; + dms.Kinds = new Set([EventKind.DirectMessage]); + dms.PTags = new Set([pubKey]); - let dmsFromME = new Subscriptions(); - dmsFromME.Authors = new Set([pubKey]); - dmsFromME.Kinds = new Set([EventKind.DirectMessage]); - dms.AddSubscription(dmsFromME); + let dmsFromME = new Subscriptions(); + dmsFromME.Authors = new Set([pubKey]); + dmsFromME.Kinds = new Set([EventKind.DirectMessage]); + dms.AddSubscription(dmsFromME); - return dms; - }, [pubKey]); + return dms; + }, [pubKey]); - const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true }); - const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true }); - const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true }); - const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true }); + const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true }); + const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true }); + const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true }); + const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true }); - useEffect(() => { - let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList); - let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata); - let profiles = metadata.map(a => mapEventToProfile(a)) - .filter(a => a !== undefined) - .map(a => a!); + useEffect(() => { + let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList); + let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata); + let profiles = metadata.map(a => mapEventToProfile(a)) + .filter(a => a !== undefined) + .map(a => a!); - for (let cl of contactList) { - if (cl.content !== "" && cl.content !== "{}") { - let relays = JSON.parse(cl.content); - dispatch(setRelays({ relays, createdAt: cl.created_at })); - } - let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]); - dispatch(setFollows({ keys: pTags, createdAt: cl.created_at })); - } - - (async () => { - let maxProfile = profiles.reduce((acc, v) => { - if (v.created > acc.created) { - acc.profile = v; - acc.created = v.created; - } - return acc; - }, { created: 0, profile: null as MetadataCache | null }); - if (maxProfile.profile) { - const db = getDb() - let existing = await db.find(maxProfile.profile.pubkey); - if ((existing?.created ?? 0) < maxProfile.created) { - await db.put(maxProfile.profile); - } - } - })().catch(console.warn); - }, [dispatch, metadataFeed.store]); - - useEffect(() => { - const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey)) - replies.forEach(nx => { - makeNotification(nx).then(notification => { - if (notification) { - // @ts-ignore - dispatch(sendNotification(notification)) - } - }) - }) - }, [dispatch, notificationFeed.store]); - - useEffect(() => { - const muted = getMutedKeys(mutedFeed.store.notes) - dispatch(setMuted(muted)) - - const newest = getNewest(mutedFeed.store.notes) - if (newest && newest.content.length > 0 && pubKey) { - decryptBlocked(newest, pubKey, privKey).then((plaintext) => { - try { - const blocked = JSON.parse(plaintext) - const keys = blocked.filter((p:any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1]) - dispatch(setBlocked({ - keys, - createdAt: newest.created_at, - })) - } catch(error) { - console.debug("Couldn't parse JSON") - } - }).catch((error) => console.warn(error)) + for (let cl of contactList) { + if (cl.content !== "" && cl.content !== "{}") { + let relays = JSON.parse(cl.content); + dispatch(setRelays({ relays, createdAt: cl.created_at })); } - }, [dispatch, mutedFeed.store]) + let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]); + dispatch(setFollows({ keys: pTags, createdAt: cl.created_at })); + } - useEffect(() => { - let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage); - dispatch(addDirectMessage(dms)); - }, [dispatch, dmsFeed.store]); + (async () => { + let maxProfile = profiles.reduce((acc, v) => { + if (v.created > acc.created) { + acc.profile = v; + acc.created = v.created; + } + return acc; + }, { created: 0, profile: null as MetadataCache | null }); + if (maxProfile.profile) { + let existing = await db.find(maxProfile.profile.pubkey); + if ((existing?.created ?? 0) < maxProfile.created) { + await db.put(maxProfile.profile); + } + } + })().catch(console.warn); + }, [dispatch, metadataFeed.store, db]); + + useEffect(() => { + const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey)) + replies.forEach(nx => { + makeNotification(db, nx).then(notification => { + if (notification) { + // @ts-ignore + dispatch(sendNotification(notification)) + } + }) + }) + }, [dispatch, notificationFeed.store, db]); + + useEffect(() => { + const muted = getMutedKeys(mutedFeed.store.notes) + dispatch(setMuted(muted)) + + const newest = getNewest(mutedFeed.store.notes) + if (newest && newest.content.length > 0 && pubKey) { + decryptBlocked(newest, pubKey, privKey).then((plaintext) => { + try { + const blocked = JSON.parse(plaintext) + const keys = blocked.filter((p: any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1]) + dispatch(setBlocked({ + keys, + createdAt: newest.created_at, + })) + } catch (error) { + console.debug("Couldn't parse JSON") + } + }).catch((error) => console.warn(error)) + } + }, [dispatch, mutedFeed.store]) + + useEffect(() => { + let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage); + dispatch(addDirectMessage(dms)); + }, [dispatch, dmsFeed.store]); } diff --git a/src/Nostr/System.ts b/src/Nostr/System.ts index 7c1fcb4c..9314206e 100644 --- a/src/Nostr/System.ts +++ b/src/Nostr/System.ts @@ -1,7 +1,6 @@ import { HexKey, TaggedRawEvent } from "Nostr"; -import { getDb } from "State/Users/Db"; import { ProfileCacheExpire } from "Const"; -import { mapEventToProfile, MetadataCache } from "State/Users"; +import { mapEventToProfile, MetadataCache, UsersDb } from "State/Users"; import Connection, { RelaySettings } from "Nostr/Connection"; import Event from "Nostr/Event"; import EventKind from "Nostr/EventKind"; @@ -31,6 +30,11 @@ export class NostrSystem { */ WantsMetadata: Set; + /** + * User db store + */ + UserDb?: UsersDb; + constructor() { this.Sockets = new Map(); this.Subscriptions = new Map(); @@ -166,54 +170,54 @@ export class NostrSystem { } async _FetchMetadata() { - let missing = new Set(); - const db = getDb() - let meta = await db.bulkGet(Array.from(this.WantsMetadata)); - let expire = new Date().getTime() - ProfileCacheExpire; - for (let pk of this.WantsMetadata) { - let m = meta.find(a => a?.pubkey === pk); - if (!m || m.loaded < expire) { - missing.add(pk); - // cap 100 missing profiles - if (missing.size >= 100) { - break; - } - } - } - - if (missing.size > 0) { - console.debug("Wants profiles: ", missing); - - let sub = new Subscriptions(); - sub.Id = `profiles:${sub.Id}`; - sub.Kinds = new Set([EventKind.SetMetadata]); - sub.Authors = missing; - sub.OnEvent = async (e) => { - let profile = mapEventToProfile(e); - if (profile) { - let existing = await db.find(profile.pubkey); - if ((existing?.created ?? 0) < profile.created) { - await db.put(profile); - } else if (existing) { - await db.update(profile.pubkey, { loaded: new Date().getTime() }); + if (this.UserDb) { + let missing = new Set(); + let meta = await this.UserDb.bulkGet(Array.from(this.WantsMetadata)); + let expire = new Date().getTime() - ProfileCacheExpire; + for (let pk of this.WantsMetadata) { + let m = meta.find(a => a?.pubkey === pk); + if (!m || m.loaded < expire) { + missing.add(pk); + // cap 100 missing profiles + if (missing.size >= 100) { + break; } } } - let results = await this.RequestSubscription(sub); - let couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a)); - console.debug("No profiles: ", couldNotFetch); - await db.bulkPut(couldNotFetch.map(a => { - return { - pubkey: a, - loaded: new Date().getTime() - } as MetadataCache; - })); - } + if (missing.size > 0) { + console.debug("Wants profiles: ", missing); + + let sub = new Subscriptions(); + sub.Id = `profiles:${sub.Id}`; + sub.Kinds = new Set([EventKind.SetMetadata]); + sub.Authors = missing; + sub.OnEvent = async (e) => { + let profile = mapEventToProfile(e); + if (profile) { + let existing = await this.UserDb!.find(profile.pubkey); + if ((existing?.created ?? 0) < profile.created) { + await this.UserDb!.put(profile); + } else if (existing) { + await this.UserDb!.update(profile.pubkey, { loaded: new Date().getTime() }); + } + } + } + let results = await this.RequestSubscription(sub); + let couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a)); + console.debug("No profiles: ", couldNotFetch); + await this.UserDb!.bulkPut(couldNotFetch.map(a => { + return { + pubkey: a, + loaded: new Date().getTime() + } as MetadataCache; + })); + } + } setTimeout(() => this._FetchMetadata(), 500); } - async nip42Auth(challenge: string, relay:string): Promise { + async nip42Auth(challenge: string, relay: string): Promise { return } } diff --git a/src/Notifications.ts b/src/Notifications.ts index 074c23c2..7ce9b58d 100644 --- a/src/Notifications.ts +++ b/src/Notifications.ts @@ -3,13 +3,11 @@ import Nostrich from "nostrich.webp"; import { TaggedRawEvent } from "Nostr"; import EventKind from "Nostr/EventKind"; import type { NotificationRequest } from "State/Login"; -import { MetadataCache } from "State/Users"; -import { getDb } from "State/Users/Db"; +import { MetadataCache, UsersDb } from "State/Users"; import { getDisplayName } from "Element/ProfileImage"; import { MentionRegex } from "Const"; -export async function makeNotification(ev: TaggedRawEvent): Promise { - const db = getDb() +export async function makeNotification(db: UsersDb, ev: TaggedRawEvent): Promise { switch (ev.kind) { case EventKind.TextNote: { const pubkeys = new Set([ev.pubkey, ...ev.tags.filter(a => a[0] === "p").map(a => a[1]!)]); diff --git a/src/Pages/Layout.tsx b/src/Pages/Layout.tsx index 0e847447..40a54302 100644 --- a/src/Pages/Layout.tsx +++ b/src/Pages/Layout.tsx @@ -17,6 +17,7 @@ import { totalUnread } from "Pages/MessagesPage"; import { SearchRelays } from 'Const'; import useEventPublisher from "Feed/EventPublisher"; import useModeration from "Hooks/useModeration"; +import { IndexedUDB, useDb } from "State/Users/Db"; export default function Layout() { @@ -31,12 +32,17 @@ export default function Layout() { const { isMuted } = useModeration(); const filteredDms = dms.filter(a => !isMuted(a.pubkey)) const prefs = useSelector(s => s.login.preferences); + const usingDb = useDb(); const pub = useEventPublisher(); useLoginFeed(); useEffect(() => { System.nip42Auth = pub.nip42Auth - },[pub]) + }, [pub]) + + useEffect(() => { + System.UserDb = usingDb; + }, [usingDb]) useEffect(() => { if (relays) { @@ -73,7 +79,14 @@ export default function Layout() { }, [prefs.theme]); useEffect(() => { - dispatch(init()); + // check DB support then init + IndexedUDB.isAvailable() + .then(a => { + const db = a ? "indexdDb" : "redux"; + console.debug(`Using db: ${db}`); + dispatch(init(db)); + }) + }, []); async function goToNotifications(e: any) { diff --git a/src/State/Login.ts b/src/State/Login.ts index ddaee2ac..7974192f 100644 --- a/src/State/Login.ts +++ b/src/State/Login.ts @@ -61,8 +61,14 @@ export interface UserPreferences { */ imgProxyConfig: ImgProxySettings | null } +export type DbType = "indexdDb" | "redux"; export interface LoginStore { + /** + * Which db we will use to cache data + */ + useDb: DbType, + /** * If there is no login */ @@ -146,6 +152,7 @@ const DefaultImgProxy = { }; export const InitState = { + useDb: "redux", loggedOut: undefined, publicKey: undefined, privateKey: undefined, @@ -186,7 +193,8 @@ const LoginSlice = createSlice({ name: "Login", initialState: InitState, reducers: { - init: (state) => { + init: (state, action: PayloadAction) => { + state.useDb = action.payload; state.privateKey = window.localStorage.getItem(PrivateKeyItem) ?? undefined; if (state.privateKey) { window.localStorage.removeItem(PublicKeyItem); // reset nip07 if using private key diff --git a/src/State/Users/Db.ts b/src/State/Users/Db.ts index 8983a414..22ed60bd 100644 --- a/src/State/Users/Db.ts +++ b/src/State/Users/Db.ts @@ -2,9 +2,10 @@ import { HexKey } from "Nostr"; import { db as idb } from "Db"; import { UsersDb, MetadataCache, setUsers } from "State/Users"; -import store from "State/Store"; +import store, { RootState } from "State/Store"; +import { useSelector } from "react-redux"; -class IndexedDb implements UsersDb { +class IndexedUsersDb implements UsersDb { ready: boolean = false; isAvailable() { @@ -132,21 +133,13 @@ class ReduxUsersDb implements UsersDb { } } +export const IndexedUDB = new IndexedUsersDb(); +export const ReduxUDB = new ReduxUsersDb(); -const indexedDb = new IndexedDb() -export const inMemoryDb = new ReduxUsersDb() - -let db: UsersDb = inMemoryDb -indexedDb.isAvailable().then((available) => { - if (available) { - console.debug('Using Indexed DB') - indexedDb.ready = true; - db = indexedDb; - } else { - console.debug('Using in-memory DB') +export function useDb(): UsersDb { + const db = useSelector((s: RootState) => s.login.useDb); + switch (db) { + case "indexdDb": return IndexedUDB + default: return ReduxUDB } -}) - -export function getDb() { - return db -} +} \ No newline at end of file diff --git a/src/State/Users/Hooks.ts b/src/State/Users/Hooks.ts index 345daff1..d6628328 100644 --- a/src/State/Users/Hooks.ts +++ b/src/State/Users/Hooks.ts @@ -1,27 +1,17 @@ import { useSelector } from "react-redux" import { useLiveQuery } from "dexie-react-hooks"; import { MetadataCache } from "State/Users"; -import { getDb, inMemoryDb } from "State/Users/Db"; import type { RootState } from "State/Store" import { HexKey } from "Nostr"; +import { useDb } from "./Db"; export function useQuery(query: string, limit: number = 5) { - const db = getDb() - - const allUsers = useLiveQuery( - () => db.query(query) - .catch((err) => { - console.error(err) - return inMemoryDb.query(query) - }), - [query], - ) - - return allUsers + const db = useDb() + return useLiveQuery(async () => db.query(query), [query],) } export function useKey(pubKey: HexKey) { - const db = getDb() + const db = useDb() const { users } = useSelector((state: RootState) => state.users) const defaultUser = users[pubKey] @@ -40,7 +30,9 @@ export function useKey(pubKey: HexKey) { } export function useKeys(pubKeys: HexKey[]): Map { - const db = getDb() + const db = useDb() + const { users } = useSelector((state: RootState) => state.users) + const dbUsers = useLiveQuery(async () => { if (pubKeys) { try { @@ -48,12 +40,11 @@ export function useKeys(pubKeys: HexKey[]): Map { return new Map(ret.map(a => [a.pubkey, a])) } catch (error) { console.error(error) - const ret = await inMemoryDb.bulkGet(pubKeys); - return new Map(ret.map(a => [a.pubkey, a])) + return new Map(pubKeys.map(a => [a, users[a]])) } } return new Map() - }, [pubKeys]); + }, [pubKeys, users]); return dbUsers! } From 162ab55445d438160b9ebfa59c391606e68b6c7b Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 1 Feb 2023 20:34:23 +0000 Subject: [PATCH 3/3] cleanup old data at startup --- src/Pages/Layout.tsx | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Pages/Layout.tsx b/src/Pages/Layout.tsx index 40a54302..a4bd1c24 100644 --- a/src/Pages/Layout.tsx +++ b/src/Pages/Layout.tsx @@ -18,6 +18,7 @@ import { SearchRelays } from 'Const'; import useEventPublisher from "Feed/EventPublisher"; import useModeration from "Hooks/useModeration"; import { IndexedUDB, useDb } from "State/Users/Db"; +import { db } from "Db"; export default function Layout() { @@ -81,10 +82,24 @@ export default function Layout() { useEffect(() => { // check DB support then init IndexedUDB.isAvailable() - .then(a => { - const db = a ? "indexdDb" : "redux"; - console.debug(`Using db: ${db}`); - dispatch(init(db)); + .then(async a => { + const dbType = a ? "indexdDb" : "redux"; + + // cleanup on load + if (dbType === "indexdDb") { + await db.feeds.clear(); + const now = Math.floor(new Date().getTime() / 1000); + + const cleanupEvents = await db.events + .where("created_at") + .above(now - (60 * 60)) + .primaryKeys(); + console.debug(`Cleanup ${cleanupEvents.length} events`); + await db.events.bulkDelete(cleanupEvents) + } + + console.debug(`Using db: ${dbType}`); + dispatch(init(dbType)); }) }, []);