feed cache

src/Db/index.ts

@@ -1,23 +1,35 @@
 import Dexie, { Table } from "dexie";
+import { TaggedRawEvent, u256 } from "Nostr";
 import { MetadataCache } from "State/Users";
 import { hexToBech32 } from "Util";
 
 export const NAME = 'snortDB'
-export const VERSION = 2
+export const VERSION = 3
+
+export interface SubCache {
+  id: string,
+  ids: u256[],
+  until?: number,
+  since?: number,
+}
 
 const STORES = {
-  users: '++pubkey, name, display_name, picture, nip05, npub'
+  users: '++pubkey, name, display_name, picture, nip05, npub',
+  events: '++id, pubkey, created_at',
+  feeds: '++id'
 }
 
 export class SnortDB extends Dexie {
   users!: Table<MetadataCache>;
+  events!: Table<TaggedRawEvent>;
+  feeds!: Table<SubCache>;
 
   constructor() {
     super(NAME);
-    this.version(VERSION).stores(STORES).upgrade(tx => {
+    this.version(VERSION).stores(STORES).upgrade(async tx => {
-      return tx.table("users").toCollection().modify(user => {
+      await tx.table("users").toCollection().modify(user => {
         user.npub = hexToBech32("npub", user.pubkey)
-      })
+      });
     });
   }
 }

src/Feed/EventPublisher.ts

@@ -78,8 +78,8 @@ export default function useEventPublisher() {
     }
 
     return {
-        nip42Auth: async (challenge: string, relay:string) => {
+        nip42Auth: async (challenge: string, relay: string) => {
-            if(pubKey) {
+            if (pubKey) {
                 const ev = NEvent.ForPubKey(pubKey);
                 ev.Kind = EventKind.Auth;
                 ev.Content = "";
@@ -112,17 +112,17 @@ export default function useEventPublisher() {
                 ev.Kind = EventKind.Lists;
                 ev.Tags.push(new Tag(["d", Lists.Muted], ev.Tags.length))
                 keys.forEach(p => {
-                  ev.Tags.push(new Tag(["p", p], ev.Tags.length))
+                    ev.Tags.push(new Tag(["p", p], ev.Tags.length))
                 })
                 let content = ""
                 if (priv.length > 0) {
-                  const ps = priv.map(p => ["p", p])
+                    const ps = priv.map(p => ["p", p])
-                  const plaintext = JSON.stringify(ps)
+                    const plaintext = JSON.stringify(ps)
-                  if (hasNip07 && !privKey) {
+                    if (hasNip07 && !privKey) {
-                      content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
+                        content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
-                  } else if (privKey) {
+                    } else if (privKey) {
-                      content = await ev.EncryptData(plaintext, pubKey, privKey)
+                        content = await ev.EncryptData(plaintext, pubKey, privKey)
-                  }
+                    }
                 }
                 ev.Content = content;
                 return await signEvent(ev);

src/Feed/FollowersFeed.ts

@@ -7,7 +7,7 @@ import useSubscription from "Feed/Subscription";
 export default function useFollowersFeed(pubkey: HexKey) {
     const sub = useMemo(() => {
         let x = new Subscriptions();
-        x.Id = "followers";
+        x.Id = `followers:${pubkey.slice(0, 12)}`;
         x.Kinds = new Set([EventKind.ContactList]);
         x.PTags = new Set([pubkey]);
 

src/Feed/FollowsFeed.ts

@@ -7,7 +7,7 @@ import useSubscription, { NoteStore } from "Feed/Subscription";
 export default function useFollowsFeed(pubkey: HexKey) {
     const sub = useMemo(() => {
         let x = new Subscriptions();
-        x.Id = "follows";
+        x.Id = `follows:${pubkey.slice(0, 12)}`;
         x.Kinds = new Set([EventKind.ContactList]);
         x.Authors = new Set([pubkey]);
 

src/Feed/LoginFeed.ts

@@ -77,10 +77,10 @@ export default function useLoginFeed() {
         return dms;
     }, [pubKey]);
 
-    const metadataFeed = useSubscription(subMetadata, { leaveOpen: true });
+    const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true });
-    const notificationFeed = useSubscription(subNotification, { leaveOpen: true });
+    const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true });
-    const dmsFeed = useSubscription(subDms, { leaveOpen: true });
+    const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true });
-    const mutedFeed = useSubscription(subMuted, { leaveOpen: true });
+    const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true });
 
     useEffect(() => {
         let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);

src/Feed/ProfileFeed.ts

@@ -1,6 +1,4 @@
-import { useLiveQuery } from "dexie-react-hooks";
+import { useEffect } from "react";
-import { useEffect, useMemo } from "react";
-import { RootState } from "State/Store";
 import { MetadataCache } from "State/Users";
 import { useKey, useKeys } from "State/Users/Hooks";
 import { HexKey } from "Nostr";

src/Feed/Subscription.ts

@@ -3,6 +3,7 @@ import { System } from "Nostr/System";
 import { TaggedRawEvent } from "Nostr";
 import { Subscriptions } from "Nostr/Subscriptions";
 import { debounce } from "Util";
+import { db } from "Db";
 
 export type NoteStore = {
     notes: Array<TaggedRawEvent>,
@@ -10,7 +11,8 @@ export type NoteStore = {
 };
 
 export type UseSubscriptionOptions = {
-    leaveOpen: boolean
+    leaveOpen: boolean,
+    cache: boolean
 }
 
 interface ReducerArg {
@@ -77,33 +79,49 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
     const [state, dispatch] = useReducer(notesReducer, initStore);
     const [debounceOutput, setDebounceOutput] = useState<number>(0);
     const [subDebounce, setSubDebounced] = useState<Subscriptions>();
+    const useCache = useMemo(() => options?.cache === true, [options]);
 
     useEffect(() => {
         if (sub) {
             return debounce(DebounceMs, () => {
-                dispatch({
-                    type: "END",
-                    end: false
-                });
                 setSubDebounced(sub);
             });
         }
     }, [sub, options]);
 
     useEffect(() => {
-        if (sub) {
+        if (subDebounce) {
-            sub.OnEvent = (e) => {
+            dispatch({
+                type: "END",
+                end: false
+            });
+
+            if (useCache) {
+                // preload notes from db
+                PreloadNotes(subDebounce.Id)
+                    .then(ev => {
+                        dispatch({
+                            type: "EVENT",
+                            ev: ev
+                        });
+                    })
+                    .catch(console.warn);
+            }
+            subDebounce.OnEvent = (e) => {
                 dispatch({
                     type: "EVENT",
                     ev: e
                 });
+                if (useCache) {
+                    db.events.put(e);
+                }
             };
 
-            sub.OnEnd = (c) => {
+            subDebounce.OnEnd = (c) => {
                 if (!(options?.leaveOpen ?? false)) {
-                    c.RemoveSubscription(sub.Id);
+                    c.RemoveSubscription(subDebounce.Id);
-                    if (sub.IsFinished()) {
+                    if (subDebounce.IsFinished()) {
-                        System.RemoveSubscription(sub.Id);
+                        System.RemoveSubscription(subDebounce.Id);
                     }
                 }
                 dispatch({
@@ -112,14 +130,23 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
                 });
             };
 
-            console.debug("Adding sub: ", sub.ToObject());
+            console.debug("Adding sub: ", subDebounce.ToObject());
-            System.AddSubscription(sub);
+            System.AddSubscription(subDebounce);
             return () => {
-                console.debug("Removing sub: ", sub.ToObject());
+                console.debug("Removing sub: ", subDebounce.ToObject());
-                System.RemoveSubscription(sub.Id);
+                System.RemoveSubscription(subDebounce.Id);
             };
         }
-    }, [subDebounce]);
+    }, [subDebounce, useCache]);
+
+    useEffect(() => {
+        if (subDebounce && useCache) {
+            return debounce(500, () => {
+                TrackNotesInFeed(subDebounce.Id, state.notes)
+                    .catch(console.warn);
+            });
+        }
+    }, [state, useCache]);
 
     useEffect(() => {
         return debounce(DebounceMs, () => {
@@ -140,4 +167,24 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
             });
         }
     }
+}
+
+/**
+ * Lookup cached copy of feed
+ */
+const PreloadNotes = async (id: string): Promise<TaggedRawEvent[]> => {
+    const feed = await db.feeds.get(id);
+    if (feed) {
+        const events = await db.events.bulkGet(feed.ids);
+        return events.filter(a => a !== undefined).map(a => a!);
+    }
+    return [];
+}
+
+const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => {
+    const existing = await db.feeds.get(id);
+    const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)]));
+    const since = notes.reduce((acc, v) => acc > v.created_at ? v.created_at : acc, +Infinity);
+    const until = notes.reduce((acc, v) => acc < v.created_at ? v.created_at : acc, -Infinity);
+    await db.feeds.put({ id, ids, since, until });
 }

src/Feed/ThreadFeed.ts

@@ -38,7 +38,7 @@ export default function useThreadFeed(id: u256) {
         return thisSub;
     }, [trackingEvents, pref, id]);
 
-    const main = useSubscription(sub, { leaveOpen: true });
+    const main = useSubscription(sub, { leaveOpen: true, cache: true });
 
     useEffect(() => {
         if (main.store) {

src/Feed/TimelineFeed.ts

@@ -14,6 +14,7 @@ export interface TimelineFeedOptions {
 
 export interface TimelineSubject {
     type: "pubkey" | "hashtag" | "global" | "ptag" | "keyword",
+    discriminator: string,
     items: string[]
 }
 
@@ -32,7 +33,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         }
 
         let sub = new Subscriptions();
-        sub.Id = `timeline:${subject.type}`;
+        sub.Id = `timeline:${subject.type}:${subject.discriminator}`;
         sub.Kinds = new Set([EventKind.TextNote, EventKind.Repost]);
         switch (subject.type) {
             case "pubkey": {
@@ -54,7 +55,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
             }
         }
         return sub;
-    }, [subject.type, subject.items]);
+    }, [subject.type, subject.items, subject.discriminator]);
 
     const sub = useMemo(() => {
         let sub = createSub();
@@ -86,7 +87,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return sub;
     }, [until, since, options.method, pref, createSub]);
 
-    const main = useSubscription(sub, { leaveOpen: true });
+    const main = useSubscription(sub, { leaveOpen: true, cache: true });
 
     const subRealtime = useMemo(() => {
         let subLatest = createSub();
@@ -98,7 +99,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return subLatest;
     }, [pref, createSub]);
 
-    const latest = useSubscription(subRealtime, { leaveOpen: true });
+    const latest = useSubscription(subRealtime, { leaveOpen: true, cache: false });
 
     const subNext = useMemo(() => {
         let sub: Subscriptions | undefined;
@@ -111,7 +112,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return sub ?? null;
     }, [trackingEvents, pref, subject.type]);
 
-    const others = useSubscription(subNext, { leaveOpen: true });
+    const others = useSubscription(subNext, { leaveOpen: true, cache: true });
 
     const subParents = useMemo(() => {
         if (trackingParentEvents.length > 0) {

src/Pages/HashTagsPage.tsx

@@ -8,7 +8,7 @@ const HashTagsPage = () => {
     return (
         <>
             <h2>#{tag}</h2>
-            <Timeline key={tag} subject={{ type: "hashtag", items: [tag] }} postsOnly={false} method={"TIME_RANGE"} />
+            <Timeline key={tag} subject={{ type: "hashtag", items: [tag], discriminator: tag }} postsOnly={false} method={"TIME_RANGE"} />
         </>
     )
 }

src/Pages/Notifications.tsx

@@ -17,7 +17,7 @@ export default function NotificationsPage() {
     return (
         <>
             {pubkey ?
-                <Timeline subject={{ type: "ptag", items: [pubkey!] }} postsOnly={false} method={"TIME_RANGE"} />
+                <Timeline subject={{ type: "ptag", items: [pubkey!], discriminator: pubkey!.slice(0, 12) }} postsOnly={false} method={"TIME_RANGE"} />
                 : null}
         </>
     )

src/Pages/ProfilePage.tsx

@@ -105,7 +105,7 @@ export default function ProfilePage() {
   function tabContent() {
     switch (tab) {
       case ProfileTab.Notes:
-        return <Timeline key={id} subject={{ type: "pubkey", items: [id] }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
+        return <Timeline key={id} subject={{ type: "pubkey", items: [id], discriminator: id.slice(0, 12) }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
       case ProfileTab.Follows: {
         if (isMe) {
           return (
@@ -195,7 +195,7 @@ export default function ProfilePage() {
   return (
     <>
       <div className="profile flex">
-        {user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w}/>}
+        {user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w} />}
         <div className="profile-wrapper flex">
           {avatar()}
           {userDetails()}

src/Pages/Root.tsx

@@ -29,7 +29,7 @@ export default function RootPage() {
     }
 
     const isGlobal = loggedOut || tab === RootTab.Global;
-    const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [] } : { type: "pubkey", items: follows };
+    const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [], discriminator: "all" } : { type: "pubkey", items: follows, discriminator: "follows" };
     return (
         <>
             {pubKey ? <>

src/Pages/SearchPage.tsx

@@ -43,7 +43,7 @@ const SearchPage = () => {
             <div className="flex mb10">
                 <input type="text" className="f-grow mr10" placeholder="Search.." value={search} onChange={e => setSearch(e.target.value)} />
             </div>
-            {keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword] }} postsOnly={false} method={"LIMIT_UNTIL"} />}
+            {keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword], discriminator: keyword }} postsOnly={false} method={"LIMIT_UNTIL"} />}
         </>
     )
 }

src/State/Users.ts

@@ -35,20 +35,20 @@ export function mapEventToProfile(ev: TaggedRawEvent) {
             ...data
         } as MetadataCache;
     } catch (e) {
-      console.error("Failed to parse JSON", ev, e);
+        console.error("Failed to parse JSON", ev, e);
     }
 }
 
 export interface UsersDb {
-  isAvailable(): Promise<boolean>
+    isAvailable(): Promise<boolean>
-  query(str: string): Promise<MetadataCache[]>
+    query(str: string): Promise<MetadataCache[]>
-  find(key: HexKey): Promise<MetadataCache | undefined>
+    find(key: HexKey): Promise<MetadataCache | undefined>
-  add(user: MetadataCache): Promise<any>
+    add(user: MetadataCache): Promise<any>
-  put(user: MetadataCache): Promise<any>
+    put(user: MetadataCache): Promise<any>
-  bulkAdd(users: MetadataCache[]): Promise<any>
+    bulkAdd(users: MetadataCache[]): Promise<any>
-  bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
+    bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
-  bulkPut(users: MetadataCache[]): Promise<any>
+    bulkPut(users: MetadataCache[]): Promise<any>
-  update(key: HexKey, fields: Record<string, any>): Promise<any>
+    update(key: HexKey, fields: Record<string, any>): Promise<any>
 }
 
 export interface UsersStore {
@@ -65,7 +65,7 @@ const UsersSlice = createSlice({
     initialState: InitState,
     reducers: {
         setUsers(state, action: PayloadAction<Record<HexKey, MetadataCache>>) {
-          state.users = action.payload
+            state.users = action.payload
         }
     }
 });

src/State/Users/Db.ts

@@ -5,6 +5,8 @@ import { UsersDb, MetadataCache, setUsers } from "State/Users";
 import store from "State/Store";
 
 class IndexedDb implements UsersDb {
+  ready: boolean = false;
+
   isAvailable() {
     if ("indexedDB" in window) {
       return new Promise<boolean>((resolve) => {
@@ -26,12 +28,12 @@ class IndexedDb implements UsersDb {
 
   query(q: string) {
     return idb.users
-            .where("npub").startsWithIgnoreCase(q)
+      .where("npub").startsWithIgnoreCase(q)
-            .or("name").startsWithIgnoreCase(q)
+      .or("name").startsWithIgnoreCase(q)
-            .or("display_name").startsWithIgnoreCase(q)
+      .or("display_name").startsWithIgnoreCase(q)
-            .or("nip05").startsWithIgnoreCase(q)
+      .or("nip05").startsWithIgnoreCase(q)
-            .limit(5)
+      .limit(5)
-            .toArray()
+      .toArray()
   }
 
   bulkGet(keys: HexKey[]) {
@@ -41,7 +43,7 @@ class IndexedDb implements UsersDb {
   add(user: MetadataCache) {
     return idb.users.add(user)
   }
-  
+
   put(user: MetadataCache) {
     return idb.users.put(user)
   }
@@ -88,21 +90,21 @@ class ReduxUsersDb implements UsersDb {
   async add(user: MetadataCache) {
     const state = store.getState()
     const { users } = state.users
-    store.dispatch(setUsers({...users, [user.pubkey]: user }))
+    store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
   }
 
 
   async put(user: MetadataCache) {
     const state = store.getState()
     const { users } = state.users
-    store.dispatch(setUsers({...users, [user.pubkey]: user }))
+    store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
   }
 
   async bulkAdd(newUserProfiles: MetadataCache[]) {
     const state = store.getState()
     const { users } = state.users
     const newUsers = newUserProfiles.reduce(groupByPubkey, {})
-    store.dispatch(setUsers({...users, ...newUsers }))
+    store.dispatch(setUsers({ ...users, ...newUsers }))
   }
 
   async bulkGet(keys: HexKey[]) {
@@ -118,8 +120,8 @@ class ReduxUsersDb implements UsersDb {
     const state = store.getState()
     const { users } = state.users
     const current = users[key]
-    const updated = {...current, ...fields }
+    const updated = { ...current, ...fields }
-    store.dispatch(setUsers({...users, [key]: updated }))
+    store.dispatch(setUsers({ ...users, [key]: updated }))
   }
 
   async bulkPut(newUsers: MetadataCache[]) {
@@ -138,6 +140,7 @@ let db: UsersDb = inMemoryDb
 indexedDb.isAvailable().then((available) => {
   if (available) {
     console.debug('Using Indexed DB')
+    indexedDb.ready = true;
     db = indexedDb;
   } else {
     console.debug('Using in-memory DB')

src/State/Users/Hooks.ts

@@ -10,10 +10,10 @@ export function useQuery(query: string, limit: number = 5) {
 
   const allUsers = useLiveQuery(
     () => db.query(query)
-          .catch((err) => {
+      .catch((err) => {
-            console.error(err)
+        console.error(err)
-            return inMemoryDb.query(query)
+        return inMemoryDb.query(query)
-          }),
+      }),
     [query],
   )
 
@@ -26,14 +26,14 @@ export function useKey(pubKey: HexKey) {
   const defaultUser = users[pubKey]
 
   const user = useLiveQuery(async () => {
-      if (pubKey) {
+    if (pubKey) {
-        try {
+      try {
-          return await db.find(pubKey);
+        return await db.find(pubKey);
-        } catch (error) {
+      } catch (error) {
-          console.error(error)
+        console.error(error)
-          return defaultUser
+        return defaultUser
-        }
+      }
-      } 
+    }
   }, [pubKey, defaultUser]);
 
   return user
@@ -42,17 +42,17 @@ export function useKey(pubKey: HexKey) {
 export function useKeys(pubKeys: HexKey[]): Map<HexKey, MetadataCache> {
   const db = getDb()
   const dbUsers = useLiveQuery(async () => {
-      if (pubKeys) {
+    if (pubKeys) {
-        try {
+      try {
-          const ret = await db.bulkGet(pubKeys);
+        const ret = await db.bulkGet(pubKeys);
-          return new Map(ret.map(a => [a.pubkey, a]))
+        return new Map(ret.map(a => [a.pubkey, a]))
-        } catch (error) {
+      } catch (error) {
-          console.error(error)
+        console.error(error)
-          const ret = await inMemoryDb.bulkGet(pubKeys);
+        const ret = await inMemoryDb.bulkGet(pubKeys);
-          return new Map(ret.map(a => [a.pubkey, a]))
+        return new Map(ret.map(a => [a.pubkey, a]))
-        }
       }
-      return new Map()
+    }
+    return new Map()
   }, [pubKeys]);
 
   return dbUsers!