Merge pull request #184 from v0l/feed-cache

feed cache

src/Db/index.ts

@@ -1,23 +1,35 @@
 import Dexie, { Table } from "dexie";
+import { TaggedRawEvent, u256 } from "Nostr";
 import { MetadataCache } from "State/Users";
 import { hexToBech32 } from "Util";
 
 export const NAME = 'snortDB'
-export const VERSION = 2
+export const VERSION = 3
+
+export interface SubCache {
+  id: string,
+  ids: u256[],
+  until?: number,
+  since?: number,
+}
 
 const STORES = {
-  users: '++pubkey, name, display_name, picture, nip05, npub'
+  users: '++pubkey, name, display_name, picture, nip05, npub',
+  events: '++id, pubkey, created_at',
+  feeds: '++id'
 }
 
 export class SnortDB extends Dexie {
   users!: Table<MetadataCache>;
+  events!: Table<TaggedRawEvent>;
+  feeds!: Table<SubCache>;
 
   constructor() {
     super(NAME);
-    this.version(VERSION).stores(STORES).upgrade(tx => {
+    this.version(VERSION).stores(STORES).upgrade(async tx => {
-      return tx.table("users").toCollection().modify(user => {
+      await tx.table("users").toCollection().modify(user => {
         user.npub = hexToBech32("npub", user.pubkey)
-      })
+      });
     });
   }
 }

src/Feed/EventPublisher.ts

@@ -78,8 +78,8 @@ export default function useEventPublisher() {
     }
 
     return {
-        nip42Auth: async (challenge: string, relay:string) => {
+        nip42Auth: async (challenge: string, relay: string) => {
-            if(pubKey) {
+            if (pubKey) {
                 const ev = NEvent.ForPubKey(pubKey);
                 ev.Kind = EventKind.Auth;
                 ev.Content = "";
@@ -112,17 +112,17 @@ export default function useEventPublisher() {
                 ev.Kind = EventKind.Lists;
                 ev.Tags.push(new Tag(["d", Lists.Muted], ev.Tags.length))
                 keys.forEach(p => {
-                  ev.Tags.push(new Tag(["p", p], ev.Tags.length))
+                    ev.Tags.push(new Tag(["p", p], ev.Tags.length))
                 })
                 let content = ""
                 if (priv.length > 0) {
-                  const ps = priv.map(p => ["p", p])
+                    const ps = priv.map(p => ["p", p])
-                  const plaintext = JSON.stringify(ps)
+                    const plaintext = JSON.stringify(ps)
-                  if (hasNip07 && !privKey) {
+                    if (hasNip07 && !privKey) {
-                      content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
+                        content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
-                  } else if (privKey) {
+                    } else if (privKey) {
-                      content = await ev.EncryptData(plaintext, pubKey, privKey)
+                        content = await ev.EncryptData(plaintext, pubKey, privKey)
-                  }
+                    }
                 }
                 ev.Content = content;
                 return await signEvent(ev);

src/Feed/FollowersFeed.ts

@@ -7,7 +7,7 @@ import useSubscription from "Feed/Subscription";
 export default function useFollowersFeed(pubkey: HexKey) {
     const sub = useMemo(() => {
         let x = new Subscriptions();
-        x.Id = "followers";
+        x.Id = `followers:${pubkey.slice(0, 12)}`;
         x.Kinds = new Set([EventKind.ContactList]);
         x.PTags = new Set([pubkey]);
 

src/Feed/FollowsFeed.ts

@@ -7,7 +7,7 @@ import useSubscription, { NoteStore } from "Feed/Subscription";
 export default function useFollowsFeed(pubkey: HexKey) {
     const sub = useMemo(() => {
         let x = new Subscriptions();
-        x.Id = "follows";
+        x.Id = `follows:${pubkey.slice(0, 12)}`;
         x.Kinds = new Set([EventKind.ContactList]);
         x.Authors = new Set([pubkey]);
 

src/Feed/LoginFeed.ts

@@ -8,151 +8,149 @@ import Event from "Nostr/Event";
 import { Subscriptions } from "Nostr/Subscriptions";
 import { addDirectMessage, setFollows, setRelays, setMuted, setBlocked, sendNotification } from "State/Login";
 import { RootState } from "State/Store";
-import { mapEventToProfile, MetadataCache  } from "State/Users";
+import { mapEventToProfile, MetadataCache } from "State/Users";
-import { getDb } from "State/Users/Db";
+import { useDb } from "State/Users/Db";
 import useSubscription from "Feed/Subscription";
-import { getDisplayName } from "Element/ProfileImage";
 import { barierNip07 } from "Feed/EventPublisher";
 import { getMutedKeys, getNewest } from "Feed/MuteList";
-import { MentionRegex } from "Const";
 import useModeration from "Hooks/useModeration";
 
 /**
  * Managed loading data for the current logged in user
  */
 export default function useLoginFeed() {
-    const dispatch = useDispatch();
+  const dispatch = useDispatch();
-    const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login);
+  const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login);
-    const { isMuted } = useModeration();
+  const { isMuted } = useModeration();
+  const db = useDb();
 
-    const subMetadata = useMemo(() => {
+  const subMetadata = useMemo(() => {
-        if (!pubKey) return null;
+    if (!pubKey) return null;
 
-        let sub = new Subscriptions();
+    let sub = new Subscriptions();
-        sub.Id = `login:meta`;
+    sub.Id = `login:meta`;
-        sub.Authors = new Set([pubKey]);
+    sub.Authors = new Set([pubKey]);
-        sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]);
+    sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]);
-        sub.Limit = 2
+    sub.Limit = 2
 
-        return sub;
+    return sub;
-    }, [pubKey]);
+  }, [pubKey]);
 
-    const subNotification = useMemo(() => {
+  const subNotification = useMemo(() => {
-        if (!pubKey) return null;
+    if (!pubKey) return null;
 
-        let sub = new Subscriptions();
+    let sub = new Subscriptions();
-        sub.Id = "login:notifications";
+    sub.Id = "login:notifications";
-        sub.Kinds = new Set([EventKind.TextNote]);
+    sub.Kinds = new Set([EventKind.TextNote]);
-        sub.PTags = new Set([pubKey]);
+    sub.PTags = new Set([pubKey]);
-        sub.Limit = 1;
+    sub.Limit = 1;
-        return sub;
+    return sub;
-    }, [pubKey]);
+  }, [pubKey]);
 
-    const subMuted = useMemo(() => {
+  const subMuted = useMemo(() => {
-        if (!pubKey) return null;
+    if (!pubKey) return null;
 
-        let sub = new Subscriptions();
+    let sub = new Subscriptions();
-        sub.Id = "login:muted";
+    sub.Id = "login:muted";
-        sub.Kinds = new Set([EventKind.Lists]);
+    sub.Kinds = new Set([EventKind.Lists]);
-        sub.Authors = new Set([pubKey]);
+    sub.Authors = new Set([pubKey]);
-        sub.DTags = new Set([Lists.Muted]);
+    sub.DTags = new Set([Lists.Muted]);
-        sub.Limit = 1;
+    sub.Limit = 1;
 
-        return sub;
+    return sub;
-    }, [pubKey]);
+  }, [pubKey]);
 
-    const subDms = useMemo(() => {
+  const subDms = useMemo(() => {
-        if (!pubKey) return null;
+    if (!pubKey) return null;
 
-        let dms = new Subscriptions();
+    let dms = new Subscriptions();
-        dms.Id = "login:dms";
+    dms.Id = "login:dms";
-        dms.Kinds = new Set([EventKind.DirectMessage]);
+    dms.Kinds = new Set([EventKind.DirectMessage]);
-        dms.PTags = new Set([pubKey]);
+    dms.PTags = new Set([pubKey]);
 
-        let dmsFromME = new Subscriptions();
+    let dmsFromME = new Subscriptions();
-        dmsFromME.Authors = new Set([pubKey]);
+    dmsFromME.Authors = new Set([pubKey]);
-        dmsFromME.Kinds = new Set([EventKind.DirectMessage]);
+    dmsFromME.Kinds = new Set([EventKind.DirectMessage]);
-        dms.AddSubscription(dmsFromME);
+    dms.AddSubscription(dmsFromME);
 
-        return dms;
+    return dms;
-    }, [pubKey]);
+  }, [pubKey]);
 
-    const metadataFeed = useSubscription(subMetadata, { leaveOpen: true });
+  const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true });
-    const notificationFeed = useSubscription(subNotification, { leaveOpen: true });
+  const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true });
-    const dmsFeed = useSubscription(subDms, { leaveOpen: true });
+  const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true });
-    const mutedFeed = useSubscription(subMuted, { leaveOpen: true });
+  const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true });
 
-    useEffect(() => {
+  useEffect(() => {
-        let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);
+    let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);
-        let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata);
+    let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata);
-        let profiles = metadata.map(a => mapEventToProfile(a))
+    let profiles = metadata.map(a => mapEventToProfile(a))
-            .filter(a => a !== undefined)
+      .filter(a => a !== undefined)
-            .map(a => a!);
+      .map(a => a!);
 
-        for (let cl of contactList) {
+    for (let cl of contactList) {
-            if (cl.content !== "" && cl.content !== "{}") {
+      if (cl.content !== "" && cl.content !== "{}") {
-                let relays = JSON.parse(cl.content);
+        let relays = JSON.parse(cl.content);
-                dispatch(setRelays({ relays, createdAt: cl.created_at }));
+        dispatch(setRelays({ relays, createdAt: cl.created_at }));
-            }
-            let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]);
-            dispatch(setFollows({ keys: pTags, createdAt: cl.created_at }));
-        }
-
-        (async () => {
-            let maxProfile = profiles.reduce((acc, v) => {
-                if (v.created > acc.created) {
-                    acc.profile = v;
-                    acc.created = v.created;
-                }
-                return acc;
-            }, { created: 0, profile: null as MetadataCache | null });
-            if (maxProfile.profile) {
-                const db = getDb()
-                let existing = await db.find(maxProfile.profile.pubkey);
-                if ((existing?.created ?? 0) < maxProfile.created) {
-                    await db.put(maxProfile.profile);
-                }
-            }
-        })().catch(console.warn);
-    }, [dispatch, metadataFeed.store]);
-
-    useEffect(() => {
-        const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey))
-        replies.forEach(nx => {
-          makeNotification(nx).then(notification => {
-            if (notification) {
-              // @ts-ignore
-              dispatch(sendNotification(notification))
-            }
-          })
-        })
-    }, [dispatch, notificationFeed.store]);
-
-    useEffect(() => {
-      const muted = getMutedKeys(mutedFeed.store.notes)
-      dispatch(setMuted(muted))
-
-      const newest = getNewest(mutedFeed.store.notes)
-      if (newest && newest.content.length > 0 && pubKey) {
-        decryptBlocked(newest, pubKey, privKey).then((plaintext) => {
-          try {
-            const blocked = JSON.parse(plaintext)
-            const keys = blocked.filter((p:any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1])
-            dispatch(setBlocked({
-              keys,
-              createdAt: newest.created_at,
-            }))
-          } catch(error) {
-            console.debug("Couldn't parse JSON")
-          }
-        }).catch((error) => console.warn(error))
       }
-    }, [dispatch, mutedFeed.store])
+      let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]);
+      dispatch(setFollows({ keys: pTags, createdAt: cl.created_at }));
+    }
 
-    useEffect(() => {
+    (async () => {
-        let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage);
+      let maxProfile = profiles.reduce((acc, v) => {
-        dispatch(addDirectMessage(dms));
+        if (v.created > acc.created) {
-    }, [dispatch, dmsFeed.store]);
+          acc.profile = v;
+          acc.created = v.created;
+        }
+        return acc;
+      }, { created: 0, profile: null as MetadataCache | null });
+      if (maxProfile.profile) {
+        let existing = await db.find(maxProfile.profile.pubkey);
+        if ((existing?.created ?? 0) < maxProfile.created) {
+          await db.put(maxProfile.profile);
+        }
+      }
+    })().catch(console.warn);
+  }, [dispatch, metadataFeed.store, db]);
+
+  useEffect(() => {
+    const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey))
+    replies.forEach(nx => {
+      makeNotification(db, nx).then(notification => {
+        if (notification) {
+          // @ts-ignore
+          dispatch(sendNotification(notification))
+        }
+      })
+    })
+  }, [dispatch, notificationFeed.store, db]);
+
+  useEffect(() => {
+    const muted = getMutedKeys(mutedFeed.store.notes)
+    dispatch(setMuted(muted))
+
+    const newest = getNewest(mutedFeed.store.notes)
+    if (newest && newest.content.length > 0 && pubKey) {
+      decryptBlocked(newest, pubKey, privKey).then((plaintext) => {
+        try {
+          const blocked = JSON.parse(plaintext)
+          const keys = blocked.filter((p: any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1])
+          dispatch(setBlocked({
+            keys,
+            createdAt: newest.created_at,
+          }))
+        } catch (error) {
+          console.debug("Couldn't parse JSON")
+        }
+      }).catch((error) => console.warn(error))
+    }
+  }, [dispatch, mutedFeed.store])
+
+  useEffect(() => {
+    let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage);
+    dispatch(addDirectMessage(dms));
+  }, [dispatch, dmsFeed.store]);
 }
 
 

src/Feed/ProfileFeed.ts

@@ -1,6 +1,4 @@
-import { useLiveQuery } from "dexie-react-hooks";
+import { useEffect } from "react";
-import { useEffect, useMemo } from "react";
-import { RootState } from "State/Store";
 import { MetadataCache } from "State/Users";
 import { useKey, useKeys } from "State/Users/Hooks";
 import { HexKey } from "Nostr";

src/Feed/Subscription.ts

@@ -3,6 +3,7 @@ import { System } from "Nostr/System";
 import { TaggedRawEvent } from "Nostr";
 import { Subscriptions } from "Nostr/Subscriptions";
 import { debounce } from "Util";
+import { db } from "Db";
 
 export type NoteStore = {
     notes: Array<TaggedRawEvent>,
@@ -10,7 +11,8 @@ export type NoteStore = {
 };
 
 export type UseSubscriptionOptions = {
-    leaveOpen: boolean
+    leaveOpen: boolean,
+    cache: boolean
 }
 
 interface ReducerArg {
@@ -77,33 +79,49 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
     const [state, dispatch] = useReducer(notesReducer, initStore);
     const [debounceOutput, setDebounceOutput] = useState<number>(0);
     const [subDebounce, setSubDebounced] = useState<Subscriptions>();
+    const useCache = useMemo(() => options?.cache === true, [options]);
 
     useEffect(() => {
         if (sub) {
             return debounce(DebounceMs, () => {
-                dispatch({
-                    type: "END",
-                    end: false
-                });
                 setSubDebounced(sub);
             });
         }
     }, [sub, options]);
 
     useEffect(() => {
-        if (sub) {
+        if (subDebounce) {
-            sub.OnEvent = (e) => {
+            dispatch({
+                type: "END",
+                end: false
+            });
+
+            if (useCache) {
+                // preload notes from db
+                PreloadNotes(subDebounce.Id)
+                    .then(ev => {
+                        dispatch({
+                            type: "EVENT",
+                            ev: ev
+                        });
+                    })
+                    .catch(console.warn);
+            }
+            subDebounce.OnEvent = (e) => {
                 dispatch({
                     type: "EVENT",
                     ev: e
                 });
+                if (useCache) {
+                    db.events.put(e);
+                }
             };
 
-            sub.OnEnd = (c) => {
+            subDebounce.OnEnd = (c) => {
                 if (!(options?.leaveOpen ?? false)) {
-                    c.RemoveSubscription(sub.Id);
+                    c.RemoveSubscription(subDebounce.Id);
-                    if (sub.IsFinished()) {
+                    if (subDebounce.IsFinished()) {
-                        System.RemoveSubscription(sub.Id);
+                        System.RemoveSubscription(subDebounce.Id);
                     }
                 }
                 dispatch({
@@ -112,14 +130,23 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
                 });
             };
 
-            console.debug("Adding sub: ", sub.ToObject());
+            console.debug("Adding sub: ", subDebounce.ToObject());
-            System.AddSubscription(sub);
+            System.AddSubscription(subDebounce);
             return () => {
-                console.debug("Removing sub: ", sub.ToObject());
+                console.debug("Removing sub: ", subDebounce.ToObject());
-                System.RemoveSubscription(sub.Id);
+                System.RemoveSubscription(subDebounce.Id);
             };
         }
-    }, [subDebounce]);
+    }, [subDebounce, useCache]);
+
+    useEffect(() => {
+        if (subDebounce && useCache) {
+            return debounce(500, () => {
+                TrackNotesInFeed(subDebounce.Id, state.notes)
+                    .catch(console.warn);
+            });
+        }
+    }, [state, useCache]);
 
     useEffect(() => {
         return debounce(DebounceMs, () => {
@@ -140,4 +167,24 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
             });
         }
     }
+}
+
+/**
+ * Lookup cached copy of feed
+ */
+const PreloadNotes = async (id: string): Promise<TaggedRawEvent[]> => {
+    const feed = await db.feeds.get(id);
+    if (feed) {
+        const events = await db.events.bulkGet(feed.ids);
+        return events.filter(a => a !== undefined).map(a => a!);
+    }
+    return [];
+}
+
+const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => {
+    const existing = await db.feeds.get(id);
+    const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)]));
+    const since = notes.reduce((acc, v) => acc > v.created_at ? v.created_at : acc, +Infinity);
+    const until = notes.reduce((acc, v) => acc < v.created_at ? v.created_at : acc, -Infinity);
+    await db.feeds.put({ id, ids, since, until });
 }

src/Feed/ThreadFeed.ts

@@ -38,7 +38,7 @@ export default function useThreadFeed(id: u256) {
         return thisSub;
     }, [trackingEvents, pref, id]);
 
-    const main = useSubscription(sub, { leaveOpen: true });
+    const main = useSubscription(sub, { leaveOpen: true, cache: true });
 
     useEffect(() => {
         if (main.store) {

src/Feed/TimelineFeed.ts

@@ -14,6 +14,7 @@ export interface TimelineFeedOptions {
 
 export interface TimelineSubject {
     type: "pubkey" | "hashtag" | "global" | "ptag" | "keyword",
+    discriminator: string,
     items: string[]
 }
 
@@ -32,7 +33,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         }
 
         let sub = new Subscriptions();
-        sub.Id = `timeline:${subject.type}`;
+        sub.Id = `timeline:${subject.type}:${subject.discriminator}`;
         sub.Kinds = new Set([EventKind.TextNote, EventKind.Repost]);
         switch (subject.type) {
             case "pubkey": {
@@ -54,7 +55,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
             }
         }
         return sub;
-    }, [subject.type, subject.items]);
+    }, [subject.type, subject.items, subject.discriminator]);
 
     const sub = useMemo(() => {
         let sub = createSub();
@@ -86,7 +87,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return sub;
     }, [until, since, options.method, pref, createSub]);
 
-    const main = useSubscription(sub, { leaveOpen: true });
+    const main = useSubscription(sub, { leaveOpen: true, cache: true });
 
     const subRealtime = useMemo(() => {
         let subLatest = createSub();
@@ -98,7 +99,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return subLatest;
     }, [pref, createSub]);
 
-    const latest = useSubscription(subRealtime, { leaveOpen: true });
+    const latest = useSubscription(subRealtime, { leaveOpen: true, cache: false });
 
     const subNext = useMemo(() => {
         let sub: Subscriptions | undefined;
@@ -111,7 +112,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
         return sub ?? null;
     }, [trackingEvents, pref, subject.type]);
 
-    const others = useSubscription(subNext, { leaveOpen: true });
+    const others = useSubscription(subNext, { leaveOpen: true, cache: true });
 
     const subParents = useMemo(() => {
         if (trackingParentEvents.length > 0) {

src/Nostr/System.ts

@@ -1,7 +1,6 @@
 import { HexKey, TaggedRawEvent } from "Nostr";
-import { getDb } from "State/Users/Db";
 import { ProfileCacheExpire } from "Const";
-import { mapEventToProfile, MetadataCache } from "State/Users";
+import { mapEventToProfile, MetadataCache, UsersDb } from "State/Users";
 import Connection, { RelaySettings } from "Nostr/Connection";
 import Event from "Nostr/Event";
 import EventKind from "Nostr/EventKind";
@@ -31,6 +30,11 @@ export class NostrSystem {
      */
     WantsMetadata: Set<HexKey>;
 
+    /**
+     * User db store
+     */
+    UserDb?: UsersDb;
+
     constructor() {
         this.Sockets = new Map();
         this.Subscriptions = new Map();
@@ -166,54 +170,54 @@ export class NostrSystem {
     }
 
     async _FetchMetadata() {
-        let missing = new Set<HexKey>();
+        if (this.UserDb) {
-        const db = getDb()
+            let missing = new Set<HexKey>();
-        let meta = await db.bulkGet(Array.from(this.WantsMetadata));
+            let meta = await this.UserDb.bulkGet(Array.from(this.WantsMetadata));
-        let expire = new Date().getTime() - ProfileCacheExpire;
+            let expire = new Date().getTime() - ProfileCacheExpire;
-        for (let pk of this.WantsMetadata) {
+            for (let pk of this.WantsMetadata) {
-            let m = meta.find(a => a?.pubkey === pk);
+                let m = meta.find(a => a?.pubkey === pk);
-            if (!m || m.loaded < expire) {
+                if (!m || m.loaded < expire) {
-                missing.add(pk);
+                    missing.add(pk);
-                // cap 100 missing profiles
+                    // cap 100 missing profiles
-                if (missing.size >= 100) {
+                    if (missing.size >= 100) {
-                    break;
+                        break;
-                }
-            }
-        }
-
-        if (missing.size > 0) {
-            console.debug("Wants profiles: ", missing);
-
-            let sub = new Subscriptions();
-            sub.Id = `profiles:${sub.Id}`;
-            sub.Kinds = new Set([EventKind.SetMetadata]);
-            sub.Authors = missing;
-            sub.OnEvent = async (e) => {
-                let profile = mapEventToProfile(e);
-                if (profile) {
-                    let existing = await db.find(profile.pubkey);
-                    if ((existing?.created ?? 0) < profile.created) {
-                        await db.put(profile);
-                    } else if (existing) {
-                        await db.update(profile.pubkey, { loaded: new Date().getTime() });
                     }
                 }
             }
-            let results = await this.RequestSubscription(sub);
-            let couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a));
-            console.debug("No profiles: ", couldNotFetch);
-            await db.bulkPut(couldNotFetch.map(a => {
-                return {
-                    pubkey: a,
-                    loaded: new Date().getTime()
-                } as MetadataCache;
-            }));
-        }
 
+            if (missing.size > 0) {
+                console.debug("Wants profiles: ", missing);
+
+                let sub = new Subscriptions();
+                sub.Id = `profiles:${sub.Id}`;
+                sub.Kinds = new Set([EventKind.SetMetadata]);
+                sub.Authors = missing;
+                sub.OnEvent = async (e) => {
+                    let profile = mapEventToProfile(e);
+                    if (profile) {
+                        let existing = await this.UserDb!.find(profile.pubkey);
+                        if ((existing?.created ?? 0) < profile.created) {
+                            await this.UserDb!.put(profile);
+                        } else if (existing) {
+                            await this.UserDb!.update(profile.pubkey, { loaded: new Date().getTime() });
+                        }
+                    }
+                }
+                let results = await this.RequestSubscription(sub);
+                let couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a));
+                console.debug("No profiles: ", couldNotFetch);
+                await this.UserDb!.bulkPut(couldNotFetch.map(a => {
+                    return {
+                        pubkey: a,
+                        loaded: new Date().getTime()
+                    } as MetadataCache;
+                }));
+            }
+        }
         setTimeout(() => this._FetchMetadata(), 500);
     }
 
-    async nip42Auth(challenge: string, relay:string): Promise<Event|undefined> {
+    async nip42Auth(challenge: string, relay: string): Promise<Event | undefined> {
         return
     }
 }

src/Notifications.ts

@@ -3,13 +3,11 @@ import Nostrich from "nostrich.webp";
 import { TaggedRawEvent } from "Nostr";
 import EventKind from "Nostr/EventKind";
 import type { NotificationRequest } from "State/Login";
-import { MetadataCache } from "State/Users";
+import { MetadataCache, UsersDb } from "State/Users";
-import { getDb } from "State/Users/Db";
 import { getDisplayName } from "Element/ProfileImage";
 import { MentionRegex } from "Const";
 
-export async function makeNotification(ev: TaggedRawEvent): Promise<NotificationRequest | null> {
+export async function makeNotification(db: UsersDb, ev: TaggedRawEvent): Promise<NotificationRequest | null> {
-    const db = getDb()
     switch (ev.kind) {
         case EventKind.TextNote: {
             const pubkeys = new Set([ev.pubkey, ...ev.tags.filter(a => a[0] === "p").map(a => a[1]!)]);

src/Pages/HashTagsPage.tsx

@@ -8,7 +8,7 @@ const HashTagsPage = () => {
     return (
         <>
             <h2>#{tag}</h2>
-            <Timeline key={tag} subject={{ type: "hashtag", items: [tag] }} postsOnly={false} method={"TIME_RANGE"} />
+            <Timeline key={tag} subject={{ type: "hashtag", items: [tag], discriminator: tag }} postsOnly={false} method={"TIME_RANGE"} />
         </>
     )
 }

src/Pages/Layout.tsx

@@ -17,6 +17,8 @@ import { totalUnread } from "Pages/MessagesPage";
 import { SearchRelays } from 'Const';
 import useEventPublisher from "Feed/EventPublisher";
 import useModeration from "Hooks/useModeration";
+import { IndexedUDB, useDb } from "State/Users/Db";
+import { db } from "Db";
 
 
 export default function Layout() {
@@ -31,12 +33,17 @@ export default function Layout() {
     const { isMuted } = useModeration();
     const filteredDms = dms.filter(a => !isMuted(a.pubkey))
     const prefs = useSelector<RootState, UserPreferences>(s => s.login.preferences);
+    const usingDb = useDb();
     const pub = useEventPublisher();
     useLoginFeed();
 
     useEffect(() => {
         System.nip42Auth = pub.nip42Auth
-    },[pub])
+    }, [pub])
+
+    useEffect(() => {
+        System.UserDb = usingDb;
+    }, [usingDb])
 
     useEffect(() => {
         if (relays) {
@@ -73,7 +80,28 @@ export default function Layout() {
     }, [prefs.theme]);
 
     useEffect(() => {
-        dispatch(init());
+        // check DB support then init
+        IndexedUDB.isAvailable()
+            .then(async a => {
+                const dbType = a ? "indexdDb" : "redux";
+
+                // cleanup on load
+                if (dbType === "indexdDb") {
+                    await db.feeds.clear();
+                    const now = Math.floor(new Date().getTime() / 1000);
+
+                    const cleanupEvents = await db.events
+                        .where("created_at")
+                        .above(now - (60 * 60))
+                        .primaryKeys();
+                    console.debug(`Cleanup ${cleanupEvents.length} events`);
+                    await db.events.bulkDelete(cleanupEvents)
+                }
+
+                console.debug(`Using db: ${dbType}`);
+                dispatch(init(dbType));
+            })
+
     }, []);
 
     async function goToNotifications(e: any) {

src/Pages/Notifications.tsx

@@ -17,7 +17,7 @@ export default function NotificationsPage() {
     return (
         <>
             {pubkey ?
-                <Timeline subject={{ type: "ptag", items: [pubkey!] }} postsOnly={false} method={"TIME_RANGE"} />
+                <Timeline subject={{ type: "ptag", items: [pubkey!], discriminator: pubkey!.slice(0, 12) }} postsOnly={false} method={"TIME_RANGE"} />
                 : null}
         </>
     )

src/Pages/ProfilePage.tsx

@@ -105,7 +105,7 @@ export default function ProfilePage() {
   function tabContent() {
     switch (tab) {
       case ProfileTab.Notes:
-        return <Timeline key={id} subject={{ type: "pubkey", items: [id] }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
+        return <Timeline key={id} subject={{ type: "pubkey", items: [id], discriminator: id.slice(0, 12) }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
       case ProfileTab.Follows: {
         if (isMe) {
           return (
@@ -195,7 +195,7 @@ export default function ProfilePage() {
   return (
     <>
       <div className="profile flex">
-        {user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w}/>}
+        {user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w} />}
         <div className="profile-wrapper flex">
           {avatar()}
           {userDetails()}

src/Pages/Root.tsx

@@ -29,7 +29,7 @@ export default function RootPage() {
     }
 
     const isGlobal = loggedOut || tab === RootTab.Global;
-    const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [] } : { type: "pubkey", items: follows };
+    const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [], discriminator: "all" } : { type: "pubkey", items: follows, discriminator: "follows" };
     return (
         <>
             {pubKey ? <>

src/Pages/SearchPage.tsx

@@ -43,7 +43,7 @@ const SearchPage = () => {
             <div className="flex mb10">
                 <input type="text" className="f-grow mr10" placeholder="Search.." value={search} onChange={e => setSearch(e.target.value)} />
             </div>
-            {keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword] }} postsOnly={false} method={"LIMIT_UNTIL"} />}
+            {keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword], discriminator: keyword }} postsOnly={false} method={"LIMIT_UNTIL"} />}
         </>
     )
 }

src/State/Login.ts

@@ -61,8 +61,14 @@ export interface UserPreferences {
      */
     imgProxyConfig: ImgProxySettings | null
 }
+export type DbType = "indexdDb" | "redux";
 
 export interface LoginStore {
+    /**
+     * Which db we will use to cache data
+     */
+    useDb: DbType,
+
     /**
      * If there is no login
      */
@@ -146,6 +152,7 @@ const DefaultImgProxy = {
 };
 
 export const InitState = {
+    useDb: "redux",
     loggedOut: undefined,
     publicKey: undefined,
     privateKey: undefined,
@@ -186,7 +193,8 @@ const LoginSlice = createSlice({
     name: "Login",
     initialState: InitState,
     reducers: {
-        init: (state) => {
+        init: (state, action: PayloadAction<DbType>) => {
+            state.useDb = action.payload;
             state.privateKey = window.localStorage.getItem(PrivateKeyItem) ?? undefined;
             if (state.privateKey) {
                 window.localStorage.removeItem(PublicKeyItem); // reset nip07 if using private key

src/State/Users.ts

@@ -35,20 +35,20 @@ export function mapEventToProfile(ev: TaggedRawEvent) {
             ...data
         } as MetadataCache;
     } catch (e) {
-      console.error("Failed to parse JSON", ev, e);
+        console.error("Failed to parse JSON", ev, e);
     }
 }
 
 export interface UsersDb {
-  isAvailable(): Promise<boolean>
+    isAvailable(): Promise<boolean>
-  query(str: string): Promise<MetadataCache[]>
+    query(str: string): Promise<MetadataCache[]>
-  find(key: HexKey): Promise<MetadataCache | undefined>
+    find(key: HexKey): Promise<MetadataCache | undefined>
-  add(user: MetadataCache): Promise<any>
+    add(user: MetadataCache): Promise<any>
-  put(user: MetadataCache): Promise<any>
+    put(user: MetadataCache): Promise<any>
-  bulkAdd(users: MetadataCache[]): Promise<any>
+    bulkAdd(users: MetadataCache[]): Promise<any>
-  bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
+    bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
-  bulkPut(users: MetadataCache[]): Promise<any>
+    bulkPut(users: MetadataCache[]): Promise<any>
-  update(key: HexKey, fields: Record<string, any>): Promise<any>
+    update(key: HexKey, fields: Record<string, any>): Promise<any>
 }
 
 export interface UsersStore {
@@ -65,7 +65,7 @@ const UsersSlice = createSlice({
     initialState: InitState,
     reducers: {
         setUsers(state, action: PayloadAction<Record<HexKey, MetadataCache>>) {
-          state.users = action.payload
+            state.users = action.payload
         }
     }
 });

src/State/Users/Db.ts

@@ -2,9 +2,12 @@ import { HexKey } from "Nostr";
 import { db as idb } from "Db";
 
 import { UsersDb, MetadataCache, setUsers } from "State/Users";
-import store from "State/Store";
+import store, { RootState } from "State/Store";
+import { useSelector } from "react-redux";
+
+class IndexedUsersDb implements UsersDb {
+  ready: boolean = false;
 
-class IndexedDb implements UsersDb {
   isAvailable() {
     if ("indexedDB" in window) {
       return new Promise<boolean>((resolve) => {
@@ -26,12 +29,12 @@ class IndexedDb implements UsersDb {
 
   query(q: string) {
     return idb.users
-            .where("npub").startsWithIgnoreCase(q)
+      .where("npub").startsWithIgnoreCase(q)
-            .or("name").startsWithIgnoreCase(q)
+      .or("name").startsWithIgnoreCase(q)
-            .or("display_name").startsWithIgnoreCase(q)
+      .or("display_name").startsWithIgnoreCase(q)
-            .or("nip05").startsWithIgnoreCase(q)
+      .or("nip05").startsWithIgnoreCase(q)
-            .limit(5)
+      .limit(5)
-            .toArray()
+      .toArray()
   }
 
   bulkGet(keys: HexKey[]) {
@@ -41,7 +44,7 @@ class IndexedDb implements UsersDb {
   add(user: MetadataCache) {
     return idb.users.add(user)
   }
-  
+
   put(user: MetadataCache) {
     return idb.users.put(user)
   }
@@ -88,21 +91,21 @@ class ReduxUsersDb implements UsersDb {
   async add(user: MetadataCache) {
     const state = store.getState()
     const { users } = state.users
-    store.dispatch(setUsers({...users, [user.pubkey]: user }))
+    store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
   }
 
 
   async put(user: MetadataCache) {
     const state = store.getState()
     const { users } = state.users
-    store.dispatch(setUsers({...users, [user.pubkey]: user }))
+    store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
   }
 
   async bulkAdd(newUserProfiles: MetadataCache[]) {
     const state = store.getState()
     const { users } = state.users
     const newUsers = newUserProfiles.reduce(groupByPubkey, {})
-    store.dispatch(setUsers({...users, ...newUsers }))
+    store.dispatch(setUsers({ ...users, ...newUsers }))
   }
 
   async bulkGet(keys: HexKey[]) {
@@ -118,8 +121,8 @@ class ReduxUsersDb implements UsersDb {
     const state = store.getState()
     const { users } = state.users
     const current = users[key]
-    const updated = {...current, ...fields }
+    const updated = { ...current, ...fields }
-    store.dispatch(setUsers({...users, [key]: updated }))
+    store.dispatch(setUsers({ ...users, [key]: updated }))
   }
 
   async bulkPut(newUsers: MetadataCache[]) {
@@ -130,20 +133,13 @@ class ReduxUsersDb implements UsersDb {
   }
 }
 
+export const IndexedUDB = new IndexedUsersDb();
+export const ReduxUDB = new ReduxUsersDb();
 
-const indexedDb = new IndexedDb()
+export function useDb(): UsersDb {
-export const inMemoryDb = new ReduxUsersDb()
+  const db = useSelector((s: RootState) => s.login.useDb);
-
+  switch (db) {
-let db: UsersDb = inMemoryDb
+    case "indexdDb": return IndexedUDB
-indexedDb.isAvailable().then((available) => {
+    default: return ReduxUDB
-  if (available) {
-    console.debug('Using Indexed DB')
-    db = indexedDb;
-  } else {
-    console.debug('Using in-memory DB')
   }
-})
+}
-
-export function getDb() {
-  return db
-}

src/State/Users/Hooks.ts

@@ -1,59 +1,50 @@
 import { useSelector } from "react-redux"
 import { useLiveQuery } from "dexie-react-hooks";
 import { MetadataCache } from "State/Users";
-import { getDb, inMemoryDb } from "State/Users/Db";
 import type { RootState } from "State/Store"
 import { HexKey } from "Nostr";
+import { useDb } from "./Db";
 
 export function useQuery(query: string, limit: number = 5) {
-  const db = getDb()
+  const db = useDb()
-
+  return useLiveQuery(async () => db.query(query), [query],)
-  const allUsers = useLiveQuery(
-    () => db.query(query)
-          .catch((err) => {
-            console.error(err)
-            return inMemoryDb.query(query)
-          }),
-    [query],
-  )
-
-  return allUsers
 }
 
 export function useKey(pubKey: HexKey) {
-  const db = getDb()
+  const db = useDb()
   const { users } = useSelector((state: RootState) => state.users)
   const defaultUser = users[pubKey]
 
   const user = useLiveQuery(async () => {
-      if (pubKey) {
+    if (pubKey) {
-        try {
+      try {
-          return await db.find(pubKey);
+        return await db.find(pubKey);
-        } catch (error) {
+      } catch (error) {
-          console.error(error)
+        console.error(error)
-          return defaultUser
+        return defaultUser
-        }
+      }
-      } 
+    }
   }, [pubKey, defaultUser]);
 
   return user
 }
 
 export function useKeys(pubKeys: HexKey[]): Map<HexKey, MetadataCache> {
-  const db = getDb()
+  const db = useDb()
+  const { users } = useSelector((state: RootState) => state.users)
+
   const dbUsers = useLiveQuery(async () => {
-      if (pubKeys) {
+    if (pubKeys) {
-        try {
+      try {
-          const ret = await db.bulkGet(pubKeys);
+        const ret = await db.bulkGet(pubKeys);
-          return new Map(ret.map(a => [a.pubkey, a]))
+        return new Map(ret.map(a => [a.pubkey, a]))
-        } catch (error) {
+      } catch (error) {
-          console.error(error)
+        console.error(error)
-          const ret = await inMemoryDb.bulkGet(pubKeys);
+        return new Map(pubKeys.map(a => [a, users[a]]))
-          return new Map(ret.map(a => [a.pubkey, a]))
-        }
       }
-      return new Map()
+    }
-  }, [pubKeys]);
+    return new Map()
+  }, [pubKeys, users]);
 
   return dbUsers!
 }