Merge pull request #339 from v0l/feed-cache-rework

Feed cache rework

packages/app/src/Db/index.ts

@@ -1,10 +1,9 @@
 import Dexie, { Table } from "dexie";
-import { TaggedRawEvent, u256 } from "@snort/nostr";
+import { u256 } from "@snort/nostr";
 import { MetadataCache } from "State/Users";
-import { hexToBech32 } from "Util";
 
 export const NAME = "snortDB";
-export const VERSION = 3;
+export const VERSION = 4;
 
 export interface SubCache {
   id: string;
@@ -15,27 +14,14 @@ export interface SubCache {
 
 const STORES = {
   users: "++pubkey, name, display_name, picture, nip05, npub",
-  events: "++id, pubkey, created_at",
-  feeds: "++id",
 };
 
 export class SnortDB extends Dexie {
   users!: Table<MetadataCache>;
-  events!: Table<TaggedRawEvent>;
-  feeds!: Table<SubCache>;
 
   constructor() {
     super(NAME);
-    this.version(VERSION)
-      .stores(STORES)
-      .upgrade(async tx => {
-        await tx
-          .table("users")
-          .toCollection()
-          .modify(user => {
-            user.npub = hexToBech32("npub", user.pubkey);
-          });
-      });
+    this.version(VERSION).stores(STORES);
   }
 }
 

packages/app/src/Element/Note.tsx

@@ -168,7 +168,7 @@ export default function Note(props: NoteProps) {
 
   useLayoutEffect(() => {
     if (entry && inView && extendable === false) {
-      const h = entry?.target.clientHeight ?? 0;
+      const h = (entry?.target as HTMLDivElement)?.offsetHeight ?? 0;
       if (h > 650) {
         setExtendable(true);
       }

packages/app/src/Element/Timeline.tsx

@@ -1,10 +1,10 @@
 import "./Timeline.css";
 import { FormattedMessage } from "react-intl";
-import { useCallback, useMemo } from "react";
+import { useCallback, useEffect, useMemo } from "react";
 import { useInView } from "react-intersection-observer";
 
 import ArrowUp from "Icons/ArrowUp";
-import { dedupeByPubkey, tagFilterOfTextRepost } from "Util";
+import { dedupeById, dedupeByPubkey, tagFilterOfTextRepost } from "Util";
 import ProfileImage from "Element/ProfileImage";
 import useTimelineFeed, { TimelineSubject } from "Feed/TimelineFeed";
 import { TaggedRawEvent } from "@snort/nostr";
@@ -16,6 +16,9 @@ import NoteReaction from "Element/NoteReaction";
 import useModeration from "Hooks/useModeration";
 import ProfilePreview from "./ProfilePreview";
 import Skeleton from "Element/Skeleton";
+import { useDispatch, useSelector } from "react-redux";
+import { RootState } from "State/Store";
+import { setTimeline } from "State/Cache";
 
 export interface TimelineProps {
   postsOnly: boolean;
@@ -38,7 +41,9 @@ export default function Timeline({
   relay,
 }: TimelineProps) {
   const { muted, isMuted } = useModeration();
-  const { main, related, latest, parent, loadMore, showLatest } = useTimelineFeed(subject, {
+  const dispatch = useDispatch();
+  const cache = useSelector((s: RootState) => s.cache.timeline);
+  const feed = useTimelineFeed(subject, {
     method,
     window: timeWindow,
     relay,
@@ -52,20 +57,34 @@ export default function Timeline({
         ?.filter(a => (postsOnly ? !a.tags.some(b => b[0] === "e") : true))
         .filter(a => ignoreModeration || !isMuted(a.pubkey));
     },
-    [postsOnly, muted]
+    [postsOnly, muted, ignoreModeration]
   );
 
   const mainFeed = useMemo(() => {
-    return filterPosts(main.notes);
-  }, [main, filterPosts]);
+    return filterPosts(cache.main);
+  }, [cache, filterPosts]);
 
   const latestFeed = useMemo(() => {
-    return filterPosts(latest.notes).filter(a => !mainFeed.some(b => b.id === a.id));
-  }, [latest, mainFeed, filterPosts]);
+    return filterPosts(cache.latest).filter(a => !mainFeed.some(b => b.id === a.id));
+  }, [cache, filterPosts]);
   const latestAuthors = useMemo(() => {
     return dedupeByPubkey(latestFeed).map(e => e.pubkey);
   }, [latestFeed]);
 
+  useEffect(() => {
+    const key = `${subject.type}-${subject.discriminator}`;
+    const newFeed = key !== cache.key;
+    dispatch(
+      setTimeline({
+        key: key,
+        main: dedupeById([...(newFeed ? [] : cache.main), ...feed.main.notes]),
+        latest: [...feed.latest.notes],
+        related: dedupeById([...(newFeed ? [] : cache.related), ...feed.related.notes]),
+        parent: dedupeById([...(newFeed ? [] : cache.parent), ...feed.parent.notes]),
+      })
+    );
+  }, [feed.main, feed.latest, feed.related, feed.parent]);
+
   function eventElement(e: TaggedRawEvent) {
     switch (e.kind) {
       case EventKind.SetMetadata: {
@@ -74,9 +93,9 @@ export default function Timeline({
       case EventKind.TextNote: {
         const eRef = e.tags.find(tagFilterOfTextRepost(e))?.at(1);
         if (eRef) {
-          return <NoteReaction data={e} key={e.id} root={parent.notes.find(a => a.id === eRef)} />;
+          return <NoteReaction data={e} key={e.id} root={cache.parent.find(a => a.id === eRef)} />;
         }
-        return <Note key={e.id} data={e} related={related.notes} ignoreModeration={ignoreModeration} />;
+        return <Note key={e.id} data={e} related={cache.related} ignoreModeration={ignoreModeration} />;
       }
       case EventKind.ZapReceipt: {
         const zap = parseZap(e);
@@ -85,18 +104,17 @@ export default function Timeline({
       case EventKind.Reaction:
       case EventKind.Repost: {
         const eRef = e.tags.find(a => a[0] === "e")?.at(1);
-        return <NoteReaction data={e} key={e.id} root={parent.notes.find(a => a.id === eRef)} />;
+        return <NoteReaction data={e} key={e.id} root={cache.parent.find(a => a.id === eRef)} />;
       }
     }
   }
 
   function onShowLatest(scrollToTop = false) {
-    showLatest();
+    feed.showLatest();
     if (scrollToTop) {
       window.scrollTo(0, 0);
     }
   }
-
   return (
     <div className="main-content">
       {latestFeed.length > 0 && (
@@ -126,7 +144,7 @@ export default function Timeline({
         </>
       )}
       {mainFeed.map(eventElement)}
-      <LoadMore onLoadMore={loadMore} shouldLoadMore={main.end}>
+      <LoadMore onLoadMore={feed.loadMore} shouldLoadMore={feed.main.end}>
         <Skeleton width="100%" height="120px" margin="0 0 16px 0" />
         <Skeleton width="100%" height="120px" margin="0 0 16px 0" />
         <Skeleton width="100%" height="120px" margin="0 0 16px 0" />

packages/app/src/Feed/Subscription.ts

@@ -2,8 +2,7 @@ import { useEffect, useMemo, useReducer, useState } from "react";
 import { TaggedRawEvent } from "@snort/nostr";
 import { Subscriptions } from "@snort/nostr";
 import { System } from "System";
-import { debounce, unwrap } from "Util";
-import { db } from "Db";
+import { debounce } from "Util";
 
 export type NoteStore = {
   notes: Array<TaggedRawEvent>;
@@ -80,7 +79,6 @@ export default function useSubscription(
   const [state, dispatch] = useReducer(notesReducer, initStore);
   const [debounceOutput, setDebounceOutput] = useState<number>(0);
   const [subDebounce, setSubDebounced] = useState<Subscriptions>();
-  const useCache = useMemo(() => options?.cache === true, [options]);
 
   useEffect(() => {
     if (sub) {
@@ -97,25 +95,11 @@ export default function useSubscription(
         end: false,
       });
 
-      if (useCache) {
-        // preload notes from db
-        PreloadNotes(subDebounce.Id)
-          .then(ev => {
-            dispatch({
-              type: "EVENT",
-              ev: ev,
-            });
-          })
-          .catch(console.warn);
-      }
       subDebounce.OnEvent = e => {
         dispatch({
           type: "EVENT",
           ev: e,
         });
-        if (useCache) {
-          db.events.put(e);
-        }
       };
 
       subDebounce.OnEnd = c => {
@@ -144,15 +128,7 @@ export default function useSubscription(
         System.RemoveSubscription(subDebounce.Id);
       };
     }
-  }, [subDebounce, useCache]);
-
-  useEffect(() => {
-    if (subDebounce && useCache) {
-      return debounce(500, () => {
-        TrackNotesInFeed(subDebounce.Id, state.notes).catch(console.warn);
-      });
-    }
-  }, [state, useCache]);
+  }, [subDebounce]);
 
   useEffect(() => {
     return debounce(DebounceMs, () => {
@@ -174,23 +150,3 @@ export default function useSubscription(
     },
   };
 }
-
-/**
- * Lookup cached copy of feed
- */
-const PreloadNotes = async (id: string): Promise<TaggedRawEvent[]> => {
-  const feed = await db.feeds.get(id);
-  if (feed) {
-    const events = await db.events.bulkGet(feed.ids);
-    return events.filter(a => a !== undefined).map(a => unwrap(a));
-  }
-  return [];
-};
-
-const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => {
-  const existing = await db.feeds.get(id);
-  const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)]));
-  const since = notes.reduce((acc, v) => (acc > v.created_at ? v.created_at : acc), +Infinity);
-  const until = notes.reduce((acc, v) => (acc < v.created_at ? v.created_at : acc), -Infinity);
-  await db.feeds.put({ id, ids, since, until });
-};

packages/app/src/Pages/Layout.tsx

@@ -17,7 +17,6 @@ import { SearchRelays, SnortPubKey } from "Const";
 import useEventPublisher from "Feed/EventPublisher";
 import useModeration from "Hooks/useModeration";
 import { IndexedUDB } from "State/Users/Db";
-import { db } from "Db";
 import { bech32ToHex } from "Util";
 import { NoteCreator } from "Element/NoteCreator";
 import Plus from "Icons/Plus";
@@ -120,15 +119,6 @@ export default function Layout() {
       // cleanup on load
       if (dbType === "indexdDb") {
         IndexedUDB.ready = true;
-        await db.feeds.clear();
-        const now = Math.floor(new Date().getTime() / 1000);
-
-        const cleanupEvents = await db.events
-          .where("created_at")
-          .above(now - 60 * 60)
-          .primaryKeys();
-        console.debug(`Cleanup ${cleanupEvents.length} events`);
-        await db.events.bulkDelete(cleanupEvents);
       }
 
       console.debug(`Using db: ${dbType}`);

packages/app/src/Pages/Root.tsx

@@ -11,7 +11,7 @@ import { System } from "System";
 import { TimelineSubject } from "Feed/TimelineFeed";
 
 import messages from "./messages";
-import { debounce } from "Util";
+import { debounce, unwrap } from "Util";
 
 interface RelayOption {
   url: string;
@@ -46,11 +46,12 @@ export default function RootPage() {
     }
   });
   const [relay, setRelay] = useState<RelayOption>();
-  const [globalRelays, setGlobalRelays] = useState<RelayOption[]>([]);
+  const [allRelays, setAllRelays] = useState<RelayOption[]>();
   const tagTabs = tags.map((t, idx) => {
     return { text: `#${t}`, value: idx + 3 };
   });
   const tabs = [RootTab.Posts, RootTab.PostsAndReplies, RootTab.Global, ...tagTabs];
+  const isGlobal = loggedOut || tab.value === RootTab.Global.value;
 
   function followHints() {
     if (follows?.length === 0 && pubKey && tab !== RootTab.Global) {
@@ -69,51 +70,19 @@ export default function RootPage() {
     }
   }
 
-  useEffect(() => {
-    return debounce(1_000, () => {
-      const ret: RelayOption[] = [];
-      System.Sockets.forEach((v, k) => {
-        ret.push({
-          url: k,
-          paid: v.Info?.limitation?.payment_required ?? false,
-        });
-      });
-      ret.sort(a => (a.paid ? 1 : -1));
+  function globalRelaySelector() {
+    if (!isGlobal || !allRelays || allRelays.length === 0) return null;
 
-      if (ret.length > 0 && !relay) {
-        setRelay(ret[0]);
-      }
-      setGlobalRelays(ret);
-    });
-  }, [relays, relay]);
-
-  const isGlobal = loggedOut || tab.value === RootTab.Global.value;
-  const timelineSubect: TimelineSubject = (() => {
-    if (isGlobal) {
-      return { type: "global", items: [], discriminator: "all" };
-    }
-    if (tab.value >= 3) {
-      const hashtag = tab.text.slice(1);
-      return { type: "hashtag", items: [hashtag], discriminator: hashtag };
-    }
-
-    return { type: "pubkey", items: follows, discriminator: "follows" };
-  })();
-
-  const paidRelays = globalRelays.filter(a => a.paid);
-  const publicRelays = globalRelays.filter(a => !a.paid);
+    const paidRelays = allRelays.filter(a => a.paid);
+    const publicRelays = allRelays.filter(a => !a.paid);
     return (
-    <>
-      <div className="main-content">
-        {pubKey && <Tabs tabs={tabs} tab={tab} setTab={setTab} />}
-        {isGlobal && globalRelays.length > 0 && (
       <div className="flex mb10 f-end">
         <FormattedMessage
           defaultMessage="Read global from"
           description="Label for reading global feed from specific relays"
         />
         &nbsp;
-            <select onChange={e => setRelay(globalRelays.find(a => a.url === e.target.value))}>
+        <select onChange={e => setRelay(allRelays.find(a => a.url === e.target.value))} value={relay?.url}>
           {paidRelays.length > 0 && (
             <optgroup label="Paid Relays">
               {paidRelays.map(a => (
@@ -132,17 +101,64 @@ export default function RootPage() {
           </optgroup>
         </select>
       </div>
-        )}
-      </div>
-      {followHints()}
+    );
+  }
+
+  useEffect(() => {
+    if (isGlobal) {
+      return debounce(500, () => {
+        const ret: RelayOption[] = [];
+        System.Sockets.forEach((v, k) => {
+          ret.push({
+            url: k,
+            paid: v.Info?.limitation?.payment_required ?? false,
+          });
+        });
+        ret.sort(a => (a.paid ? -1 : 1));
+
+        if (ret.length > 0 && !relay) {
+          setRelay(ret[0]);
+        }
+        setAllRelays(ret);
+      });
+    }
+  }, [relays, relay, tab]);
+
+  const timelineSubect: TimelineSubject = (() => {
+    if (isGlobal) {
+      return { type: "global", items: [], discriminator: `all-${relay?.url}` };
+    }
+    if (tab.value >= 3) {
+      const hashtag = tab.text.slice(1);
+      return { type: "hashtag", items: [hashtag], discriminator: hashtag };
+    }
+
+    return { type: "pubkey", items: follows, discriminator: "follows" };
+  })();
+
+  function renderTimeline() {
+    if (isGlobal && !relay) return null;
+
+    return (
       <Timeline
         key={tab.value}
         subject={timelineSubect}
         postsOnly={tab.value === RootTab.Posts.value}
         method={"TIME_RANGE"}
         window={undefined}
-        relay={isGlobal ? relay?.url : undefined}
+        relay={isGlobal ? unwrap(relay).url : undefined}
       />
+    );
+  }
+
+  return (
+    <>
+      <div className="main-content">
+        {pubKey && <Tabs tabs={tabs} tab={tab} setTab={setTab} />}
+        {globalRelaySelector()}
+      </div>
+      {followHints()}
+      {renderTimeline()}
     </>
   );
 }

packages/app/src/State/Cache.ts

@@ -0,0 +1,38 @@
+import { createSlice, PayloadAction } from "@reduxjs/toolkit";
+import { TaggedRawEvent } from "@snort/nostr";
+
+export interface TimelineCache {
+  key: string;
+  main: TaggedRawEvent[];
+  related: TaggedRawEvent[];
+  latest: TaggedRawEvent[];
+  parent: TaggedRawEvent[];
+}
+
+export interface FeedCache {
+  timeline: TimelineCache;
+}
+
+const InitState = {
+  timeline: {
+    key: "",
+    main: [],
+    related: [],
+    latest: [],
+    parent: [],
+  },
+} as FeedCache;
+
+const CacheSlice = createSlice({
+  name: "Cache",
+  initialState: InitState,
+  reducers: {
+    setTimeline: (state, action: PayloadAction<TimelineCache>) => {
+      state.timeline = action.payload;
+    },
+  },
+});
+
+export const { setTimeline } = CacheSlice.actions;
+
+export const reducer = CacheSlice.reducer;

packages/app/src/State/Store.ts

@@ -1,11 +1,13 @@
 import { configureStore } from "@reduxjs/toolkit";
 import { reducer as LoginReducer } from "State/Login";
 import { reducer as UsersReducer } from "State/Users";
+import { reducer as CacheReducer } from "State/Cache";
 
 const store = configureStore({
   reducer: {
     login: LoginReducer,
     users: UsersReducer,
+    cache: CacheReducer,
   },
 });
 

packages/app/src/Util.ts

@@ -177,6 +177,23 @@ export function dedupeByPubkey(events: TaggedRawEvent[]) {
   return deduped.list as TaggedRawEvent[];
 }
 
+export function dedupeById(events: TaggedRawEvent[]) {
+  const deduped = events.reduce(
+    ({ list, seen }: { list: TaggedRawEvent[]; seen: Set<HexKey> }, ev) => {
+      if (seen.has(ev.id)) {
+        return { list, seen };
+      }
+      seen.add(ev.id);
+      return {
+        seen,
+        list: [...list, ev],
+      };
+    },
+    { list: [], seen: new Set([]) }
+  );
+  return deduped.list as TaggedRawEvent[];
+}
+
 export function unwrap<T>(v: T | undefined | null): T {
   if (v === undefined || v === null) {
     throw new Error("missing value");