From a49121c05a264d23aabf1011a562f4e1456eb977 Mon Sep 17 00:00:00 2001 From: Kieran Date: Tue, 21 Feb 2023 12:03:14 +0000 Subject: [PATCH 1/3] chore: remove feed cache --- packages/app/src/Db/index.ts | 20 ++--------- packages/app/src/Feed/Subscription.ts | 48 ++------------------------- packages/app/src/Pages/Layout.tsx | 10 ------ 3 files changed, 5 insertions(+), 73 deletions(-) diff --git a/packages/app/src/Db/index.ts b/packages/app/src/Db/index.ts index 2a0162b..eb870f2 100644 --- a/packages/app/src/Db/index.ts +++ b/packages/app/src/Db/index.ts @@ -1,10 +1,9 @@ import Dexie, { Table } from "dexie"; -import { TaggedRawEvent, u256 } from "@snort/nostr"; +import { u256 } from "@snort/nostr"; import { MetadataCache } from "State/Users"; -import { hexToBech32 } from "Util"; export const NAME = "snortDB"; -export const VERSION = 3; +export const VERSION = 4; export interface SubCache { id: string; @@ -15,27 +14,14 @@ export interface SubCache { const STORES = { users: "++pubkey, name, display_name, picture, nip05, npub", - events: "++id, pubkey, created_at", - feeds: "++id", }; export class SnortDB extends Dexie { users!: Table; - events!: Table; - feeds!: Table; constructor() { super(NAME); - this.version(VERSION) - .stores(STORES) - .upgrade(async tx => { - await tx - .table("users") - .toCollection() - .modify(user => { - user.npub = hexToBech32("npub", user.pubkey); - }); - }); + this.version(VERSION).stores(STORES); } } diff --git a/packages/app/src/Feed/Subscription.ts b/packages/app/src/Feed/Subscription.ts index d8bd08d..39575a3 100644 --- a/packages/app/src/Feed/Subscription.ts +++ b/packages/app/src/Feed/Subscription.ts @@ -2,8 +2,7 @@ import { useEffect, useMemo, useReducer, useState } from "react"; import { TaggedRawEvent } from "@snort/nostr"; import { Subscriptions } from "@snort/nostr"; import { System } from "System"; -import { debounce, unwrap } from "Util"; -import { db } from "Db"; +import { debounce } from "Util"; export type NoteStore = { notes: Array; @@ -80,7 +79,6 @@ export default function useSubscription( const [state, dispatch] = useReducer(notesReducer, initStore); const [debounceOutput, setDebounceOutput] = useState(0); const [subDebounce, setSubDebounced] = useState(); - const useCache = useMemo(() => options?.cache === true, [options]); useEffect(() => { if (sub) { @@ -97,25 +95,11 @@ export default function useSubscription( end: false, }); - if (useCache) { - // preload notes from db - PreloadNotes(subDebounce.Id) - .then(ev => { - dispatch({ - type: "EVENT", - ev: ev, - }); - }) - .catch(console.warn); - } subDebounce.OnEvent = e => { dispatch({ type: "EVENT", ev: e, }); - if (useCache) { - db.events.put(e); - } }; subDebounce.OnEnd = c => { @@ -144,15 +128,7 @@ export default function useSubscription( System.RemoveSubscription(subDebounce.Id); }; } - }, [subDebounce, useCache]); - - useEffect(() => { - if (subDebounce && useCache) { - return debounce(500, () => { - TrackNotesInFeed(subDebounce.Id, state.notes).catch(console.warn); - }); - } - }, [state, useCache]); + }, [subDebounce]); useEffect(() => { return debounce(DebounceMs, () => { @@ -174,23 +150,3 @@ export default function useSubscription( }, }; } - -/** - * Lookup cached copy of feed - */ -const PreloadNotes = async (id: string): Promise => { - const feed = await db.feeds.get(id); - if (feed) { - const events = await db.events.bulkGet(feed.ids); - return events.filter(a => a !== undefined).map(a => unwrap(a)); - } - return []; -}; - -const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => { - const existing = await db.feeds.get(id); - const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)])); - const since = notes.reduce((acc, v) => (acc > v.created_at ? v.created_at : acc), +Infinity); - const until = notes.reduce((acc, v) => (acc < v.created_at ? v.created_at : acc), -Infinity); - await db.feeds.put({ id, ids, since, until }); -}; diff --git a/packages/app/src/Pages/Layout.tsx b/packages/app/src/Pages/Layout.tsx index 30c9e8d..0a0368b 100644 --- a/packages/app/src/Pages/Layout.tsx +++ b/packages/app/src/Pages/Layout.tsx @@ -17,7 +17,6 @@ import { SearchRelays, SnortPubKey } from "Const"; import useEventPublisher from "Feed/EventPublisher"; import useModeration from "Hooks/useModeration"; import { IndexedUDB } from "State/Users/Db"; -import { db } from "Db"; import { bech32ToHex } from "Util"; import { NoteCreator } from "Element/NoteCreator"; import Plus from "Icons/Plus"; @@ -120,15 +119,6 @@ export default function Layout() { // cleanup on load if (dbType === "indexdDb") { IndexedUDB.ready = true; - await db.feeds.clear(); - const now = Math.floor(new Date().getTime() / 1000); - - const cleanupEvents = await db.events - .where("created_at") - .above(now - 60 * 60) - .primaryKeys(); - console.debug(`Cleanup ${cleanupEvents.length} events`); - await db.events.bulkDelete(cleanupEvents); } console.debug(`Using db: ${dbType}`); From e6f64e9b9e5f93682b862fe73dad1ee06cb4feed Mon Sep 17 00:00:00 2001 From: Kieran Date: Tue, 21 Feb 2023 14:35:53 +0000 Subject: [PATCH 2/3] feat: redux timeline cache --- packages/app/src/Element/Note.tsx | 2 +- packages/app/src/Element/Timeline.tsx | 46 +++++++--- packages/app/src/Pages/Root.tsx | 124 +++++++++++++++----------- packages/app/src/State/Cache.ts | 38 ++++++++ packages/app/src/State/Store.ts | 2 + packages/app/src/Util.ts | 17 ++++ 6 files changed, 160 insertions(+), 69 deletions(-) create mode 100644 packages/app/src/State/Cache.ts diff --git a/packages/app/src/Element/Note.tsx b/packages/app/src/Element/Note.tsx index eac4178..b858572 100644 --- a/packages/app/src/Element/Note.tsx +++ b/packages/app/src/Element/Note.tsx @@ -168,7 +168,7 @@ export default function Note(props: NoteProps) { useLayoutEffect(() => { if (entry && inView && extendable === false) { - const h = entry?.target.clientHeight ?? 0; + const h = (entry?.target as HTMLDivElement)?.offsetHeight ?? 0; if (h > 650) { setExtendable(true); } diff --git a/packages/app/src/Element/Timeline.tsx b/packages/app/src/Element/Timeline.tsx index d5b2999..5628258 100644 --- a/packages/app/src/Element/Timeline.tsx +++ b/packages/app/src/Element/Timeline.tsx @@ -1,10 +1,10 @@ import "./Timeline.css"; import { FormattedMessage } from "react-intl"; -import { useCallback, useMemo } from "react"; +import { useCallback, useEffect, useMemo } from "react"; import { useInView } from "react-intersection-observer"; import ArrowUp from "Icons/ArrowUp"; -import { dedupeByPubkey, tagFilterOfTextRepost } from "Util"; +import { dedupeById, dedupeByPubkey, tagFilterOfTextRepost } from "Util"; import ProfileImage from "Element/ProfileImage"; import useTimelineFeed, { TimelineSubject } from "Feed/TimelineFeed"; import { TaggedRawEvent } from "@snort/nostr"; @@ -16,6 +16,9 @@ import NoteReaction from "Element/NoteReaction"; import useModeration from "Hooks/useModeration"; import ProfilePreview from "./ProfilePreview"; import Skeleton from "Element/Skeleton"; +import { useDispatch, useSelector } from "react-redux"; +import { RootState } from "State/Store"; +import { setTimeline } from "State/Cache"; export interface TimelineProps { postsOnly: boolean; @@ -38,7 +41,9 @@ export default function Timeline({ relay, }: TimelineProps) { const { muted, isMuted } = useModeration(); - const { main, related, latest, parent, loadMore, showLatest } = useTimelineFeed(subject, { + const dispatch = useDispatch(); + const cache = useSelector((s: RootState) => s.cache.timeline); + const feed = useTimelineFeed(subject, { method, window: timeWindow, relay, @@ -52,20 +57,34 @@ export default function Timeline({ ?.filter(a => (postsOnly ? !a.tags.some(b => b[0] === "e") : true)) .filter(a => ignoreModeration || !isMuted(a.pubkey)); }, - [postsOnly, muted] + [postsOnly, muted, ignoreModeration] ); const mainFeed = useMemo(() => { - return filterPosts(main.notes); - }, [main, filterPosts]); + return filterPosts(cache.main); + }, [cache, filterPosts]); const latestFeed = useMemo(() => { - return filterPosts(latest.notes).filter(a => !mainFeed.some(b => b.id === a.id)); - }, [latest, mainFeed, filterPosts]); + return filterPosts(cache.latest).filter(a => !mainFeed.some(b => b.id === a.id)); + }, [cache, filterPosts]); const latestAuthors = useMemo(() => { return dedupeByPubkey(latestFeed).map(e => e.pubkey); }, [latestFeed]); + useEffect(() => { + const key = `${subject.type}-${subject.discriminator}`; + const newFeed = key !== cache.key; + dispatch( + setTimeline({ + key: key, + main: dedupeById([...(newFeed ? [] : cache.main), ...feed.main.notes]), + latest: [...feed.latest.notes], + related: dedupeById([...(newFeed ? [] : cache.related), ...feed.related.notes]), + parent: dedupeById([...(newFeed ? [] : cache.parent), ...feed.parent.notes]), + }) + ); + }, [feed.main, feed.latest, feed.related, feed.parent]); + function eventElement(e: TaggedRawEvent) { switch (e.kind) { case EventKind.SetMetadata: { @@ -74,9 +93,9 @@ export default function Timeline({ case EventKind.TextNote: { const eRef = e.tags.find(tagFilterOfTextRepost(e))?.at(1); if (eRef) { - return a.id === eRef)} />; + return a.id === eRef)} />; } - return ; + return ; } case EventKind.ZapReceipt: { const zap = parseZap(e); @@ -85,18 +104,17 @@ export default function Timeline({ case EventKind.Reaction: case EventKind.Repost: { const eRef = e.tags.find(a => a[0] === "e")?.at(1); - return a.id === eRef)} />; + return a.id === eRef)} />; } } } function onShowLatest(scrollToTop = false) { - showLatest(); + feed.showLatest(); if (scrollToTop) { window.scrollTo(0, 0); } } - return (
{latestFeed.length > 0 && ( @@ -126,7 +144,7 @@ export default function Timeline({ )} {mainFeed.map(eventElement)} - + diff --git a/packages/app/src/Pages/Root.tsx b/packages/app/src/Pages/Root.tsx index 9249843..04acb31 100644 --- a/packages/app/src/Pages/Root.tsx +++ b/packages/app/src/Pages/Root.tsx @@ -11,7 +11,7 @@ import { System } from "System"; import { TimelineSubject } from "Feed/TimelineFeed"; import messages from "./messages"; -import { debounce } from "Util"; +import { debounce, unwrap } from "Util"; interface RelayOption { url: string; @@ -46,11 +46,12 @@ export default function RootPage() { } }); const [relay, setRelay] = useState(); - const [globalRelays, setGlobalRelays] = useState([]); + const [allRelays, setAllRelays] = useState(); const tagTabs = tags.map((t, idx) => { return { text: `#${t}`, value: idx + 3 }; }); const tabs = [RootTab.Posts, RootTab.PostsAndReplies, RootTab.Global, ...tagTabs]; + const isGlobal = loggedOut || tab.value === RootTab.Global.value; function followHints() { if (follows?.length === 0 && pubKey && tab !== RootTab.Global) { @@ -69,28 +70,63 @@ export default function RootPage() { } } + function globalRelaySelector() { + if (!isGlobal || !allRelays || allRelays.length === 0) return null; + + const paidRelays = allRelays.filter(a => a.paid); + const publicRelays = allRelays.filter(a => !a.paid); + return ( +
+ +   + +
+ ); + } + useEffect(() => { - return debounce(1_000, () => { - const ret: RelayOption[] = []; - System.Sockets.forEach((v, k) => { - ret.push({ - url: k, - paid: v.Info?.limitation?.payment_required ?? false, + if (isGlobal) { + return debounce(500, () => { + const ret: RelayOption[] = []; + System.Sockets.forEach((v, k) => { + ret.push({ + url: k, + paid: v.Info?.limitation?.payment_required ?? false, + }); }); + ret.sort(a => (a.paid ? -1 : 1)); + + if (ret.length > 0 && !relay) { + setRelay(ret[0]); + } + setAllRelays(ret); }); - ret.sort(a => (a.paid ? 1 : -1)); + } + }, [relays, relay, tab]); - if (ret.length > 0 && !relay) { - setRelay(ret[0]); - } - setGlobalRelays(ret); - }); - }, [relays, relay]); - - const isGlobal = loggedOut || tab.value === RootTab.Global.value; const timelineSubect: TimelineSubject = (() => { if (isGlobal) { - return { type: "global", items: [], discriminator: "all" }; + return { type: "global", items: [], discriminator: `all-${relay?.url}` }; } if (tab.value >= 3) { const hashtag = tab.text.slice(1); @@ -100,49 +136,29 @@ export default function RootPage() { return { type: "pubkey", items: follows, discriminator: "follows" }; })(); - const paidRelays = globalRelays.filter(a => a.paid); - const publicRelays = globalRelays.filter(a => !a.paid); - return ( - <> -
- {pubKey && } - {isGlobal && globalRelays.length > 0 && ( -
- -   - -
- )} -
- {followHints()} + function renderTimeline() { + if (isGlobal && !relay) return null; + + return ( + ); + } + + return ( + <> +
+ {pubKey && } + {globalRelaySelector()} +
+ {followHints()} + {renderTimeline()} ); } diff --git a/packages/app/src/State/Cache.ts b/packages/app/src/State/Cache.ts new file mode 100644 index 0000000..9d2201e --- /dev/null +++ b/packages/app/src/State/Cache.ts @@ -0,0 +1,38 @@ +import { createSlice, PayloadAction } from "@reduxjs/toolkit"; +import { TaggedRawEvent, u256 } from "@snort/nostr"; + +export interface TimelineCache { + key: string; + main: TaggedRawEvent[]; + related: TaggedRawEvent[]; + latest: TaggedRawEvent[]; + parent: TaggedRawEvent[]; +} + +export interface FeedCache { + timeline: TimelineCache; +} + +const InitState = { + timeline: { + key: "", + main: [], + related: [], + latest: [], + parent: [], + }, +} as FeedCache; + +const CacheSlice = createSlice({ + name: "Cache", + initialState: InitState, + reducers: { + setTimeline: (state, action: PayloadAction) => { + state.timeline = action.payload; + }, + }, +}); + +export const { setTimeline } = CacheSlice.actions; + +export const reducer = CacheSlice.reducer; diff --git a/packages/app/src/State/Store.ts b/packages/app/src/State/Store.ts index 1665055..196dad2 100644 --- a/packages/app/src/State/Store.ts +++ b/packages/app/src/State/Store.ts @@ -1,11 +1,13 @@ import { configureStore } from "@reduxjs/toolkit"; import { reducer as LoginReducer } from "State/Login"; import { reducer as UsersReducer } from "State/Users"; +import { reducer as CacheReducer } from "State/Cache"; const store = configureStore({ reducer: { login: LoginReducer, users: UsersReducer, + cache: CacheReducer, }, }); diff --git a/packages/app/src/Util.ts b/packages/app/src/Util.ts index 8d9fea1..49e236b 100644 --- a/packages/app/src/Util.ts +++ b/packages/app/src/Util.ts @@ -177,6 +177,23 @@ export function dedupeByPubkey(events: TaggedRawEvent[]) { return deduped.list as TaggedRawEvent[]; } +export function dedupeById(events: TaggedRawEvent[]) { + const deduped = events.reduce( + ({ list, seen }: { list: TaggedRawEvent[]; seen: Set }, ev) => { + if (seen.has(ev.id)) { + return { list, seen }; + } + seen.add(ev.id); + return { + seen, + list: [...list, ev], + }; + }, + { list: [], seen: new Set([]) } + ); + return deduped.list as TaggedRawEvent[]; +} + export function unwrap(v: T | undefined | null): T { if (v === undefined || v === null) { throw new Error("missing value"); From 9ea8910fcd0af8f63c41f3078c084e803919481e Mon Sep 17 00:00:00 2001 From: Kieran Date: Tue, 21 Feb 2023 14:36:12 +0000 Subject: [PATCH 3/3] chore: warnings --- packages/app/src/State/Cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/src/State/Cache.ts b/packages/app/src/State/Cache.ts index 9d2201e..bdb4206 100644 --- a/packages/app/src/State/Cache.ts +++ b/packages/app/src/State/Cache.ts @@ -1,5 +1,5 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; -import { TaggedRawEvent, u256 } from "@snort/nostr"; +import { TaggedRawEvent } from "@snort/nostr"; export interface TimelineCache { key: string;