From 988416f3531b0ca75d9be8526adb76769d7c9abb Mon Sep 17 00:00:00 2001 From: Kieran Date: Thu, 25 May 2023 10:31:47 +0100 Subject: [PATCH] fix: cache preload feat: Fetch only newest relay list updates --- packages/app/src/Cache/DMCache.ts | 4 +-- .../app/src/Cache/EventInteractionCache.ts | 4 +-- packages/app/src/Cache/FeedCache.ts | 20 +++++++-------- packages/app/src/Cache/UserCache.ts | 8 ++++++ packages/app/src/Cache/UserRelayCache.ts | 13 ++++++++++ packages/app/src/Cache/index.ts | 13 ++++++---- packages/app/src/Db/index.ts | 1 + packages/app/src/Feed/LoginFeed.ts | 2 ++ packages/app/src/Feed/RelaysFeedFollows.tsx | 14 +++++++---- packages/app/src/Pages/Layout.tsx | 25 ------------------- packages/app/src/System/Query.ts | 2 +- packages/app/src/System/index.ts | 2 +- packages/app/src/index.tsx | 22 ++++++++++++++++ 13 files changed, 79 insertions(+), 51 deletions(-) diff --git a/packages/app/src/Cache/DMCache.ts b/packages/app/src/Cache/DMCache.ts index a147559f..2ea1820d 100644 --- a/packages/app/src/Cache/DMCache.ts +++ b/packages/app/src/Cache/DMCache.ts @@ -11,8 +11,8 @@ class DMCache extends FeedCache { return of.id; } - override async preload(): Promise { - await super.preload(); + override async preload(follows?: Array): Promise { + await super.preload(follows); // load all dms to memory await this.buffer([...this.onTable]); } diff --git a/packages/app/src/Cache/EventInteractionCache.ts b/packages/app/src/Cache/EventInteractionCache.ts index 1a727c06..08893705 100644 --- a/packages/app/src/Cache/EventInteractionCache.ts +++ b/packages/app/src/Cache/EventInteractionCache.ts @@ -12,8 +12,8 @@ class EventInteractionCache extends FeedCache { return sha256(of.event + of.by); } - override async preload(): Promise { - await super.preload(); + override async preload(follows?: Array): Promise { + await super.preload(follows); const data = window.localStorage.getItem("zap-cache"); if (data) { diff --git a/packages/app/src/Cache/FeedCache.ts b/packages/app/src/Cache/FeedCache.ts index ccd91667..8d7966ed 100644 --- a/packages/app/src/Cache/FeedCache.ts +++ b/packages/app/src/Cache/FeedCache.ts @@ -12,18 +12,18 @@ interface HookFilter { export default abstract class FeedCache { #name: string; - #table: Table; #hooks: Array = []; #snapshot: Readonly> = []; #changed = true; #hits = 0; #miss = 0; + protected table: Table; protected onTable: Set = new Set(); protected cache: Map = new Map(); constructor(name: string, table: Table) { this.#name = name; - this.#table = table; + this.table = table; setInterval(() => { debug(this.#name)( "%d loaded, %d on-disk, %d hooks, %d% hit", @@ -35,9 +35,9 @@ export default abstract class FeedCache { }, 30_000); } - async preload() { + async preload(follows?: Array) { if (db.ready) { - const keys = await this.#table.toCollection().primaryKeys(); + const keys = await this.table.toCollection().primaryKeys(); this.onTable = new Set(keys.map(a => a as string)); } } @@ -75,7 +75,7 @@ export default abstract class FeedCache { async get(key?: string) { if (key && !this.cache.has(key) && db.ready) { - const cached = await this.#table.get(key); + const cached = await this.table.get(key); if (cached) { this.cache.set(this.key(cached), cached); this.notifyChange([key]); @@ -88,7 +88,7 @@ export default abstract class FeedCache { async bulkGet(keys: Array) { const missing = keys.filter(a => !this.cache.has(a)); if (missing.length > 0 && db.ready) { - const cached = await this.#table.bulkGet(missing); + const cached = await this.table.bulkGet(missing); cached.forEach(a => { if (a) { this.cache.set(this.key(a), a); @@ -105,7 +105,7 @@ export default abstract class FeedCache { const k = this.key(obj); this.cache.set(k, obj); if (db.ready) { - await this.#table.put(obj); + await this.table.put(obj); this.onTable.add(k); } this.notifyChange([k]); @@ -113,7 +113,7 @@ export default abstract class FeedCache { async bulkSet(obj: Array) { if (db.ready) { - await this.#table.bulkPut(obj); + await this.table.bulkPut(obj); obj.forEach(a => this.onTable.add(this.key(a))); } obj.forEach(v => this.cache.set(this.key(v), v)); @@ -164,7 +164,7 @@ export default abstract class FeedCache { key: a, })); const start = unixNowMs(); - const fromCache = await this.#table.bulkGet(mapped.filter(a => a.has).map(a => a.key)); + const fromCache = await this.table.bulkGet(mapped.filter(a => a.has).map(a => a.key)); const fromCacheFiltered = fromCache.filter(a => a !== undefined).map(a => unwrap(a)); fromCacheFiltered.forEach(a => { this.cache.set(this.key(a), a); @@ -184,7 +184,7 @@ export default abstract class FeedCache { } async clear() { - await this.#table.clear(); + await this.table.clear(); this.cache.clear(); this.onTable.clear(); } diff --git a/packages/app/src/Cache/UserCache.ts b/packages/app/src/Cache/UserCache.ts index 5dc2a3d4..06c99f73 100644 --- a/packages/app/src/Cache/UserCache.ts +++ b/packages/app/src/Cache/UserCache.ts @@ -18,6 +18,14 @@ class UserProfileCache extends FeedCache { return of.pubkey; } + override async preload(follows?: Array): Promise { + await super.preload(follows); + // load follows profiles + if (follows) { + await this.buffer(follows); + } + } + async search(q: string): Promise> { if (db.ready) { // on-disk cache will always have more data diff --git a/packages/app/src/Cache/UserRelayCache.ts b/packages/app/src/Cache/UserRelayCache.ts index 2f0a6ede..532d8a69 100644 --- a/packages/app/src/Cache/UserRelayCache.ts +++ b/packages/app/src/Cache/UserRelayCache.ts @@ -10,6 +10,19 @@ class UsersRelaysCache extends FeedCache { return of.pubkey; } + override async preload(follows?: Array): Promise { + await super.preload(follows); + if (follows) { + await this.buffer(follows); + } + } + + newest(): number { + let ret = 0; + this.cache.forEach(v => (ret = v.created_at > ret ? v.created_at : ret)); + return ret; + } + takeSnapshot(): Array { return [...this.cache.values()]; } diff --git a/packages/app/src/Cache/index.ts b/packages/app/src/Cache/index.ts index f147ecfa..8942fede 100644 --- a/packages/app/src/Cache/index.ts +++ b/packages/app/src/Cache/index.ts @@ -52,11 +52,14 @@ export function mapEventToProfile(ev: RawEvent) { } } -export async function preload() { - await UserCache.preload(); - await DmCache.preload(); - await InteractionCache.preload(); - await UserRelays.preload(); +export async function preload(follows?: Array) { + const preloads = [ + UserCache.preload(follows), + DmCache.preload(follows), + InteractionCache.preload(follows), + UserRelays.preload(follows), + ]; + await Promise.all(preloads); } export { UserCache, DmCache }; diff --git a/packages/app/src/Db/index.ts b/packages/app/src/Db/index.ts index 4ed7ddaf..df0fe111 100644 --- a/packages/app/src/Db/index.ts +++ b/packages/app/src/Db/index.ts @@ -21,6 +21,7 @@ export interface RelayMetrics { export interface UsersRelays { pubkey: HexKey; + created_at: number; relays: FullRelaySettings[]; } diff --git a/packages/app/src/Feed/LoginFeed.ts b/packages/app/src/Feed/LoginFeed.ts index fab34810..9f329067 100644 --- a/packages/app/src/Feed/LoginFeed.ts +++ b/packages/app/src/Feed/LoginFeed.ts @@ -1,5 +1,6 @@ import { useEffect, useMemo } from "react"; import { TaggedRawEvent, Lists, EventKind } from "@snort/nostr"; +import debug from "debug"; import { bech32ToHex, getNewest, getNewestEventTagsByKey, unwrap } from "SnortUtils"; import { makeNotification, sendNotification } from "Notifications"; @@ -41,6 +42,7 @@ export default function useLoginFeed() { .limit(1); const dmSince = DmCache.newest(); + debug("LoginFeed")("Loading dms since %s", new Date(dmSince * 1000).toISOString()); b.withFilter().authors([pubKey]).kinds([EventKind.DirectMessage]).since(dmSince); b.withFilter().kinds([EventKind.DirectMessage]).tag("p", [pubKey]).since(dmSince); return b; diff --git a/packages/app/src/Feed/RelaysFeedFollows.tsx b/packages/app/src/Feed/RelaysFeedFollows.tsx index 90bfb0f3..0f67815f 100644 --- a/packages/app/src/Feed/RelaysFeedFollows.tsx +++ b/packages/app/src/Feed/RelaysFeedFollows.tsx @@ -1,20 +1,24 @@ import { useMemo } from "react"; import { HexKey, FullRelaySettings, TaggedRawEvent, RelaySettings, EventKind } from "@snort/nostr"; +import debug from "debug"; import { sanitizeRelayUrl } from "SnortUtils"; import { PubkeyReplaceableNoteStore, RequestBuilder } from "System"; import useRequestBuilder from "Hooks/useRequestBuilder"; +import { UserRelays } from "Cache/UserRelayCache"; interface RelayList { pubkey: string; - created: number; + created_at: number; relays: FullRelaySettings[]; } export default function useRelaysFeedFollows(pubkeys: HexKey[]): Array { const sub = useMemo(() => { const b = new RequestBuilder(`relays:follows`); - b.withFilter().authors(pubkeys).kinds([EventKind.Relays, EventKind.ContactList]); + const since = UserRelays.newest(); + debug("LoginFeed")("Loading relay lists since %s", new Date(since * 1000).toISOString()); + b.withFilter().authors(pubkeys).kinds([EventKind.Relays, EventKind.ContactList]).since(since); return b; }, [pubkeys]); @@ -22,7 +26,7 @@ export default function useRelaysFeedFollows(pubkeys: HexKey[]): Array { return { pubkey: ev.pubkey, - created: ev.created_at, + created_at: ev.created_at, relays: ev.tags .map(a => { return { @@ -45,7 +49,7 @@ export default function useRelaysFeedFollows(pubkeys: HexKey[]): Array = JSON.parse(ev.content); return { pubkey: ev.pubkey, - created: ev.created_at, + created_at: ev.created_at, relays: Object.entries(relays) .map(([k, v]) => { return { @@ -61,7 +65,7 @@ export default function useRelaysFeedFollows(pubkeys: HexKey[]): Array { - // check DB support then init - db.isAvailable().then(async a => { - db.ready = a; - if (a) { - await preload(); - } - console.debug(`Using db: ${a ? "IndexedDB" : "In-Memory"}`); - - try { - if ("registerProtocolHandler" in window.navigator) { - window.navigator.registerProtocolHandler( - "web+nostr", - `${window.location.protocol}//${window.location.host}/%s` - ); - console.info("Registered protocol handler for 'web+nostr'"); - } - } catch (e) { - console.error("Failed to register protocol handler", e); - } - }); - }, []); - return (
{!shouldHideHeader && ( diff --git a/packages/app/src/System/Query.ts b/packages/app/src/System/Query.ts index d877f3ab..cb06c125 100644 --- a/packages/app/src/System/Query.ts +++ b/packages/app/src/System/Query.ts @@ -247,7 +247,7 @@ export class Query implements QueryBase { return false; } if ((q.relays?.length ?? 0) === 0 && c.Ephemeral) { - this.#log("Cant send non-specific REQ to ephemeral connection"); + this.#log("Cant send non-specific REQ to ephemeral connection %o", q.relays); return false; } if (q.filters.some(a => a.search) && !c.SupportsNip(Nips.Search)) { diff --git a/packages/app/src/System/index.ts b/packages/app/src/System/index.ts index 5c8a1e49..89146f19 100644 --- a/packages/app/src/System/index.ts +++ b/packages/app/src/System/index.ts @@ -221,7 +221,7 @@ export class NostrSystem extends ExternalStore { if (rb.options?.leaveOpen) { q.leaveOpen = rb.options.leaveOpen; } - if (rb.options?.relays) { + if (rb.options?.relays && (rb.options?.relays?.length ?? 0) > 0) { q.relays = rb.options.relays; } diff --git a/packages/app/src/index.tsx b/packages/app/src/index.tsx index 863e2bf6..0d012cf1 100644 --- a/packages/app/src/index.tsx +++ b/packages/app/src/index.tsx @@ -32,6 +32,9 @@ import Thread from "Element/Thread"; import { SubscribeRoutes } from "Pages/subscribe"; import ZapPoolPage from "Pages/ZapPool"; import DebugPage from "Pages/Debug"; +import { db } from "Db"; +import { preload } from "Cache"; +import { LoginStore } from "Login"; // @ts-ignore window.__webpack_nonce__ = "ZmlhdGphZiBzYWlkIHNub3J0LnNvY2lhbCBpcyBwcmV0dHkgZ29vZCwgd2UgbWFkZSBpdCE="; @@ -42,6 +45,25 @@ export const router = createBrowserRouter([ { element: , errorElement: , + loader: async () => { + db.ready = await db.isAvailable(); + if (db.ready) { + await preload(LoginStore.takeSnapshot().follows.item); + } + + try { + if ("registerProtocolHandler" in window.navigator) { + window.navigator.registerProtocolHandler( + "web+nostr", + `${window.location.protocol}//${window.location.host}/%s` + ); + console.info("Registered protocol handler for 'web+nostr'"); + } + } catch (e) { + console.error("Failed to register protocol handler", e); + } + return null; + }, children: [ ...RootRoutes, {