Cache all the things

packages/app/src/Cache/FollowsFeed.ts

@@ -0,0 +1,124 @@
+import debug from "debug";
+import { EventKind, RequestBuilder, SystemInterface, TaggedNostrEvent } from "@snort/system";
+import { unixNow, unixNowMs } from "@snort/shared";
+
+import { db } from "Db";
+import { RefreshFeedCache, TWithCreated } from "./RefreshFeedCache";
+import { LoginSession } from "Login";
+import { Day, Hour } from "Const";
+
+const WindowSize = Hour * 6;
+const MaxCacheWindow = Day * 7;
+
+export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
+  #kinds = [EventKind.TextNote, EventKind.Repost, EventKind.Polls];
+  #oldest: number = 0;
+  
+  constructor() {
+    super("FollowsFeedCache", db.followsFeed);
+  }
+
+  key(of: TWithCreated<TaggedNostrEvent>): string {
+    return of.id;
+  }
+
+  takeSnapshot(): TWithCreated<TaggedNostrEvent>[] {
+    return [...this.cache.values()];
+  }
+
+  buildSub(session: LoginSession, rb: RequestBuilder): void {
+    const since = this.newest();
+    rb.withFilter()
+    .kinds(this.#kinds)
+    .authors(session.follows.item)
+    .since(since === 0 ? unixNow() - WindowSize : since);
+  }
+
+  async onEvent(evs: readonly TaggedNostrEvent[]): Promise<void> {
+    const filtered = evs.filter(a => this.#kinds.includes(a.kind));
+    if(filtered.length > 0) {
+      await this.bulkSet(filtered);
+      this.notifyChange(filtered.map(a => this.key(a)));
+    }
+  }
+
+  override async preload() {
+    const start = unixNowMs();
+    const keys = (await this.table?.toCollection().primaryKeys()) ?? [];
+    this.onTable = new Set<string>(keys.map(a => a as string));
+
+    // load only latest 10 posts, rest can be loaded on-demand
+    const latest = await this.table?.orderBy("created_at").reverse().limit(50).toArray();
+    latest?.forEach(v => this.cache.set(this.key(v), v));
+
+    // cleanup older than 7 days
+    await this.table?.where("created_at").below(unixNow() - MaxCacheWindow).delete();
+
+    const oldest = await this.table?.orderBy("created_at").first();
+    this.#oldest = oldest?.created_at ?? 0;
+    this.notifyChange(latest?.map(a => this.key(a)) ?? []);
+    
+    debug(this.name)(
+      `Loaded %d/%d in %d ms`,
+      latest?.length ?? 0,
+      keys.length,
+      (unixNowMs() - start).toLocaleString(),
+    );
+  }
+
+  async loadMore(system: SystemInterface, session: LoginSession, before: number) {
+    if(before <= this.#oldest) {
+      const rb = new RequestBuilder(`${this.name}-loadmore`);
+      rb.withFilter()
+        .kinds(this.#kinds)
+        .authors(session.follows.item)
+        .until(before)
+        .since(before - WindowSize);
+      await system.Fetch(rb, async evs => {
+        await this.bulkSet(evs);
+      });
+    } else {
+      const latest = await this.table?.where("created_at").between(before - WindowSize, before).reverse().sortBy("created_at");
+      latest?.forEach(v => {
+        const k = this.key(v);
+        this.cache.set(k, v);
+        this.onTable.add(k);
+      });
+      
+      this.notifyChange(latest?.map(a => this.key(a)) ?? []);
+    }
+  }
+
+  /**
+   * Backfill cache with new follows
+   */
+  async backFill(system: SystemInterface, keys: Array<string>) {
+    if(keys.length === 0) return;
+
+    const rb = new RequestBuilder(`${this.name}-backfill`);
+    rb.withFilter()
+      .kinds(this.#kinds)
+      .authors(keys)
+      .until(unixNow())
+      .since(this.#oldest ?? unixNow() - MaxCacheWindow);
+    await system.Fetch(rb, async evs => {
+      await this.bulkSet(evs);
+    });
+  }
+
+  /**
+   * Backfill cache based on follows list
+   */
+  async backFillIfMissing(system: SystemInterface, keys: Array<string>) {
+    const start = unixNowMs();
+    const everything = await this.table?.toArray();
+    const allKeys = new Set(everything?.map(a => a.pubkey));
+    const missingKeys = keys.filter(a => !allKeys.has(a));
+    await this.backFill(system, missingKeys);
+    debug(this.name)(
+      `Backfilled %d keys in %d ms`,
+      missingKeys.length,
+      (unixNowMs() - start).toLocaleString(),
+    );
+  }
+}

packages/app/src/Cache/GiftWrapCache.ts

@@ -1,9 +1,10 @@
-import { FeedCache } from "@snort/shared";
-import { EventKind, EventPublisher, TaggedNostrEvent } from "@snort/system";
+import { EventKind, EventPublisher, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { UnwrappedGift, db } from "Db";
 import { findTag, unwrap } from "SnortUtils";
+import { RefreshFeedCache } from "./RefreshFeedCache";
+import { LoginSession } from "Login";
 
-export class GiftWrapCache extends FeedCache<UnwrappedGift> {
+export class GiftWrapCache extends RefreshFeedCache<UnwrappedGift>  {
   constructor() {
     super("GiftWrapCache", db.gifts);
   }
@@ -12,22 +13,20 @@ export class GiftWrapCache extends FeedCache<UnwrappedGift> {
     return of.id;
   }
 
-  override async preload(): Promise<void> {
-    await super.preload();
-    await this.buffer([...this.onTable]);
+  buildSub(session: LoginSession, rb: RequestBuilder): void {
+    const pubkey = session.publicKey;
+    if(pubkey) {
+      rb.withFilter()
+      .kinds([EventKind.GiftWrap])
+      .tag("p", [pubkey]).since(this.newest());
+    }
   }
-
-  newest(): number {
-    let ret = 0;
-    this.cache.forEach(v => (ret = v.created_at > ret ? v.created_at : ret));
-    return ret;
-  }
-
+  
   takeSnapshot(): Array<UnwrappedGift> {
     return [...this.cache.values()];
   }
 
-  async onEvent(evs: Array<TaggedNostrEvent>, pub: EventPublisher) {
+  override async onEvent(evs: Array<TaggedNostrEvent>, pub: EventPublisher) {
     const unwrapped = (
       await Promise.all(
         evs.map(async v => {

packages/app/src/Cache/Notifications.ts

@@ -1,8 +1,9 @@
 import { EventKind, NostrEvent, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { RefreshFeedCache, TWithCreated } from "./RefreshFeedCache";
 import { LoginSession } from "Login";
-import { unixNow } from "SnortUtils";
 import { db } from "Db";
+import { Day } from "Const";
+import { unixNow } from "@snort/shared";
 
 export class NotificationsCache extends RefreshFeedCache<NostrEvent> {
   #kinds = [EventKind.TextNote, EventKind.Reaction, EventKind.Repost, EventKind.ZapReceipt];
@@ -17,7 +18,7 @@ export class NotificationsCache extends RefreshFeedCache<NostrEvent> {
       rb.withFilter()
         .kinds(this.#kinds)
         .tag("p", [session.publicKey])
-        .since(newest === 0 ? unixNow() - 60 * 60 * 24 * 30 : newest);
+        .since(newest === 0 ? unixNow() -  (Day * 30): newest);
     }
   }
 

packages/app/src/Cache/RefreshFeedCache.ts

@@ -1,12 +1,12 @@
 import { FeedCache } from "@snort/shared";
-import { RequestBuilder, TaggedNostrEvent } from "@snort/system";
+import { EventPublisher, RequestBuilder, TaggedNostrEvent } from "@snort/system";
 import { LoginSession } from "Login";
 
-export type TWithCreated<T> = T & { created_at: number };
+export type TWithCreated<T> = (T | Readonly<T>) & { created_at: number };
 
 export abstract class RefreshFeedCache<T> extends FeedCache<TWithCreated<T>> {
   abstract buildSub(session: LoginSession, rb: RequestBuilder): void;
-  abstract onEvent(evs: Readonly<Array<TaggedNostrEvent>>): void;
+  abstract onEvent(evs: Readonly<Array<TaggedNostrEvent>>, pub: EventPublisher): void;
 
   /**
    * Get latest event

packages/app/src/Cache/index.ts

@@ -4,6 +4,7 @@ import { ChatCache } from "./ChatCache";
 import { Payments } from "./PaymentsCache";
 import { GiftWrapCache } from "./GiftWrapCache";
 import { NotificationsCache } from "./Notifications";
+import { FollowsFeedCache } from "./FollowsFeed";
 
 export const UserCache = new UserProfileCache();
 export const UserRelays = new UserRelaysCache();
@@ -13,6 +14,7 @@ export const PaymentsCache = new Payments();
 export const InteractionCache = new EventInteractionCache();
 export const GiftsCache = new GiftWrapCache();
 export const Notifications = new NotificationsCache();
+export const FollowsFeed = new FollowsFeedCache();
 
 export async function preload(follows?: Array<string>) {
   const preloads = [
@@ -23,6 +25,7 @@ export async function preload(follows?: Array<string>) {
     RelayMetrics.preload(),
     GiftsCache.preload(),
     Notifications.preload(),
+    FollowsFeed.preload(),
   ];
   await Promise.all(preloads);
 }