feat: improve profile loading

packages/app/src/Const.ts

@@ -33,7 +33,7 @@ export const DefaultConnectTimeout = 2000;
 /**
  * How long profile cache should be considered valid for
  */
-export const ProfileCacheExpire = 1_000 * 60 * 5;
+export const ProfileCacheExpire = 1_000 * 60 * 30;
 
 /**
  * Default bootstrap relays

packages/app/src/Feed/EventPublisher.ts

@@ -2,11 +2,12 @@ import { useSelector } from "react-redux";
 import * as secp from "@noble/secp256k1";
 
 import { TaggedRawEvent } from "@snort/nostr";
-import { EventKind, Tag, Event as NEvent, System, RelaySettings } from "@snort/nostr";
+import { EventKind, Tag, Event as NEvent, RelaySettings } from "@snort/nostr";
 import { RootState } from "State/Store";
 import { HexKey, RawEvent, u256, UserMetadata, Lists } from "@snort/nostr";
 import { bech32ToHex, unwrap } from "Util";
 import { DefaultRelays, HashtagRegex } from "Const";
+import { System } from "System";
 
 declare global {
   interface Window {

packages/app/src/Feed/LoginFeed.ts

@@ -19,13 +19,13 @@ import {
 } from "State/Login";
 import { RootState } from "State/Store";
 import { mapEventToProfile, MetadataCache } from "State/Users";
-import { useDb } from "State/Users/Db";
 import useSubscription from "Feed/Subscription";
 import { barrierNip07 } from "Feed/EventPublisher";
 import { getMutedKeys } from "Feed/MuteList";
 import useModeration from "Hooks/useModeration";
 import { unwrap } from "Util";
 import { AnyAction, ThunkDispatch } from "@reduxjs/toolkit";
+import { ReduxUDB } from "State/Users/Db";
 
 /**
  * Managed loading data for the current logged in user
@@ -39,7 +39,6 @@ export default function useLoginFeed() {
     readNotifications,
   } = useSelector((s: RootState) => s.login);
   const { isMuted } = useModeration();
-  const db = useDb();
 
   const subMetadata = useMemo(() => {
     if (!pubKey) return null;
@@ -176,13 +175,13 @@ export default function useLoginFeed() {
         { created: 0, profile: null as MetadataCache | null }
       );
       if (maxProfile.profile) {
-        const existing = await db.find(maxProfile.profile.pubkey);
+        const existing = await ReduxUDB.find(maxProfile.profile.pubkey);
         if ((existing?.created ?? 0) < maxProfile.created) {
-          await db.put(maxProfile.profile);
+          await ReduxUDB.put(maxProfile.profile);
         }
       }
     })().catch(console.warn);
-  }, [dispatch, metadataFeed.store, db]);
+  }, [dispatch, metadataFeed.store, ReduxUDB]);
 
   useEffect(() => {
     const replies = notificationFeed.store.notes.filter(
@@ -190,13 +189,13 @@ export default function useLoginFeed() {
     );
     replies.forEach(nx => {
       dispatch(setLatestNotifications(nx.created_at));
-      makeNotification(db, nx).then(notification => {
+      makeNotification(ReduxUDB, nx).then(notification => {
         if (notification) {
           (dispatch as ThunkDispatch<RootState, undefined, AnyAction>)(sendNotification(notification));
         }
       });
     });
-  }, [dispatch, notificationFeed.store, db, readNotifications]);
+  }, [dispatch, notificationFeed.store, ReduxUDB, readNotifications]);
 
   useEffect(() => {
     const muted = getMutedKeys(mutedFeed.store.notes);

packages/app/src/Feed/ProfileFeed.ts

@@ -1,7 +1,8 @@
 import { useEffect } from "react";
 import { MetadataCache } from "State/Users";
 import { useKey, useKeys } from "State/Users/Hooks";
-import { System, HexKey } from "@snort/nostr";
+import { HexKey } from "@snort/nostr";
+import { System } from "System";
 
 export function useUserProfile(pubKey?: HexKey): MetadataCache | undefined {
   const users = useKey(pubKey);

packages/app/src/Feed/RelayState.ts

@@ -1,5 +1,6 @@
 import { useSyncExternalStore } from "react";
-import { System, StateSnapshot } from "@snort/nostr";
+import { StateSnapshot } from "@snort/nostr";
+import { System } from "System";
 
 const noop = () => {
   return () => undefined;

packages/app/src/Feed/Subscription.ts

@@ -1,6 +1,7 @@
 import { useEffect, useMemo, useReducer, useState } from "react";
 import { TaggedRawEvent } from "@snort/nostr";
-import { System, Subscriptions } from "@snort/nostr";
+import { Subscriptions } from "@snort/nostr";
+import { System } from "System";
 import { debounce, unwrap } from "Util";
 import { db } from "Db";
 

packages/app/src/Pages/Layout.tsx

@@ -9,14 +9,14 @@ import Bell from "Icons/Bell";
 import Search from "Icons/Search";
 import { RootState } from "State/Store";
 import { init, setRelays } from "State/Login";
-import { System } from "@snort/nostr";
+import { System } from "System";
 import ProfileImage from "Element/ProfileImage";
 import useLoginFeed from "Feed/LoginFeed";
 import { totalUnread } from "Pages/MessagesPage";
 import { SearchRelays, SnortPubKey } from "Const";
 import useEventPublisher from "Feed/EventPublisher";
 import useModeration from "Hooks/useModeration";
-import { IndexedUDB, useDb } from "State/Users/Db";
+import { IndexedUDB } from "State/Users/Db";
 import { db } from "Db";
 import { bech32ToHex } from "Util";
 import { NoteCreator } from "Element/NoteCreator";
@@ -34,8 +34,6 @@ export default function Layout() {
     useSelector((s: RootState) => s.login);
   const { isMuted } = useModeration();
   const [pageClass, setPageClass] = useState("page");
-
-  const usingDb = useDb();
   const pub = useEventPublisher();
   useLoginFeed();
 
@@ -73,13 +71,9 @@ export default function Layout() {
   );
 
   useEffect(() => {
-    System.nip42Auth = pub.nip42Auth;
+    System.HandleAuth = pub.nip42Auth;
   }, [pub]);
 
-  useEffect(() => {
-    System.UserDb = usingDb;
-  }, [usingDb]);
-
   useEffect(() => {
     if (relays) {
       for (const [k, v] of Object.entries(relays)) {
@@ -125,6 +119,7 @@ export default function Layout() {
 
       // cleanup on load
       if (dbType === "indexdDb") {
+        IndexedUDB.ready = true;
         await db.feeds.clear();
         const now = Math.floor(new Date().getTime() / 1000);
 

packages/app/src/Pages/Root.tsx

@@ -7,7 +7,7 @@ import { useIntl, FormattedMessage } from "react-intl";
 import Tabs, { Tab } from "Element/Tabs";
 import { RootState } from "State/Store";
 import Timeline from "Element/Timeline";
-import { System } from "@snort/nostr";
+import { System } from "System";
 import { TimelineSubject } from "Feed/TimelineFeed";
 
 import messages from "./messages";

packages/app/src/Pages/SearchPage.tsx

@@ -6,7 +6,7 @@ import { useEffect, useState } from "react";
 import { debounce } from "Util";
 import { router } from "index";
 import { SearchRelays } from "Const";
-import { System } from "@snort/nostr";
+import { System } from "System";
 import { useQuery } from "State/Users/Hooks";
 
 import messages from "./messages";

packages/app/src/Pages/settings/RelayInfo.tsx

@@ -1,11 +1,11 @@
 import { FormattedMessage } from "react-intl";
 import ProfilePreview from "Element/ProfilePreview";
 import useRelayState from "Feed/RelayState";
-import { System } from "@snort/nostr";
 import { useDispatch } from "react-redux";
 import { useNavigate, useParams } from "react-router-dom";
 import { removeRelay } from "State/Login";
 import { parseId, unwrap } from "Util";
+import { System } from "System";
 
 import messages from "./messages";
 

packages/app/src/State/Users/Db.ts

@@ -2,9 +2,8 @@ import { HexKey } from "@snort/nostr";
 import { db as idb } from "Db";
 
 import { UsersDb, MetadataCache, setUsers } from "State/Users";
-import store, { RootState } from "State/Store";
-import { useSelector } from "react-redux";
-import { unwrap } from "Util";
+import store from "State/Store";
+import { groupByPubkey, unixNowMs, unwrap } from "Util";
 
 class IndexedUsersDb implements UsersDb {
   ready = false;
@@ -63,14 +62,12 @@ class IndexedUsersDb implements UsersDb {
     await idb.users.bulkPut(users);
   }
 
-  async update(key: HexKey, fields: Record<string, string>) {
+  async update(key: HexKey, fields: Record<string, string | number>) {
     await idb.users.update(key, fields);
   }
 }
 
-function groupByPubkey(acc: Record<HexKey, MetadataCache>, user: MetadataCache) {
-  return { ...acc, [user.pubkey]: user };
-}
+export const IndexedUDB = new IndexedUsersDb();
 
 class ReduxUsersDb implements UsersDb {
   async isAvailable() {
@@ -91,22 +88,49 @@ class ReduxUsersDb implements UsersDb {
     });
   }
 
+  querySync(q: string) {
+    const state = store.getState();
+    const { users } = state.users;
+    return Object.values(users).filter(user => {
+      const profile = user as MetadataCache;
+      return (
+        profile.name?.includes(q) ||
+        profile.npub?.includes(q) ||
+        profile.display_name?.includes(q) ||
+        profile.nip05?.includes(q)
+      );
+    });
+  }
+
   async find(key: HexKey) {
     const state = store.getState();
     const { users } = state.users;
-    return users[key];
+    let ret: MetadataCache | undefined = users[key];
+    if (IndexedUDB.ready && ret === undefined) {
+      ret = await IndexedUDB.find(key);
+      if (ret) {
+        await this.put(ret);
+      }
+    }
+    return ret;
   }
 
   async add(user: MetadataCache) {
     const state = store.getState();
     const { users } = state.users;
     store.dispatch(setUsers({ ...users, [user.pubkey]: user }));
+    if (IndexedUDB.ready) {
+      await IndexedUDB.add(user);
+    }
   }
 
   async put(user: MetadataCache) {
     const state = store.getState();
     const { users } = state.users;
     store.dispatch(setUsers({ ...users, [user.pubkey]: user }));
+    if (IndexedUDB.ready) {
+      await IndexedUDB.put(user);
+    }
   }
 
   async bulkAdd(newUserProfiles: MetadataCache[]) {
@@ -114,23 +138,43 @@ class ReduxUsersDb implements UsersDb {
     const { users } = state.users;
     const newUsers = newUserProfiles.reduce(groupByPubkey, {});
     store.dispatch(setUsers({ ...users, ...newUsers }));
+    if (IndexedUDB.ready) {
+      await IndexedUDB.bulkAdd(newUserProfiles);
+    }
   }
 
   async bulkGet(keys: HexKey[]) {
     const state = store.getState();
     const { users } = state.users;
     const ids = new Set([...keys]);
-    return Object.values(users).filter(user => {
+    let ret = Object.values(users).filter(user => {
       return ids.has(user.pubkey);
     });
+    if (IndexedUDB.ready && ret.length !== ids.size) {
+      const startLoad = unixNowMs();
+      const hasKeys = new Set(Object.keys(users));
+      const missing = [...ids].filter(a => !hasKeys.has(a));
+      const missingFromCache = await IndexedUDB.bulkGet(missing);
+      store.dispatch(setUsers({ ...users, ...missingFromCache.reduce(groupByPubkey, {}) }));
+      console.debug(
+        `Loaded ${missingFromCache.length}/${missing.length} profiles from cache in ${(unixNowMs() - startLoad).toFixed(
+          1
+        )} ms`
+      );
+      ret = [...ret, ...missingFromCache];
+    }
+    return ret;
   }
 
-  async update(key: HexKey, fields: Record<string, string>) {
+  async update(key: HexKey, fields: Record<string, string | number>) {
     const state = store.getState();
     const { users } = state.users;
     const current = users[key];
     const updated = { ...current, ...fields };
     store.dispatch(setUsers({ ...users, [key]: updated }));
+    if (IndexedUDB.ready) {
+      await IndexedUDB.update(key, fields);
+    }
   }
 
   async bulkPut(newUsers: MetadataCache[]) {
@@ -138,18 +182,10 @@ class ReduxUsersDb implements UsersDb {
     const { users } = state.users;
     const newProfiles = newUsers.reduce(groupByPubkey, {});
     store.dispatch(setUsers({ ...users, ...newProfiles }));
+    if (IndexedUDB.ready) {
+      await IndexedUDB.bulkPut(newUsers);
+    }
   }
 }
 
-export const IndexedUDB = new IndexedUsersDb();
 export const ReduxUDB = new ReduxUsersDb();
-
-export function useDb(): UsersDb {
-  const db = useSelector((s: RootState) => s.login.useDb);
-  switch (db) {
-    case "indexdDb":
-      return IndexedUDB;
-    default:
-      return ReduxUDB;
-  }
-}

packages/app/src/State/Users/Hooks.ts

@@ -1,50 +1,20 @@
 import { useSelector } from "react-redux";
-import { useLiveQuery } from "dexie-react-hooks";
 import { MetadataCache } from "State/Users";
 import type { RootState } from "State/Store";
 import { HexKey } from "@snort/nostr";
-import { useDb } from "./Db";
+import { ReduxUDB } from "./Db";
 
 export function useQuery(query: string) {
-  const db = useDb();
-  return useLiveQuery(async () => db.query(query), [query]);
+  // TODO: not observable
+  return ReduxUDB.querySync(query);
 }
 
 export function useKey(pubKey?: HexKey) {
-  const db = useDb();
   const { users } = useSelector((state: RootState) => state.users);
-  const defaultUser = pubKey ? users[pubKey] : undefined;
-
-  const user = useLiveQuery(async () => {
-    if (pubKey) {
-      try {
-        return await db.find(pubKey);
-      } catch (error) {
-        console.error(error);
-        return defaultUser;
-      }
-    }
-  }, [pubKey, defaultUser]);
-
-  return user;
+  return pubKey ? users[pubKey] : undefined;
 }
 
 export function useKeys(pubKeys?: HexKey[]): Map<HexKey, MetadataCache> {
-  const db = useDb();
   const { users } = useSelector((state: RootState) => state.users);
-
-  const dbUsers = useLiveQuery(async () => {
-    if (pubKeys) {
-      try {
-        const ret = await db.bulkGet(pubKeys);
-        return new Map(ret.map(a => [a.pubkey, a]));
-      } catch (error) {
-        console.error(error);
-        return new Map(pubKeys.map(a => [a, users[a]]));
-      }
-    }
-    return new Map();
-  }, [pubKeys, users]);
-
-  return dbUsers ?? new Map();
+  return new Map((pubKeys ?? []).map(a => [a, users[a]]));
 }

packages/app/src/System.ts

@@ -0,0 +1,235 @@
+import { AuthHandler, HexKey, TaggedRawEvent } from "@snort/nostr";
+
+import { Event as NEvent, EventKind, RelaySettings, Connection, Subscriptions } from "@snort/nostr";
+import { ProfileCacheExpire } from "Const";
+import { mapEventToProfile } from "State/Users";
+import { ReduxUDB } from "State/Users/Db";
+import { unwrap } from "Util";
+
+/**
+ * Manages nostr content retrieval system
+ */
+export class NostrSystem {
+  /**
+   * All currently connected websockets
+   */
+  Sockets: Map<string, Connection>;
+
+  /**
+   * All active subscriptions
+   */
+  Subscriptions: Map<string, Subscriptions>;
+
+  /**
+   * Pending subscriptions to send when sockets become open
+   */
+  PendingSubscriptions: Subscriptions[];
+
+  /**
+   * List of pubkeys to fetch metadata for
+   */
+  WantsMetadata: Set<HexKey>;
+
+  /**
+   * Handler function for NIP-42
+   */
+  HandleAuth?: AuthHandler;
+
+  constructor() {
+    this.Sockets = new Map();
+    this.Subscriptions = new Map();
+    this.PendingSubscriptions = [];
+    this.WantsMetadata = new Set();
+    this._FetchMetadata();
+  }
+
+  /**
+   * Connect to a NOSTR relay if not already connected
+   */
+  async ConnectToRelay(address: string, options: RelaySettings) {
+    try {
+      if (!this.Sockets.has(address)) {
+        const c = new Connection(address, options, this.HandleAuth);
+        await c.Connect();
+        this.Sockets.set(address, c);
+        for (const [, s] of this.Subscriptions) {
+          c.AddSubscription(s);
+        }
+      } else {
+        // update settings if already connected
+        unwrap(this.Sockets.get(address)).Settings = options;
+      }
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  /**
+   * Disconnect from a relay
+   */
+  DisconnectRelay(address: string) {
+    const c = this.Sockets.get(address);
+    if (c) {
+      this.Sockets.delete(address);
+      c.Close();
+    }
+  }
+
+  AddSubscriptionToRelay(sub: Subscriptions, relay: string) {
+    this.Sockets.get(relay)?.AddSubscription(sub);
+  }
+
+  AddSubscription(sub: Subscriptions) {
+    for (const [, s] of this.Sockets) {
+      s.AddSubscription(sub);
+    }
+    this.Subscriptions.set(sub.Id, sub);
+  }
+
+  RemoveSubscription(subId: string) {
+    for (const [, s] of this.Sockets) {
+      s.RemoveSubscription(subId);
+    }
+    this.Subscriptions.delete(subId);
+  }
+
+  /**
+   * Send events to writable relays
+   */
+  BroadcastEvent(ev: NEvent) {
+    for (const [, s] of this.Sockets) {
+      s.SendEvent(ev);
+    }
+  }
+
+  /**
+   * Write an event to a relay then disconnect
+   */
+  async WriteOnceToRelay(address: string, ev: NEvent) {
+    const c = new Connection(address, { write: true, read: false }, this.HandleAuth);
+    await c.Connect();
+    await c.SendAsync(ev);
+    c.Close();
+  }
+
+  /**
+   * Request profile metadata for a set of pubkeys
+   */
+  TrackMetadata(pk: HexKey | Array<HexKey>) {
+    for (const p of Array.isArray(pk) ? pk : [pk]) {
+      if (p.length > 0) {
+        this.WantsMetadata.add(p);
+      }
+    }
+  }
+
+  /**
+   * Stop tracking metadata for a set of pubkeys
+   */
+  UntrackMetadata(pk: HexKey | Array<HexKey>) {
+    for (const p of Array.isArray(pk) ? pk : [pk]) {
+      if (p.length > 0) {
+        this.WantsMetadata.delete(p);
+      }
+    }
+  }
+
+  /**
+   * Request/Response pattern
+   */
+  RequestSubscription(sub: Subscriptions) {
+    return new Promise<TaggedRawEvent[]>(resolve => {
+      const events: TaggedRawEvent[] = [];
+
+      // force timeout returning current results
+      const timeout = setTimeout(() => {
+        this.RemoveSubscription(sub.Id);
+        resolve(events);
+      }, 10_000);
+
+      const onEventPassthrough = sub.OnEvent;
+      sub.OnEvent = ev => {
+        if (typeof onEventPassthrough === "function") {
+          onEventPassthrough(ev);
+        }
+        if (!events.some(a => a.id === ev.id)) {
+          events.push(ev);
+        } else {
+          const existing = events.find(a => a.id === ev.id);
+          if (existing) {
+            for (const v of ev.relays) {
+              existing.relays.push(v);
+            }
+          }
+        }
+      };
+      sub.OnEnd = c => {
+        c.RemoveSubscription(sub.Id);
+        if (sub.IsFinished()) {
+          clearInterval(timeout);
+          console.debug(`[${sub.Id}] Finished`);
+          resolve(events);
+        }
+      };
+      this.AddSubscription(sub);
+    });
+  }
+
+  async _FetchMetadata() {
+    const userDb = ReduxUDB;
+
+    const missing = new Set<HexKey>();
+    const meta = await userDb.bulkGet(Array.from(this.WantsMetadata));
+    const expire = new Date().getTime() - ProfileCacheExpire;
+    for (const pk of this.WantsMetadata) {
+      const m = meta.find(a => a.pubkey === pk);
+      if ((m?.loaded ?? 0) < expire) {
+        missing.add(pk);
+        // cap 100 missing profiles
+        if (missing.size >= 100) {
+          break;
+        }
+      }
+    }
+
+    if (missing.size > 0) {
+      console.debug("Wants profiles: ", missing);
+
+      const sub = new Subscriptions();
+      sub.Id = `profiles:${sub.Id.slice(0, 8)}`;
+      sub.Kinds = new Set([EventKind.SetMetadata]);
+      sub.Authors = missing;
+      sub.OnEvent = async e => {
+        const profile = mapEventToProfile(e);
+        if (profile) {
+          const existing = await userDb.find(profile.pubkey);
+          if ((existing?.created ?? 0) < profile.created) {
+            await userDb.put(profile);
+          } else if (existing) {
+            await userDb.update(profile.pubkey, {
+              loaded: profile.loaded,
+            });
+          }
+        }
+      };
+      const results = await this.RequestSubscription(sub);
+      const couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a));
+      console.debug("No profiles: ", couldNotFetch);
+      if (couldNotFetch.length > 0) {
+        const updates = couldNotFetch
+          .map(a => {
+            return {
+              pubkey: a,
+              loaded: new Date().getTime(),
+            };
+          })
+          .map(a => userDb.update(a.pubkey, a));
+        await Promise.all(updates);
+      }
+    }
+
+    setTimeout(() => this._FetchMetadata(), 500);
+  }
+}
+
+export const System = new NostrSystem();

packages/app/src/Util.ts

@@ -2,6 +2,7 @@ import * as secp from "@noble/secp256k1";
 import { sha256 as hash } from "@noble/hashes/sha256";
 import { bech32 } from "bech32";
 import { HexKey, TaggedRawEvent, u256, EventKind, encodeTLV, NostrPrefix } from "@snort/nostr";
+import { MetadataCache } from "State/Users";
 
 export const sha256 = (str: string) => {
   return secp.utils.bytesToHex(hash(str));
@@ -144,7 +145,11 @@ export function extractLnAddress(lnurl: string) {
 }
 
 export function unixNow() {
-  return Math.floor(new Date().getTime() / 1000);
+  return Math.floor(unixNowMs() / 1000);
+}
+
+export function unixNowMs() {
+  return new Date().getTime();
 }
 
 /**
@@ -196,3 +201,7 @@ export function tagFilterOfTextRepost(note: TaggedRawEvent, id?: u256): (tag: st
   return (tag, i) =>
     tag[0] === "e" && tag[3] === "mention" && note.content === `#[${i}]` && (id ? tag[1] === id : true);
 }
+
+export function groupByPubkey(acc: Record<HexKey, MetadataCache>, user: MetadataCache) {
+  return { ...acc, [user.pubkey]: user };
+}