use @snort/system cache

packages/system/README.md

@@ -0,0 +1,66 @@
+## @snort/system
+
+A collection of caching and querying techniquies used by https://snort.social to serve all content from the nostr protocol.
+
+Simple example:
+```js
+import { 
+    NostrSystem, 
+    EventPublisher, 
+    UserRelaysCache, 
+    RequestBuilder, 
+    FlatNoteStore, 
+    StoreSnapshot 
+} from "@snort/system"
+
+// Provided in-memory / indexedDb cache for relays
+// You can also implement your own with "RelayCache" interface
+const RelaysCache = new UserRelaysCache();
+
+// example auth handler using NIP-07
+const AuthHandler = async (challenge: string, relay: string) => {
+    const pub = await EventPublisher.nip7();
+    if (pub) {
+        return await pub.nip42Auth(challenge, relay);
+    }
+}
+
+// Singleton instance to store all connections and access query fetching system
+const System = new NostrSystem({
+    relayCache: RelaysCache,
+    authHandler: AuthHandler // can be left undefined if you dont care about NIP-42 Auth
+});
+
+(async () => {
+    // connec to one "bootstrap" relay to pull profiles/relay lists from
+    // also used as a fallback relay when gossip model doesnt know which relays to pick, or "authors" are not provided in the request
+    await System.ConnectToRelay("wss://relay.snort.social", { read: true, write: false });
+
+    // ID should be unique to the use case, this is important as all data fetched from this ID will be merged into the same NoteStore
+    const rb = new RequestBuilder("get-posts");
+    rb.withFilter()
+        .authors(["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"]) // Kieran pubkey
+        .kinds([1])
+        .limit(10);
+
+    const q = System.Query<FlatNoteStore>(FlatNoteStore, rb);
+    // basic usage using "onEvent", fired for every event added to the store
+    q.onEvent = (sub, e) => {
+        console.debug(sub, e);
+    }
+
+    // Hookable type using change notification, limited to every 500ms
+    const release = q.feed.hook(() => {
+        // since we use the FlatNoteStore we expect NostrEvent[]
+        // other stores provide different data, like a single event instead of an array (latest version)
+        const state = q.feed.snapshot as StoreSnapshot<ReturnType<FlatNoteStore["getSnapshotData"]>>;
+
+        // do something with snapshot of store
+        console.log(`We have ${state.data.length} events now!`)
+    });
+
+    // release the hook when its not needed anymore
+    // these patterns will be managed in @snort/system-react to make it easier to use react or other UI frameworks
+    // release();
+})();
+```

packages/system/examples/simple.ts

@@ -0,0 +1,52 @@
+import { NostrSystem, EventPublisher, UserRelaysCache, RequestBuilder, FlatNoteStore, StoreSnapshot } from "../src"
+
+// Provided in-memory / indexedDb cache for relays
+// You can also implement your own with "RelayCache" interface
+const RelaysCache = new UserRelaysCache();
+
+// example auth handler using NIP-07
+const AuthHandler = async (challenge: string, relay: string) => {
+    const pub = await EventPublisher.nip7();
+    if (pub) {
+        return await pub.nip42Auth(challenge, relay);
+    }
+}
+
+// Singleton instance to store all connections and access query fetching system
+const System = new NostrSystem({
+    relayCache: RelaysCache,
+    authHandler: AuthHandler // can be left undefined if you dont care about NIP-42 Auth
+});
+
+(async () => {
+    // connec to one "bootstrap" relay to pull profiles/relay lists from
+    // also used as a fallback relay when gossip model doesnt know which relays to pick, or "authors" are not provided in the request
+    await System.ConnectToRelay("wss://relay.snort.social", { read: true, write: false });
+
+    // ID should be unique to the use case, this is important as all data fetched from this ID will be merged into the same NoteStore
+    const rb = new RequestBuilder("get-posts");
+    rb.withFilter()
+        .authors(["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"]) // Kieran pubkey
+        .kinds([1])
+        .limit(10);
+
+    const q = System.Query<FlatNoteStore>(FlatNoteStore, rb);
+    // basic usage using "onEvent", fired for every event added to the store
+    q.onEvent = (sub, e) => {
+        console.debug(sub, e);
+    }
+
+    // Hookable type using change notification, limited to every 500ms
+    const release = q.feed.hook(() => {
+        // since we use the FlatNoteStore we expect NostrEvent[]
+        // other stores provide different data, like a single event instead of an array (latest version)
+        const state = q.feed.snapshot as StoreSnapshot<ReturnType<FlatNoteStore["getSnapshotData"]>>;
+
+        // do something with snapshot of store
+        console.log(`We have ${state.data.length} events now!`)
+    });
+
+    // release the hook when its not needed anymore
+    // these patterns will be managed in @snort/system-react to make it easier to use react or other UI frameworks
+    // release();
+})();

packages/system/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snort/system",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Snort nostr system package",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -30,6 +30,7 @@
     "@stablelib/xchacha20": "^1.0.1",
     "bech32": "^2.0.0",
     "debug": "^4.3.4",
+    "dexie": "^3.2.4",
     "uuid": "^9.0.0"
   }
 }

packages/system/src/Connection.ts

@@ -1,11 +1,10 @@
 import { v4 as uuid } from "uuid";
+import { unwrap, ExternalStore } from "@snort/shared";
 
 import { DefaultConnectTimeout } from "./Const";
 import { ConnectionStats } from "./ConnectionStats";
 import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
 import { RelayInfo } from "./RelayInfo";
-import { unwrap } from "./Utils";
-import ExternalStore from "./ExternalStore";
 
 export type AuthHandler = (challenge: string, relay: string) => Promise<NostrEvent | undefined>;
 

packages/system/src/Const.ts

@@ -13,4 +13,4 @@ export const HashtagRegex = /(#[^\s!@#$%^&*()=+.\/,\[{\]};:'"?><]+)/g;
 /**
  * How long profile cache should be considered valid for
  */
- export const ProfileCacheExpire = 1_000 * 60 * 60 * 6;
+export const ProfileCacheExpire = 1_000 * 60 * 60 * 6;

packages/system/src/EventBuilder.ts

@@ -1,6 +1,6 @@
 import { EventKind, HexKey, NostrPrefix, NostrEvent } from ".";
 import { HashtagRegex } from "./Const";
-import { getPublicKey, unixNow } from "./Utils";
+import { getPublicKey, unixNow } from "@snort/shared";
 import { EventExt } from "./EventExt";
 import { tryParseNostrLink } from "./NostrLink";
 

packages/system/src/EventExt.ts

@@ -1,8 +1,8 @@
 import * as secp from "@noble/curves/secp256k1";
 import * as utils from "@noble/curves/abstract/utils";
+import { sha256, unixNow } from "@snort/shared";
 
 import { EventKind, HexKey, NostrEvent } from ".";
-import { sha256, unixNow } from "./Utils";
 import { Nip4WebCryptoEncryptor } from "./impl/nip4";
 
 export interface Tag {

packages/system/src/EventPublisher.ts

@@ -1,5 +1,7 @@
 import * as secp from "@noble/curves/secp256k1";
 import * as utils from "@noble/curves/abstract/utils";
+import { unwrap, barrierQueue, processWorkQueue, WorkQueueItem } from "@snort/shared";
+
 import {
   EventKind,
   FullRelaySettings,
@@ -7,16 +9,13 @@ import {
   Lists,
   NostrEvent,
   RelaySettings,
-  SystemInterface,
   TaggedRawEvent,
   u256,
   UserMetadata,
 } from ".";
 
-import { unwrap } from "./Utils";
 import { EventBuilder } from "./EventBuilder";
 import { EventExt } from "./EventExt";
-import { barrierQueue, processWorkQueue, WorkQueueItem } from "./WorkQueue";
 
 const Nip7Queue: Array<WorkQueueItem> = [];
 processWorkQueue(Nip7Queue);
@@ -39,12 +38,10 @@ declare global {
 }
 
 export class EventPublisher {
-  #system: SystemInterface;
   #pubKey: string;
   #privateKey?: string;
 
-  constructor(system: SystemInterface, pubKey: string, privKey?: string) {
-    this.#system = system;
+  constructor(pubKey: string, privKey?: string) {
     if (privKey) {
       this.#privateKey = privKey;
       this.#pubKey = utils.bytesToHex(secp.schnorr.getPublicKey(privKey));
@@ -57,6 +54,18 @@ export class EventPublisher {
     return "nostr" in window;
   }
 
+  /**
+   * Get a NIP-07 EventPublisher
+   */
+  static async nip7() {
+    if("nostr" in window) {
+      const pubkey = await window.nostr?.getPublicKey();
+      if(pubkey) {
+        return new EventPublisher(pubkey);
+      }
+    }
+  }
+
   #eb(k: EventKind) {
     const eb = new EventBuilder();
     return eb.pubKey(this.#pubKey).kind(k);
@@ -112,20 +121,6 @@ export class EventPublisher {
     return await this.#sign(eb);
   }
 
-  broadcast(ev: NostrEvent) {
-    console.debug(ev);
-    this.#system.BroadcastEvent(ev);
-  }
-
-  /**
-   * Write event to all given relays.
-   */
-  broadcastAll(ev: NostrEvent, relays: string[]) {
-    for (const k of relays) {
-      this.#system.WriteOnceToRelay(k, ev);
-    }
-  }
-
   async muted(keys: HexKey[], priv: HexKey[]) {
     const eb = this.#eb(EventKind.PubkeyLists);
 

packages/system/src/ExternalStore.ts

@@ -1,41 +0,0 @@
-type HookFn<TSnapshot> = (e?: TSnapshot) => void;
-
-interface HookFilter<TSnapshot> {
-  fn: HookFn<TSnapshot>;
-}
-
-/**
- * Simple React hookable store with manual change notifications
- */
-export default abstract class ExternalStore<TSnapshot> {
-  #hooks: Array<HookFilter<TSnapshot>> = [];
-  #snapshot: Readonly<TSnapshot> = {} as Readonly<TSnapshot>;
-  #changed = true;
-
-  hook(fn: HookFn<TSnapshot>) {
-    this.#hooks.push({
-      fn,
-    });
-    return () => {
-      const idx = this.#hooks.findIndex(a => a.fn === fn);
-      if (idx >= 0) {
-        this.#hooks.splice(idx, 1);
-      }
-    };
-  }
-
-  snapshot() {
-    if (this.#changed) {
-      this.#snapshot = this.takeSnapshot();
-      this.#changed = false;
-    }
-    return this.#snapshot;
-  }
-
-  protected notifyChange(sn?: TSnapshot) {
-    this.#changed = true;
-    this.#hooks.forEach(h => h.fn(sn));
-  }
-
-  abstract takeSnapshot(): TSnapshot;
-}

packages/system/src/GossipModel.ts

@@ -1,5 +1,5 @@
-import { FullRelaySettings, ReqFilter } from ".";
-import { unwrap } from "./Utils";
+import { ReqFilter, UsersRelays } from ".";
+import { unwrap } from "@snort/shared";
 import debug from "debug";
 
 const PickNRelays = 2;
@@ -15,7 +15,7 @@ export interface RelayTaggedFilters {
 }
 
 export interface RelayCache {
-  get(pubkey?: string): Array<FullRelaySettings> | undefined;
+  getFromCache(pubkey?: string): UsersRelays | undefined;
 }
 
 export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilter>) {
@@ -59,7 +59,7 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<
   const allRelays = unwrap(filter.authors).map(a => {
     return {
       key: a,
-      relays: cache.get(a)?.filter(a => a.settings.write).sort(() => Math.random() < 0.5 ? 1 : -1),
+      relays: cache.getFromCache(a)?.relays?.filter(a => a.settings.write).sort(() => Math.random() < 0.5 ? 1 : -1),
     };
   });
 

packages/system/src/Links.ts

@@ -1,5 +1,5 @@
 import * as utils from "@noble/curves/abstract/utils";
-import { bech32 } from "bech32";
+import { bech32 } from "@scure/base";
 import { HexKey } from "./Nostr";
 
 export enum NostrPrefix {
@@ -43,7 +43,7 @@ export function encodeTLV(prefix: NostrPrefix, id: string, relays?: string[], ki
   const tl2 = author ? [2, 32, ...utils.hexToBytes(author)] : [];
   const tl3 = kind ? [3, 4, ...new Uint8Array(new Uint32Array([kind]).buffer).reverse()] : [];
 
-  return bech32.encode(prefix, bech32.toWords([...tl0, ...tl1, ...tl2, ...tl3]), 1_000);
+  return bech32.encode(prefix, bech32.toWords(new Uint8Array([...tl0, ...tl1, ...tl2, ...tl3])), 1_000);
 }
 
 export function decodeTLV(str: string) {

packages/system/src/NostrLink.ts

@@ -1,4 +1,4 @@
-import { bech32ToHex, hexToBech32 } from "./Utils";
+import { bech32ToHex, hexToBech32 } from "@snort/shared";
 import { NostrPrefix, decodeTLV, TLVEntryType } from ".";
 
 export interface NostrLink {

packages/system/src/NostrSystem.ts

@@ -1,19 +1,20 @@
 import debug from "debug";
 
-import ExternalStore from "./ExternalStore";
+import { unwrap, sanitizeRelayUrl, ExternalStore } from "@snort/shared";
 import { NostrEvent, TaggedRawEvent } from "./Nostr";
 import { AuthHandler, Connection, RelaySettings, ConnectionStateSnapshot } from "./Connection";
 import { Query } from "./Query";
 import { RelayCache } from "./GossipModel";
 import { NoteStore } from "./NoteCollection";
 import { BuiltRawReqFilter, RequestBuilder } from "./RequestBuilder";
-import { unwrap, sanitizeRelayUrl } from "./Utils";
 import { SystemInterface, SystemSnapshot } from ".";
 
 /**
  * Manages nostr content retrieval system
  */
 export class NostrSystem extends ExternalStore<SystemSnapshot> implements SystemInterface {
+  #log = debug("System");
+
   /**
    * All currently connected websockets
    */
@@ -25,16 +26,19 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
   Queries: Map<string, Query> = new Map();
 
   /**
-   * Handler function for NIP-42
+   * NIP-42 Auth handler
    */
-  HandleAuth?: AuthHandler;
+  #handleAuth?: AuthHandler;
 
-  #log = debug("System");
+  /**
+   * Storage class for user relay lists
+   */
   #relayCache: RelayCache;
 
-  constructor(relayCache: RelayCache) {
+  constructor(props: { authHandler?: AuthHandler, relayCache: RelayCache }) {
     super();
-    this.#relayCache = relayCache;
+    this.#handleAuth = props.authHandler;
+    this.#relayCache = props.relayCache;
     this.#cleanup();
   }
 
@@ -49,7 +53,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
     try {
       const addr = unwrap(sanitizeRelayUrl(address));
       if (!this.#sockets.has(addr)) {
-        const c = new Connection(addr, options, this.HandleAuth?.bind(this));
+        const c = new Connection(addr, options, this.#handleAuth?.bind(this));
         this.#sockets.set(addr, c);
         c.OnEvent = (s, e) => this.OnEvent(s, e);
         c.OnEose = s => this.OnEndOfStoredEvents(c, s);
@@ -90,7 +94,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
     try {
       const addr = unwrap(sanitizeRelayUrl(address));
       if (!this.#sockets.has(addr)) {
-        const c = new Connection(addr, { read: true, write: false }, this.HandleAuth?.bind(this), true);
+        const c = new Connection(addr, { read: true, write: false }, this.#handleAuth?.bind(this), true);
         this.#sockets.set(addr, c);
         c.OnEvent = (s, e) => this.OnEvent(s, e);
         c.OnEose = s => this.OnEndOfStoredEvents(c, s);
@@ -200,7 +204,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
    */
   async WriteOnceToRelay(address: string, ev: NostrEvent) {
     return new Promise<void>((resolve, reject) => {
-      const c = new Connection(address, { write: true, read: false }, this.HandleAuth, true);
+      const c = new Connection(address, { write: true, read: false }, this.#handleAuth?.bind(this), true);
 
       const t = setTimeout(reject, 5_000);
       c.OnConnected = async () => {

packages/system/src/NoteCollection.ts

@@ -1,5 +1,6 @@
+import { appendDedupe } from "@snort/shared";
 import { TaggedRawEvent, u256 } from ".";
-import { appendDedupe, findTag } from "./Utils";
+import { findTag } from "./Utils";
 
 export interface StoreSnapshot<TSnapshot> {
   data: TSnapshot | undefined;

packages/system/src/ProfileCache.ts

@@ -1,12 +1,13 @@
+
+import debug from "debug";
+import { unixNowMs, FeedCache } from "@snort/shared";
 import { EventKind, HexKey, SystemInterface, TaggedRawEvent, PubkeyReplaceableNoteStore, RequestBuilder } from ".";
 import { ProfileCacheExpire } from "./Const";
-import { CacheStore, mapEventToProfile, MetadataCache } from "./cache";
-import { unixNowMs } from "./Utils";
-import debug from "debug";
+import { mapEventToProfile, MetadataCache } from "./cache";
 
 export class ProfileLoaderService {
   #system: SystemInterface;
-  #cache: CacheStore<MetadataCache>;
+  #cache: FeedCache<MetadataCache>;
 
   /**
    * List of pubkeys to fetch metadata for
@@ -15,7 +16,7 @@ export class ProfileLoaderService {
 
   readonly #log = debug("ProfileCache");
 
-  constructor(system: SystemInterface, cache: CacheStore<MetadataCache>) {
+  constructor(system: SystemInterface, cache: FeedCache<MetadataCache>) {
     this.#system = system;
     this.#cache = cache;
     this.#FetchMetadata();

packages/system/src/Query.ts

@@ -1,7 +1,9 @@
 import { v4 as uuid } from "uuid";
 import debug from "debug";
+import { unixNowMs, unwrap } from "@snort/shared";
+
 import { Connection, ReqFilter, Nips, TaggedRawEvent } from ".";
-import { reqFilterEq, unixNowMs, unwrap } from "./Utils";
+import { reqFilterEq } from "./Utils";
 import { NoteStore } from "./NoteCollection";
 import { flatMerge } from "./RequestMerger";
 import { BuiltRawReqFilter } from "./RequestBuilder";

packages/system/src/RequestBuilder.ts

@@ -1,11 +1,12 @@
+import debug from "debug";
+import { v4 as uuid } from "uuid";
+import { appendDedupe, dedupe, unixNowMs } from "@snort/shared";
+
 import { ReqFilter, u256, HexKey, EventKind } from ".";
-import { appendDedupe, dedupe, unixNowMs } from "./Utils";
 import { diffFilters } from "./RequestSplitter";
 import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel";
 import { mergeSimilar } from "./RequestMerger";
 import { FlatReqFilter, expandFilter } from "./RequestExpander";
-import debug from "debug";
-import { v4 as uuid } from "uuid";
 
 /**
  * Which strategy is used when building REQ filters

packages/system/src/RequestMerger.ts

@@ -1,4 +1,4 @@
-import { distance } from "./Utils";
+import { distance } from "@snort/shared";
 import { ReqFilter } from ".";
 import { FlatReqFilter } from "./RequestExpander";
 

packages/system/src/SystemWorker.ts

@@ -1,6 +1,7 @@
+import { ExternalStore } from "@snort/shared";
+
 import { SystemSnapshot, SystemInterface } from ".";
 import { AuthHandler, ConnectionStateSnapshot, RelaySettings } from "./Connection";
-import ExternalStore from "./ExternalStore";
 import { NostrEvent } from "./Nostr";
 import { NoteStore } from "./NoteCollection";
 import { Query } from "./Query";

packages/system/src/Utils.ts

@@ -1,179 +1,42 @@
-import * as utils from "@noble/curves/abstract/utils";
-import * as secp from "@noble/curves/secp256k1";
-import { sha256 as sha2 } from "@noble/hashes/sha256";
-import { bech32 } from "bech32";
-import { NostrEvent, ReqFilter, u256 } from "./Nostr";
-import { FlatReqFilter } from "RequestExpander";
 
-export function unwrap<T>(v: T | undefined | null): T {
-  if (v === undefined || v === null) {
-    throw new Error("missing value");
-  }
-  return v;
-}
+import { equalProp } from "@snort/shared";
+import { FlatReqFilter } from "./RequestExpander";
+import { NostrEvent, ReqFilter } from "./Nostr";
 
-/**
- * Convert hex to bech32
- */
-export function hexToBech32(hrp: string, hex?: string) {
-  if (typeof hex !== "string" || hex.length === 0 || hex.length % 2 !== 0) {
-    return "";
-  }
-
-  try {
-    const buf = utils.hexToBytes(hex);
-    return bech32.encode(hrp, bech32.toWords(buf));
-  } catch (e) {
-    console.warn("Invalid hex", hex, e);
-    return "";
-  }
-}
-
-export function sanitizeRelayUrl(url: string) {
-  try {
-    return new URL(url).toString();
-  } catch {
-    // ignore
-  }
-}
-
-export function unixNow() {
-  return Math.floor(unixNowMs() / 1000);
-}
-
-export function unixNowMs() {
-  return new Date().getTime();
-}
-
-export function deepEqual(x: any, y: any): boolean {
-  const ok = Object.keys,
-    tx = typeof x,
-    ty = typeof y;
-
-  return x && y && tx === "object" && tx === ty
-    ? ok(x).length === ok(y).length && ok(x).every(key => deepEqual(x[key], y[key]))
-    : x === y;
+export function findTag(e: NostrEvent, tag: string) {
+    const maybeTag = e.tags.find(evTag => {
+        return evTag[0] === tag;
+    });
+    return maybeTag && maybeTag[1];
 }
 
 export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
-  return equalProp(a.ids, b.ids)
-    && equalProp(a.kinds, b.kinds)
-    && equalProp(a.authors, b.authors)
-    && equalProp(a.limit, b.limit)
-    && equalProp(a.since, b.since)
-    && equalProp(a.until, b.until)
-    && equalProp(a.search, b.search)
-    && equalProp(a["#e"], b["#e"])
-    && equalProp(a["#p"], b["#p"])
-    && equalProp(a["#t"], b["#t"])
-    && equalProp(a["#d"], b["#d"])
-    && equalProp(a["#r"], b["#r"]);
+    return equalProp(a.ids, b.ids)
+        && equalProp(a.kinds, b.kinds)
+        && equalProp(a.authors, b.authors)
+        && equalProp(a.limit, b.limit)
+        && equalProp(a.since, b.since)
+        && equalProp(a.until, b.until)
+        && equalProp(a.search, b.search)
+        && equalProp(a["#e"], b["#e"])
+        && equalProp(a["#p"], b["#p"])
+        && equalProp(a["#t"], b["#t"])
+        && equalProp(a["#d"], b["#d"])
+        && equalProp(a["#r"], b["#r"]);
 }
 
 export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
-  return a.keys === b.keys
-    && a.since === b.since
-    && a.until === b.until
-    && a.limit === b.limit
-    && a.search === b.search
-    && a.ids === b.ids
-    && a.kinds === b.kinds
-    && a.authors === b.authors
-    && a["#e"] === b["#e"]
-    && a["#p"] === b["#p"]
-    && a["#t"] === b["#t"]
-    && a["#d"] === b["#d"]
-    && a["#r"] === b["#r"];
-}
-
-export function countMembers(a: any) {
-  let ret = 0;
-  for (const [k, v] of Object.entries(a)) {
-    if (Array.isArray(v)) {
-      ret += v.length;
-    }
-  }
-  return ret;
-}
-
-export function equalProp(a: string | number | Array<string | number> | undefined, b: string | number | Array<string | number> | undefined) {
-  if ((a !== undefined && b === undefined) || (a === undefined && b !== undefined)) {
-    return false;
-  }
-  if (Array.isArray(a) && Array.isArray(b)) {
-    if (a.length !== b.length) {
-      return false;
-    }
-    if (!a.every(v => b.includes(v))) {
-      return false;
-    }
-  }
-  return a === b;
-}
-
-/**
- * Compute the "distance" between two objects by comparing their difference in properties
- * Missing/Added keys result in +10 distance
- * This is not recursive
- */
-export function distance(a: any, b: any): number {
-  const keys1 = Object.keys(a);
-  const keys2 = Object.keys(b);
-  const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
-
-  let distance = 0;
-  for (const key of maxKeys) {
-    if (key in a && key in b) {
-      if (Array.isArray(a[key]) && Array.isArray(b[key])) {
-        const aa = a[key] as Array<string | number>;
-        const bb = b[key] as Array<string | number>;
-        if (aa.length === bb.length) {
-          if (aa.some(v => !bb.includes(v))) {
-            distance++;
-          }
-        } else {
-          distance++;
-        }
-      } else if (a[key] !== b[key]) {
-        distance++;
-      }
-    } else {
-      distance += 10;
-    }
-  }
-
-  return distance;
-}
-
-export function dedupe<T>(v: Array<T>) {
-  return [...new Set(v)];
-}
-
-export function appendDedupe<T>(a?: Array<T>, b?: Array<T>) {
-  return dedupe([...(a ?? []), ...(b ?? [])]);
-}
-
-export function findTag(e: NostrEvent, tag: string) {
-  const maybeTag = e.tags.find(evTag => {
-    return evTag[0] === tag;
-  });
-  return maybeTag && maybeTag[1];
-}
-
-export const sha256 = (str: string | Uint8Array): u256 => {
-  return utils.bytesToHex(sha2(str));
-}
-
-export function getPublicKey(privKey: string) {
-  return utils.bytesToHex(secp.schnorr.getPublicKey(privKey));
-}
-
-export function bech32ToHex(str: string) {
-  try {
-    const nKey = bech32.decode(str, 1_000);
-    const buff = bech32.fromWords(nKey.words);
-    return utils.bytesToHex(Uint8Array.from(buff));
-  } catch (e) {
-    return str;
-  }
-}
+    return a.keys === b.keys
+        && a.since === b.since
+        && a.until === b.until
+        && a.limit === b.limit
+        && a.search === b.search
+        && a.ids === b.ids
+        && a.kinds === b.kinds
+        && a.authors === b.authors
+        && a["#e"] === b["#e"]
+        && a["#p"] === b["#p"]
+        && a["#t"] === b["#t"]
+        && a["#d"] === b["#d"]
+        && a["#r"] === b["#r"];
+}

packages/system/src/WorkQueue.ts

@@ -1,30 +0,0 @@
-export interface WorkQueueItem {
-  next: () => Promise<unknown>;
-  resolve(v: unknown): void;
-  reject(e: unknown): void;
-}
-
-export async function processWorkQueue(queue?: Array<WorkQueueItem>, queueDelay = 200) {
-  while (queue && queue.length > 0) {
-    const v = queue.shift();
-    if (v) {
-      try {
-        const ret = await v.next();
-        v.resolve(ret);
-      } catch (e) {
-        v.reject(e);
-      }
-    }
-  }
-  setTimeout(() => processWorkQueue(queue, queueDelay), queueDelay);
-}
-
-export const barrierQueue = async <T>(queue: Array<WorkQueueItem>, then: () => Promise<T>): Promise<T> => {
-  return new Promise<T>((resolve, reject) => {
-    queue.push({
-      next: then,
-      resolve,
-      reject,
-    });
-  });
-};

packages/system/src/cache/UserCache.ts

@@ -0,0 +1,148 @@
+import { db, MetadataCache } from ".";
+import { fetchNip05Pubkey, FeedCache, LNURL } from "@snort/shared";
+
+export class UserProfileCache extends FeedCache<MetadataCache> {
+  #zapperQueue: Array<{ pubkey: string; lnurl: string }> = [];
+  #nip5Queue: Array<{ pubkey: string; nip05: string }> = [];
+
+  constructor() {
+    super("UserCache", db.users);
+    this.#processZapperQueue();
+    this.#processNip5Queue();
+  }
+
+  key(of: MetadataCache): string {
+    return of.pubkey;
+  }
+
+  override async preload(follows?: Array<string>): Promise<void> {
+    await super.preload();
+    // load follows profiles
+    if (follows) {
+      await this.buffer(follows);
+    }
+  }
+
+  async search(q: string): Promise<Array<MetadataCache>> {
+    if (db.ready) {
+      // on-disk cache will always have more data
+      return (
+        await db.users
+          .where("npub")
+          .startsWithIgnoreCase(q)
+          .or("name")
+          .startsWithIgnoreCase(q)
+          .or("display_name")
+          .startsWithIgnoreCase(q)
+          .or("nip05")
+          .startsWithIgnoreCase(q)
+          .toArray()
+      ).slice(0, 5);
+    } else {
+      return [...this.cache.values()]
+        .filter(user => {
+          const profile = user as MetadataCache;
+          return (
+            profile.name?.includes(q) ||
+            profile.npub?.includes(q) ||
+            profile.display_name?.includes(q) ||
+            profile.nip05?.includes(q)
+          );
+        })
+        .slice(0, 5);
+    }
+  }
+
+  /**
+   * Try to update the profile metadata cache with a new version
+   * @param m Profile metadata
+   * @returns
+   */
+  override async update(m: MetadataCache) {
+    const updateType = await super.update(m);
+    if (updateType !== "refresh") {
+      const lnurl = m.lud16 ?? m.lud06;
+      if (lnurl) {
+        this.#zapperQueue.push({
+          pubkey: m.pubkey,
+          lnurl,
+        });
+      }
+      if (m.nip05) {
+        this.#nip5Queue.push({
+          pubkey: m.pubkey,
+          nip05: m.nip05,
+        });
+      }
+    }
+    return updateType;
+  }
+
+  takeSnapshot(): MetadataCache[] {
+    return [];
+  }
+
+  async #processZapperQueue() {
+    await this.#batchQueue(
+      this.#zapperQueue,
+      async i => {
+        const svc = new LNURL(i.lnurl);
+        await svc.load();
+        const p = this.getFromCache(i.pubkey);
+        if (p) {
+          await this.set({
+            ...p,
+            zapService: svc.zapperPubkey,
+          });
+        }
+      },
+      5
+    );
+
+    setTimeout(() => this.#processZapperQueue(), 1_000);
+  }
+
+  async #processNip5Queue() {
+    await this.#batchQueue(
+      this.#nip5Queue,
+      async i => {
+        const [name, domain] = i.nip05.split("@");
+        const nip5pk = await fetchNip05Pubkey(name, domain);
+        const p = this.getFromCache(i.pubkey);
+        if (p) {
+          await this.set({
+            ...p,
+            isNostrAddressValid: i.pubkey === nip5pk,
+          });
+        }
+      },
+      5
+    );
+
+    setTimeout(() => this.#processNip5Queue(), 1_000);
+  }
+
+  async #batchQueue<T>(queue: Array<T>, proc: (v: T) => Promise<void>, batchSize = 3) {
+    const batch = [];
+    while (queue.length > 0) {
+      const i = queue.shift();
+      if (i) {
+        batch.push(
+          (async () => {
+            try {
+              await proc(i);
+            } catch {
+              console.warn("Failed to process item", i);
+            }
+            batch.pop(); // pop any
+          })()
+        );
+        if (batch.length === batchSize) {
+          await Promise.all(batch);
+        }
+      } else {
+        await Promise.all(batch);
+      }
+    }
+  }
+}

packages/system/src/cache/UserRelayCache.ts

@@ -0,0 +1,29 @@
+import { db, UsersRelays } from ".";
+import { FeedCache } from "@snort/shared";
+
+export class UserRelaysCache extends FeedCache<UsersRelays> {
+  constructor() {
+    super("UserRelays", db.userRelays);
+  }
+
+  key(of: UsersRelays): string {
+    return of.pubkey;
+  }
+
+  override async preload(follows?: Array<string>): Promise<void> {
+    await super.preload();
+    if (follows) {
+      await this.buffer(follows);
+    }
+  }
+
+  newest(): number {
+    let ret = 0;
+    this.cache.forEach(v => (ret = v.created_at > ret ? v.created_at : ret));
+    return ret;
+  }
+
+  takeSnapshot(): Array<UsersRelays> {
+    return [...this.cache.values()];
+  }
+}

packages/system/src/cache/db.ts

@@ -0,0 +1,42 @@
+import { MetadataCache, RelayMetrics, UsersRelays } from ".";
+import { NostrEvent } from "../Nostr";
+import Dexie, { Table } from "dexie";
+
+const NAME = "snort-system";
+const VERSION = 1;
+
+const STORES = {
+    users: "++pubkey, name, display_name, picture, nip05, npub",
+    relays: "++addr",
+    userRelays: "++pubkey",
+    events: "++id, pubkey, created_at"
+};
+
+export class SnortSystemDb extends Dexie {
+    ready = false;
+    users!: Table<MetadataCache>;
+    relayMetrics!: Table<RelayMetrics>;
+    userRelays!: Table<UsersRelays>;
+    events!: Table<NostrEvent>;
+    dms!: Table<NostrEvent>;
+
+    constructor() {
+        super(NAME);
+        this.version(VERSION).stores(STORES);
+    }
+
+    isAvailable() {
+        if ("indexedDB" in window) {
+            return new Promise<boolean>(resolve => {
+                const req = window.indexedDB.open("dummy", 1);
+                req.onsuccess = () => {
+                    resolve(true);
+                };
+                req.onerror = () => {
+                    resolve(false);
+                };
+            });
+        }
+        return Promise.resolve(false);
+    }
+}

packages/system/src/cache/index.ts

@@ -1,5 +1,8 @@
-import { HexKey, NostrEvent, UserMetadata } from "..";
-import { hexToBech32, unixNowMs } from "../Utils";
+import { FullRelaySettings, HexKey, NostrEvent, UserMetadata } from "..";
+import { hexToBech32, unixNowMs } from "@snort/shared";
+import { SnortSystemDb } from "./db";
+
+export const db = new SnortSystemDb();
 
 export interface MetadataCache extends UserMetadata {
   /**
@@ -33,6 +36,19 @@ export interface MetadataCache extends UserMetadata {
   isNostrAddressValid: boolean;
 }
 
+export interface RelayMetrics {
+  addr: string;
+  events: number;
+  disconnects: number;
+  latency: number[];
+}
+
+export interface UsersRelays {
+  pubkey: string;
+  created_at: number;
+  relays: FullRelaySettings[];
+}
+
 export function mapEventToProfile(ev: NostrEvent) {
   try {
     const data: UserMetadata = JSON.parse(ev.content);
@@ -54,23 +70,4 @@ export function mapEventToProfile(ev: NostrEvent) {
   } catch (e) {
     console.error("Failed to parse JSON", ev, e);
   }
-}
-
-export interface CacheStore<T> {
-  preload(): Promise<void>;
-  getFromCache(key?: string): T | undefined;
-  get(key?: string): Promise<T | undefined>;
-  bulkGet(keys: Array<string>): Promise<Array<T>>;
-  set(obj: T): Promise<void>;
-  bulkSet(obj: Array<T>): Promise<void>;
-  update<TCachedWithCreated extends T & { created: number; loaded: number }>(m: TCachedWithCreated): Promise<"new" | "updated" | "refresh" | "no_change">
-
-  /**
-   * Loads a list of rows from disk cache
-   * @param keys List of ids to load
-   * @returns Keys that do not exist on disk cache
-   */
-  buffer(keys: Array<string>): Promise<Array<string>>;
-
-  clear(): Promise<void>;
-}
+}

packages/system/src/index.ts

@@ -17,11 +17,15 @@ export * from "./RequestBuilder";
 export * from "./EventPublisher";
 export * from "./EventBuilder";
 export * from "./NostrLink";
-export * from "./cache";
 export * from "./ProfileCache";
+
 export * from "./impl/nip4";
 export * from "./impl/nip44";
 
+export * from "./cache";
+export * from "./cache/UserRelayCache";
+export * from "./cache/UserCache";
+
 export interface SystemInterface {
   /**
    * Handler function for NIP-42