Add storage to engine2

src/engine2/state/index.ts

@@ -1,5 +1,7 @@
-import {whereEq, find} from "ramda"
+import {sortBy, nth, map, whereEq, prop, find} from "ramda"
+import {ensurePlural} from "hurdak"
 import {collection, derived, writable} from "src/engine2/util/store"
+import {LocalStorageAdapter, IndexedDBAdapter, Storage} from "src/engine2/util/storage"
 import type {
   Event,
   KeyState,
@@ -17,11 +19,12 @@ import {deriveSigner} from "src/engine2/state/signer"
 import {deriveCrypto} from "src/engine2/state/crypto"
 import {deriveWrapper} from "src/engine2/state/wrapper"
 
-// Sync stores
+// Synchronous stores
 
 export const keys = writable<KeyState[]>([])
 export const pubkey = writable<string | null>(null)
 export const settings = writable<Record<string, any>>({})
+export const env = writable<Record<string, any>>({})
 
 // Async stores
 
@@ -47,3 +50,81 @@ export const signer = deriveSigner({user, ndk})
 export const wrapper = deriveWrapper({user, signer, crypto})
 
 // Parameterizable derivations
+
+export const derivePetnames = (pubkey: string) =>
+  socialGraph.key(pubkey).derived(g => g?.petnames || [])
+
+export const deriveMutes = (pubkey: string) => socialGraph.key(pubkey).derived(g => g?.mutes || [])
+
+export const deriveFollowsSet = (pubkeys: string | string[]) =>
+  derived(
+    ensurePlural(pubkeys).map(derivePetnames),
+    petnameGroups => new Set(petnameGroups.flatMap(map(nth(1))))
+  )
+
+export const deriveMutesSet = (pubkeys: string | string[]) =>
+  derived(
+    ensurePlural(pubkeys).map(deriveMutes),
+    petnameGroups => new Set(petnameGroups.flatMap(map(nth(1))))
+  )
+
+// Synchronization to local storage and indexeddb
+
+const sortByPubkeyWhitelist = (fallback: (x: any) => number) => (rows: Record<string, any>[]) => {
+  const pubkeys = new Set(keys.get().map(prop("pubkey")))
+  const follows = deriveFollowsSet(Array.from(pubkeys)).get()
+
+  return sortBy(x => {
+    if (pubkeys.has(x.pubkey)) {
+      return Number.MAX_SAFE_INTEGER
+    }
+
+    if (follows.has(x.pubkey)) {
+      return Number.MAX_SAFE_INTEGER - 1
+    }
+
+    return fallback(x)
+  }, rows)
+}
+
+export const storage = new Storage([
+  new LocalStorageAdapter("Keys.keyState", keys),
+  new LocalStorageAdapter("Keys.pubkey", pubkey),
+  new LocalStorageAdapter("settings", settings),
+  new IndexedDBAdapter("events", events, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("topics", topics, {
+    max: 1000,
+    sort: sortBy(prop("created_at")),
+  }),
+  new IndexedDBAdapter("lists", lists, {
+    max: 1000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("profiles", profiles, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("socialGraph", socialGraph, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("handles", handles, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("zappers", zappers, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+  new IndexedDBAdapter("relays", relays, {
+    max: 1000,
+    sort: sortBy(prop("created_at")),
+  }),
+  new IndexedDBAdapter("relayPolicies", relayPolicies, {
+    max: 10000,
+    sort: sortByPubkeyWhitelist(prop("created_at")),
+  }),
+])

src/engine2/util/indexeddb.js

@@ -0,0 +1,107 @@
+// From https://gist.github.com/underground/d50e40170d54b8a0f8a3f4fdd466eee4
+export class IndexedDB {
+  constructor(dbName, dbVersion, stores) {
+    this.db
+    this.dbName = dbName
+    this.dbVersion = dbVersion
+    this.stores = stores
+  }
+
+  open() {
+    return new Promise((resolve, reject) => {
+      if (!window.indexedDB) {
+        reject("Unsupported indexedDB")
+      }
+
+      const request = window.indexedDB.open(this.dbName, this.dbVersion)
+
+      request.onsuccess = e => {
+        this.db = request.result
+
+        resolve()
+      }
+
+      request.onerror = e => reject(e.target.error)
+
+      request.onupgradeneeded = e => {
+        this.db = e.target.result
+
+        this.stores.forEach(o => {
+          try {
+            this.db.createObjectStore(o.name, o.opts)
+          } catch (e) {
+            console.warn(e)
+          }
+        })
+      }
+    })
+  }
+
+  close() {
+    return this.db.close()
+  }
+
+  delete() {
+    window.indexedDB.deleteDatabase(this.dbName)
+  }
+
+  getAll(storeName) {
+    return new Promise((resolve, reject) => {
+      const store = this.db.transaction(storeName).objectStore(storeName)
+      const request = store.getAll()
+
+      request.onerror = e => reject(e.target.error)
+      request.onsuccess = e => resolve(e.target.result)
+    })
+  }
+
+  async bulkPut(storeName, data) {
+    const transaction = this.db.transaction(storeName, "readwrite")
+    const store = transaction.objectStore(storeName)
+
+    return Promise.all(
+      data.map(row => {
+        return new Promise((resolve, reject) => {
+          const request = store.put(row)
+
+          request.onerror = e => reject(e.target.error)
+          request.onsuccess = e => resolve(e.target.result)
+        })
+      })
+    )
+  }
+
+  async bulkDelete(storeName, ids) {
+    const transaction = this.db.transaction(storeName, "readwrite")
+    const store = transaction.objectStore(storeName)
+
+    return Promise.all(
+      ids.map(id => {
+        return new Promise((resolve, reject) => {
+          const request = store.delete(id)
+
+          request.onerror = e => reject(e.target.error)
+          request.onsuccess = e => resolve(e.target.result)
+        })
+      })
+    )
+  }
+
+  clear(storeName) {
+    return new Promise((resolve, reject) => {
+      const request = this.db.transaction(storeName, "readwrite").objectStore(storeName).clear()
+
+      request.onerror = e => reject(e.target.error)
+      request.onsuccess = e => resolve(e.target.result)
+    })
+  }
+
+  count(storeName) {
+    return new Promise((resolve, reject) => {
+      const request = this.db.transaction(storeName).objectStore(storeName).count()
+
+      request.onerror = e => reject(e.target.error)
+      request.onsuccess = e => resolve(e.target.result)
+    })
+  }
+}

src/engine2/util/storage.ts

@@ -0,0 +1,136 @@
+import {pluck, splitAt} from "ramda"
+import {sleep, defer, chunk, randomInt, throttle} from "hurdak"
+import {Storage as LocalStorage} from "hurdak"
+import type {Writable, Collection} from "src/engine2/util/store"
+import {IndexedDB} from "src/engine2/util/indexeddb"
+import {writable} from "src/engine2/util/store"
+
+export class LocalStorageAdapter {
+  constructor(readonly key: string, readonly store: Writable<any>) {}
+
+  initialize(storage: Storage) {
+    const {key, store} = this
+
+    if (Object.hasOwn(localStorage, key)) {
+      store.set(LocalStorage.getJson(key))
+    }
+
+    store.subscribe(throttle(300, $value => LocalStorage.setJson(key, $value)))
+  }
+}
+
+type IndexedDBAdapterOpts = {
+  max: number
+  sort: (xs: any[]) => any[]
+}
+
+export class IndexedDBAdapter {
+  constructor(
+    readonly key: string,
+    readonly store: Collection<any>,
+    readonly opts: IndexedDBAdapterOpts
+  ) {}
+
+  getIndexedDBConfig() {
+    return {
+      name: this.key,
+      opts: {
+        keyPath: this.store.pk,
+      },
+    }
+  }
+
+  async initialize(storage: Storage) {
+    const {key, store} = this
+
+    store.set(await storage.db.getAll(key))
+
+    store.subscribe(
+      throttle(randomInt(3000, 5000), async <T>(rows: T) => {
+        if (storage.dead.get()) {
+          return
+        }
+
+        // Do it in small steps to avoid clogging stuff up
+        for (const records of chunk(100, rows as any[])) {
+          await storage.db.bulkPut(key, records)
+          await sleep(50)
+
+          if (storage.dead.get()) {
+            return
+          }
+        }
+      })
+    )
+  }
+
+  prune(storage) {
+    const {
+      store,
+      key,
+      opts: {max, sort},
+    } = this
+    const data = store.get()
+
+    if (data.length < max * 1.1 || storage.dead.get()) {
+      return
+    }
+
+    const [discard, keep] = splitAt(max, sort(data))
+
+    store.set(keep)
+
+    storage.db.bulkDelete(key, pluck(store.pk, discard))
+  }
+}
+
+export class Storage {
+  db: IndexedDB
+  ready = defer()
+  dead = writable(false)
+
+  constructor(readonly adapters: (LocalStorageAdapter | IndexedDBAdapter)[]) {
+    this.initialize()
+  }
+
+  close = () => {
+    this.dead.set(true)
+
+    return this.db?.close()
+  }
+
+  clear = () => {
+    this.dead.set(true)
+
+    localStorage.clear()
+
+    return this.db?.delete()
+  }
+
+  async initialize() {
+    const indexedDBAdapters = this.adapters.filter(
+      a => a instanceof IndexedDBAdapter
+    ) as IndexedDBAdapter[]
+
+    if (window.indexedDB) {
+      const dbConfig = indexedDBAdapters.map(adapter => adapter.getIndexedDBConfig())
+
+      this.db = new IndexedDB("nostr-engine/Storage", 2, dbConfig)
+
+      window.addEventListener("beforeunload", () => this.close())
+
+      await this.db.open()
+    }
+
+    await Promise.all(this.adapters.map(adapter => adapter.initialize(this)))
+
+    // Every so often randomly prune a store
+    setInterval(() => {
+      const adapter = indexedDBAdapters[Math.floor(indexedDBAdapters.length * Math.random())]
+
+      adapter.prune(this)
+    }, 30_000)
+
+    this.ready.resolve()
+  }
+}