diff --git a/src/engine2/state/index.ts b/src/engine2/state/index.ts index 1e52ce73..d50d94b2 100644 --- a/src/engine2/state/index.ts +++ b/src/engine2/state/index.ts @@ -1,5 +1,7 @@ -import {whereEq, find} from "ramda" +import {sortBy, nth, map, whereEq, prop, find} from "ramda" +import {ensurePlural} from "hurdak" import {collection, derived, writable} from "src/engine2/util/store" +import {LocalStorageAdapter, IndexedDBAdapter, Storage} from "src/engine2/util/storage" import type { Event, KeyState, @@ -17,11 +19,12 @@ import {deriveSigner} from "src/engine2/state/signer" import {deriveCrypto} from "src/engine2/state/crypto" import {deriveWrapper} from "src/engine2/state/wrapper" -// Sync stores +// Synchronous stores export const keys = writable([]) export const pubkey = writable(null) export const settings = writable>({}) +export const env = writable>({}) // Async stores @@ -47,3 +50,81 @@ export const signer = deriveSigner({user, ndk}) export const wrapper = deriveWrapper({user, signer, crypto}) // Parameterizable derivations + +export const derivePetnames = (pubkey: string) => + socialGraph.key(pubkey).derived(g => g?.petnames || []) + +export const deriveMutes = (pubkey: string) => socialGraph.key(pubkey).derived(g => g?.mutes || []) + +export const deriveFollowsSet = (pubkeys: string | string[]) => + derived( + ensurePlural(pubkeys).map(derivePetnames), + petnameGroups => new Set(petnameGroups.flatMap(map(nth(1)))) + ) + +export const deriveMutesSet = (pubkeys: string | string[]) => + derived( + ensurePlural(pubkeys).map(deriveMutes), + petnameGroups => new Set(petnameGroups.flatMap(map(nth(1)))) + ) + +// Synchronization to local storage and indexeddb + +const sortByPubkeyWhitelist = (fallback: (x: any) => number) => (rows: Record[]) => { + const pubkeys = new Set(keys.get().map(prop("pubkey"))) + const follows = deriveFollowsSet(Array.from(pubkeys)).get() + + return sortBy(x => { + if (pubkeys.has(x.pubkey)) { + return Number.MAX_SAFE_INTEGER + } + + if (follows.has(x.pubkey)) { + return Number.MAX_SAFE_INTEGER - 1 + } + + return fallback(x) + }, rows) +} + +export const storage = new Storage([ + new LocalStorageAdapter("Keys.keyState", keys), + new LocalStorageAdapter("Keys.pubkey", pubkey), + new LocalStorageAdapter("settings", settings), + new IndexedDBAdapter("events", events, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("topics", topics, { + max: 1000, + sort: sortBy(prop("created_at")), + }), + new IndexedDBAdapter("lists", lists, { + max: 1000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("profiles", profiles, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("socialGraph", socialGraph, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("handles", handles, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("zappers", zappers, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), + new IndexedDBAdapter("relays", relays, { + max: 1000, + sort: sortBy(prop("created_at")), + }), + new IndexedDBAdapter("relayPolicies", relayPolicies, { + max: 10000, + sort: sortByPubkeyWhitelist(prop("created_at")), + }), +]) diff --git a/src/engine2/util/indexeddb.js b/src/engine2/util/indexeddb.js new file mode 100644 index 00000000..df7a0c24 --- /dev/null +++ b/src/engine2/util/indexeddb.js @@ -0,0 +1,107 @@ +// From https://gist.github.com/underground/d50e40170d54b8a0f8a3f4fdd466eee4 +export class IndexedDB { + constructor(dbName, dbVersion, stores) { + this.db + this.dbName = dbName + this.dbVersion = dbVersion + this.stores = stores + } + + open() { + return new Promise((resolve, reject) => { + if (!window.indexedDB) { + reject("Unsupported indexedDB") + } + + const request = window.indexedDB.open(this.dbName, this.dbVersion) + + request.onsuccess = e => { + this.db = request.result + + resolve() + } + + request.onerror = e => reject(e.target.error) + + request.onupgradeneeded = e => { + this.db = e.target.result + + this.stores.forEach(o => { + try { + this.db.createObjectStore(o.name, o.opts) + } catch (e) { + console.warn(e) + } + }) + } + }) + } + + close() { + return this.db.close() + } + + delete() { + window.indexedDB.deleteDatabase(this.dbName) + } + + getAll(storeName) { + return new Promise((resolve, reject) => { + const store = this.db.transaction(storeName).objectStore(storeName) + const request = store.getAll() + + request.onerror = e => reject(e.target.error) + request.onsuccess = e => resolve(e.target.result) + }) + } + + async bulkPut(storeName, data) { + const transaction = this.db.transaction(storeName, "readwrite") + const store = transaction.objectStore(storeName) + + return Promise.all( + data.map(row => { + return new Promise((resolve, reject) => { + const request = store.put(row) + + request.onerror = e => reject(e.target.error) + request.onsuccess = e => resolve(e.target.result) + }) + }) + ) + } + + async bulkDelete(storeName, ids) { + const transaction = this.db.transaction(storeName, "readwrite") + const store = transaction.objectStore(storeName) + + return Promise.all( + ids.map(id => { + return new Promise((resolve, reject) => { + const request = store.delete(id) + + request.onerror = e => reject(e.target.error) + request.onsuccess = e => resolve(e.target.result) + }) + }) + ) + } + + clear(storeName) { + return new Promise((resolve, reject) => { + const request = this.db.transaction(storeName, "readwrite").objectStore(storeName).clear() + + request.onerror = e => reject(e.target.error) + request.onsuccess = e => resolve(e.target.result) + }) + } + + count(storeName) { + return new Promise((resolve, reject) => { + const request = this.db.transaction(storeName).objectStore(storeName).count() + + request.onerror = e => reject(e.target.error) + request.onsuccess = e => resolve(e.target.result) + }) + } +} diff --git a/src/engine2/util/storage.ts b/src/engine2/util/storage.ts new file mode 100644 index 00000000..b6537a8e --- /dev/null +++ b/src/engine2/util/storage.ts @@ -0,0 +1,136 @@ +import {pluck, splitAt} from "ramda" +import {sleep, defer, chunk, randomInt, throttle} from "hurdak" +import {Storage as LocalStorage} from "hurdak" +import type {Writable, Collection} from "src/engine2/util/store" +import {IndexedDB} from "src/engine2/util/indexeddb" +import {writable} from "src/engine2/util/store" + +export class LocalStorageAdapter { + constructor(readonly key: string, readonly store: Writable) {} + + initialize(storage: Storage) { + const {key, store} = this + + if (Object.hasOwn(localStorage, key)) { + store.set(LocalStorage.getJson(key)) + } + + store.subscribe(throttle(300, $value => LocalStorage.setJson(key, $value))) + } +} + +type IndexedDBAdapterOpts = { + max: number + sort: (xs: any[]) => any[] +} + +export class IndexedDBAdapter { + constructor( + readonly key: string, + readonly store: Collection, + readonly opts: IndexedDBAdapterOpts + ) {} + + getIndexedDBConfig() { + return { + name: this.key, + opts: { + keyPath: this.store.pk, + }, + } + } + + async initialize(storage: Storage) { + const {key, store} = this + + store.set(await storage.db.getAll(key)) + + store.subscribe( + throttle(randomInt(3000, 5000), async (rows: T) => { + if (storage.dead.get()) { + return + } + + // Do it in small steps to avoid clogging stuff up + for (const records of chunk(100, rows as any[])) { + await storage.db.bulkPut(key, records) + await sleep(50) + + if (storage.dead.get()) { + return + } + } + }) + ) + } + + prune(storage) { + const { + store, + key, + opts: {max, sort}, + } = this + const data = store.get() + + if (data.length < max * 1.1 || storage.dead.get()) { + return + } + + const [discard, keep] = splitAt(max, sort(data)) + + store.set(keep) + + storage.db.bulkDelete(key, pluck(store.pk, discard)) + } +} + +export class Storage { + db: IndexedDB + ready = defer() + dead = writable(false) + + constructor(readonly adapters: (LocalStorageAdapter | IndexedDBAdapter)[]) { + this.initialize() + } + + close = () => { + this.dead.set(true) + + return this.db?.close() + } + + clear = () => { + this.dead.set(true) + + localStorage.clear() + + return this.db?.delete() + } + + async initialize() { + const indexedDBAdapters = this.adapters.filter( + a => a instanceof IndexedDBAdapter + ) as IndexedDBAdapter[] + + if (window.indexedDB) { + const dbConfig = indexedDBAdapters.map(adapter => adapter.getIndexedDBConfig()) + + this.db = new IndexedDB("nostr-engine/Storage", 2, dbConfig) + + window.addEventListener("beforeunload", () => this.close()) + + await this.db.open() + } + + await Promise.all(this.adapters.map(adapter => adapter.initialize(this))) + + // Every so often randomly prune a store + setInterval(() => { + const adapter = indexedDBAdapters[Math.floor(indexedDBAdapters.length * Math.random())] + + adapter.prune(this) + }, 30_000) + + this.ready.resolve() + } +}