mirror of
https://github.com/coracle-social/coracle.git
synced 2024-09-29 16:31:04 +00:00
Remove storage and pubkey loader
This commit is contained in:
parent
baf3108f3c
commit
c9183acae0
@ -11,8 +11,9 @@
|
||||
import {isNil, find, last} from "ramda"
|
||||
import {Storage, seconds, Fetch, shuffle} from "hurdak"
|
||||
import {tryFetch, hexToBech32, bech32ToHex, now} from "src/util/misc"
|
||||
import {storage} from "src/engine2"
|
||||
import {default as engine} from "src/app/engine"
|
||||
import {Keys, Nip65, user, Env, Network, Builder, Outbox, Settings, storage} from "src/app/engine"
|
||||
import {Keys, Nip65, user, Env, Network, Builder, Outbox, Settings} from "src/app/engine"
|
||||
import {loadAppData} from "src/app/state"
|
||||
import {theme, getThemeVariables, appName, modal} from "src/partials/state"
|
||||
import {logUsage} from "src/app/state"
|
||||
|
@ -22,7 +22,7 @@
|
||||
import RelayList from "src/app/views/RelayList.svelte"
|
||||
import UserProfile from "src/app/views/UserProfile.svelte"
|
||||
import UserSettings from "src/app/views/UserSettings.svelte"
|
||||
import {storage} from "src/app/engine"
|
||||
import {storage} from "src/engine2"
|
||||
|
||||
const TypedRoute = Route as ComponentType<SvelteComponentTyped>
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
import {identity} from "ramda"
|
||||
import {Engine, User, StorageAdapter, PubkeyLoader} from "src/engine"
|
||||
import {Engine, User} from "src/engine"
|
||||
|
||||
const IMGPROXY_URL = import.meta.env.VITE_IMGPROXY_URL
|
||||
|
||||
@ -40,8 +40,6 @@ const engine = new Engine({
|
||||
ENABLE_ZAPS,
|
||||
})
|
||||
|
||||
export const storage = new StorageAdapter(engine)
|
||||
export const pubkeyLoader = new PubkeyLoader(engine)
|
||||
export const user = new User(engine)
|
||||
|
||||
export default engine
|
||||
|
@ -3,7 +3,7 @@
|
||||
import Content from "src/partials/Content.svelte"
|
||||
import PersonBadgeSmall from "src/app/shared/PersonBadgeSmall.svelte"
|
||||
import NoteContentEllipsis from "src/app/shared/NoteContentEllipsis.svelte"
|
||||
import {pubkeyLoader} from "src/app/engine"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
|
||||
export let note
|
||||
export let showEntire
|
||||
@ -11,7 +11,7 @@
|
||||
const limit = showEntire ? Infinity : 5
|
||||
const pubkeys = Tags.from(note).type("p").values().all().slice(0, limit)
|
||||
|
||||
pubkeyLoader.load(pubkeys)
|
||||
loadPubkeys(pubkeys)
|
||||
</script>
|
||||
|
||||
<Content gap="gap-2" class="m-0">
|
||||
|
@ -5,7 +5,8 @@
|
||||
import Content from "src/partials/Content.svelte"
|
||||
import Spinner from "src/partials/Spinner.svelte"
|
||||
import PersonSummary from "src/app/shared/PersonSummary.svelte"
|
||||
import {Nip02, Nip65, Settings, Network, pubkeyLoader} from "src/app/engine"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
import {Nip02, Nip65, Settings, Network} from "src/app/engine"
|
||||
|
||||
export let type
|
||||
export let pubkey
|
||||
@ -22,7 +23,7 @@
|
||||
onEvent: batch(500, events => {
|
||||
const newPubkeys = pluck("pubkey", events)
|
||||
|
||||
pubkeyLoader.load(newPubkeys)
|
||||
loadPubkeys(newPubkeys)
|
||||
|
||||
pubkeys = uniq(pubkeys.concat(newPubkeys))
|
||||
}),
|
||||
|
@ -10,8 +10,9 @@ import {warn} from "src/util/logger"
|
||||
import {now} from "src/util/misc"
|
||||
import {userKinds, noteKinds} from "src/util/nostr"
|
||||
import {modal, toast} from "src/partials/state"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
import type {Event} from "src/engine/types"
|
||||
import {pubkeyLoader, Events, Nip28, Env, Network, user, Settings, Keys} from "src/app/engine"
|
||||
import {Events, Nip28, Env, Network, user, Settings, Keys} from "src/app/engine"
|
||||
|
||||
// Routing
|
||||
|
||||
@ -142,10 +143,10 @@ export const loadAppData = async () => {
|
||||
const pubkey = Keys.pubkey.get()
|
||||
|
||||
// Make sure the user and their follows are loaded
|
||||
await pubkeyLoader.load(pubkey, {force: true, kinds: userKinds})
|
||||
await loadPubkeys(pubkey, {force: true, kinds: userKinds})
|
||||
|
||||
// Load their network
|
||||
pubkeyLoader.load(user.getFollows())
|
||||
loadPubkeys(user.getFollows())
|
||||
|
||||
// Start our listener
|
||||
listenForNotifications()
|
||||
@ -164,7 +165,7 @@ export const login = async (method: string, key: string | {pubkey: string; token
|
||||
|
||||
await Promise.all([
|
||||
sleep(1500),
|
||||
pubkeyLoader.load(Keys.pubkey.get(), {force: true, kinds: userKinds}),
|
||||
loadPubkeys(Keys.pubkey.get(), {force: true, kinds: userKinds}),
|
||||
])
|
||||
|
||||
navigate("/notes")
|
||||
|
@ -12,7 +12,8 @@
|
||||
import Anchor from "src/partials/Anchor.svelte"
|
||||
import Modal from "src/partials/Modal.svelte"
|
||||
import RelayCard from "src/app/shared/RelayCard.svelte"
|
||||
import {Env, Nip65, user, Keys, pubkeyLoader, Network} from "src/app/engine"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
import {Env, Nip65, user, Keys, Network} from "src/app/engine"
|
||||
import {loadAppData} from "src/app/state"
|
||||
|
||||
const pubkey = Keys.pubkey.get()
|
||||
@ -56,7 +57,7 @@
|
||||
// Wait a bit before removing the relay to smooth out the ui
|
||||
Promise.all([
|
||||
sleep(1500),
|
||||
pubkeyLoader.load([pubkey], {
|
||||
loadPubkeys([pubkey], {
|
||||
force: true,
|
||||
relays: [relay.url],
|
||||
kinds: userKinds,
|
||||
|
@ -10,7 +10,8 @@
|
||||
import OnboardingRelays from "src/app/views/OnboardingRelays.svelte"
|
||||
import OnboardingFollows from "src/app/views/OnboardingFollows.svelte"
|
||||
import OnboardingNote from "src/app/views/OnboardingNote.svelte"
|
||||
import {Env, pubkeyLoader, Outbox, Builder, user, Keys} from "src/app/engine"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
import {Env, Outbox, Builder, user, Keys} from "src/app/engine"
|
||||
import {listenForNotifications} from "src/app/state"
|
||||
import {modal} from "src/partials/state"
|
||||
|
||||
@ -77,7 +78,7 @@
|
||||
|
||||
onMount(() => {
|
||||
// Prime our database with some defaults
|
||||
pubkeyLoader.load(Env.DEFAULT_FOLLOWS)
|
||||
loadPubkeys(Env.DEFAULT_FOLLOWS)
|
||||
})
|
||||
</script>
|
||||
|
||||
|
@ -13,7 +13,8 @@
|
||||
import PersonRelays from "src/app/shared/PersonRelays.svelte"
|
||||
import PersonHandle from "src/app/shared/PersonHandle.svelte"
|
||||
import PersonName from "src/app/shared/PersonName.svelte"
|
||||
import {Env, Settings, pubkeyLoader, Directory, Nip65} from "src/app/engine"
|
||||
import {loadPubkeys} from "src/engine2"
|
||||
import {Env, Settings, Directory, Nip65} from "src/app/engine"
|
||||
import PersonCircle from "src/app/shared/PersonCircle.svelte"
|
||||
import PersonAbout from "src/app/shared/PersonAbout.svelte"
|
||||
import PersonStats from "src/app/shared/PersonStats.svelte"
|
||||
@ -39,7 +40,7 @@
|
||||
|
||||
info("Person", npub, $profile)
|
||||
|
||||
pubkeyLoader.load([pubkey], {force: true})
|
||||
loadPubkeys([pubkey], {force: true})
|
||||
|
||||
document.title = Directory.displayProfile($profile)
|
||||
|
||||
|
@ -23,7 +23,6 @@ export {FeedLoader} from "./util/FeedLoader"
|
||||
export {ThreadLoader} from "./util/ThreadLoader"
|
||||
export {ContextLoader} from "./util/ContextLoader"
|
||||
export {Cursor, MultiCursor} from "./util/Cursor"
|
||||
export {StorageAdapter} from "./util/StorageAdapter"
|
||||
export {PubkeyLoader} from "./util/PubkeyLoader"
|
||||
export {Subscription} from "./util/Subscription"
|
||||
export {Worker} from "./util/Worker"
|
||||
|
@ -1,90 +0,0 @@
|
||||
import {without, pluck, uniq} from "ramda"
|
||||
import {chunk, seconds, ensurePlural} from "hurdak"
|
||||
import {personKinds, appDataKeys} from "src/util/nostr"
|
||||
import {now} from "src/util/misc"
|
||||
import type {Filter} from "src/engine/types"
|
||||
import type {Engine} from "src/engine/Engine"
|
||||
|
||||
export type LoadPeopleOpts = {
|
||||
relays?: string[]
|
||||
kinds?: number[]
|
||||
force?: boolean
|
||||
}
|
||||
|
||||
export class PubkeyLoader {
|
||||
engine: Engine
|
||||
attemptedPubkeys = new Set()
|
||||
|
||||
constructor(engine: Engine) {
|
||||
this.engine = engine
|
||||
}
|
||||
|
||||
getStalePubkeys = (pubkeys: string[]) => {
|
||||
const stale = new Set()
|
||||
const since = now() - seconds(3, "hour")
|
||||
|
||||
for (const pubkey of pubkeys) {
|
||||
if (stale.has(pubkey) || this.attemptedPubkeys.has(pubkey)) {
|
||||
continue
|
||||
}
|
||||
|
||||
this.attemptedPubkeys.add(pubkey)
|
||||
|
||||
if (this.engine.Directory.profiles.key(pubkey).get()?.updated_at || 0 > since) {
|
||||
continue
|
||||
}
|
||||
|
||||
stale.add(pubkey)
|
||||
}
|
||||
|
||||
return Array.from(stale)
|
||||
}
|
||||
|
||||
load = async (
|
||||
pubkeyGroups: string | string[],
|
||||
{relays, force, kinds = personKinds}: LoadPeopleOpts = {}
|
||||
) => {
|
||||
const rawPubkeys = ensurePlural(pubkeyGroups).reduce((a, b) => a.concat(b), [])
|
||||
const pubkeys = force ? uniq(rawPubkeys) : this.getStalePubkeys(rawPubkeys)
|
||||
|
||||
const getChunkRelays = (chunk: string[]) => {
|
||||
if (relays?.length > 0) {
|
||||
return relays
|
||||
}
|
||||
|
||||
const limit = this.engine.Settings.getSetting("relay_limit")
|
||||
|
||||
return this.engine.Nip65.mergeHints(
|
||||
limit,
|
||||
chunk.map(pubkey => this.engine.Nip65.getPubkeyHints(limit, pubkey, "write"))
|
||||
)
|
||||
}
|
||||
|
||||
const getChunkFilter = (chunk: string[]) => {
|
||||
const filter = [] as Filter[]
|
||||
|
||||
filter.push({kinds: without([30078], kinds), authors: chunk})
|
||||
|
||||
// Add a separate filter for app data so we're not pulling down other people's stuff,
|
||||
// or obsolete events of our own.
|
||||
if (kinds.includes(30078)) {
|
||||
filter.push({kinds: [30078], authors: chunk, "#d": Object.values(appDataKeys)})
|
||||
}
|
||||
|
||||
return filter
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
pluck(
|
||||
"result",
|
||||
chunk(256, pubkeys).map((chunk: string[]) =>
|
||||
this.engine.Network.subscribe({
|
||||
relays: getChunkRelays(chunk),
|
||||
filter: getChunkFilter(chunk),
|
||||
timeout: 10_000,
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
@ -1,165 +0,0 @@
|
||||
import {prop, pluck, splitAt, path as getPath, sortBy} from "ramda"
|
||||
import {sleep, defer, chunk, randomInt, throttle} from "hurdak"
|
||||
import {Storage as LocalStorage} from "hurdak"
|
||||
import type {Channel, Contact} from "src/engine/types"
|
||||
import type {Engine} from "src/engine/Engine"
|
||||
import {writable} from "src/engine/util/store"
|
||||
import type {Collection} from "src/engine/util/store"
|
||||
import {IndexedDB} from "src/engine/util/indexeddb"
|
||||
|
||||
const localStorageKeys = ["Keys.pubkey", "Keys.keyState", "Settings.settings"]
|
||||
|
||||
const sortChannels = sortBy((e: Channel) =>
|
||||
e.joined ? 0 : -Math.max(e.last_checked || 0, e.last_sent || 0)
|
||||
)
|
||||
|
||||
const sortContacts = sortBy((e: Contact) => -Math.max(e.last_checked || 0, e.last_sent || 0))
|
||||
|
||||
const policy = (key: string, max: number, sort: (xs: any[]) => any[]) => ({key, max, sort})
|
||||
|
||||
const getStore = (key: string, engine: Engine) => getPath(key.split("."), engine) as Collection<any>
|
||||
|
||||
export class StorageAdapter {
|
||||
engine: Engine
|
||||
db: IndexedDB
|
||||
ready = defer()
|
||||
dead = writable(false)
|
||||
|
||||
constructor(engine: Engine) {
|
||||
this.engine = engine
|
||||
|
||||
Promise.all([this.syncToLocalStorage(), this.syncToIndexedDb()]).then(() => {
|
||||
this.ready.resolve()
|
||||
})
|
||||
}
|
||||
|
||||
close = () => {
|
||||
this.dead.set(true)
|
||||
|
||||
return this.db?.close()
|
||||
}
|
||||
|
||||
clear = () => {
|
||||
this.dead.set(true)
|
||||
|
||||
localStorage.clear()
|
||||
|
||||
return this.db?.delete()
|
||||
}
|
||||
|
||||
getPubkeyWhitelist = () => {
|
||||
const pubkeys = this.engine.Keys.keyState.get().map(prop("pubkey"))
|
||||
|
||||
return [new Set(pubkeys), this.engine.Nip02.getFollowsSet(pubkeys)]
|
||||
}
|
||||
|
||||
sortByPubkeyWhitelist = (fallback: (x: any) => number) => (rows: Record<string, any>[]) => {
|
||||
const [pubkeys, follows] = this.getPubkeyWhitelist()
|
||||
|
||||
return sortBy(x => {
|
||||
if (pubkeys.has(x.pubkey)) {
|
||||
return Number.MAX_SAFE_INTEGER
|
||||
}
|
||||
|
||||
if (follows.has(x.pubkey)) {
|
||||
return Number.MAX_SAFE_INTEGER - 1
|
||||
}
|
||||
|
||||
return fallback(x)
|
||||
}, rows)
|
||||
}
|
||||
|
||||
syncToLocalStorage() {
|
||||
for (const key of localStorageKeys) {
|
||||
const store = getStore(key, this.engine)
|
||||
|
||||
if (Object.hasOwn(localStorage, key)) {
|
||||
store.set(LocalStorage.getJson(key))
|
||||
}
|
||||
|
||||
store.subscribe(throttle(300, $value => LocalStorage.setJson(key, $value)))
|
||||
}
|
||||
}
|
||||
|
||||
async syncToIndexedDb() {
|
||||
if (window.indexedDB) {
|
||||
const policies = [
|
||||
policy("Nip28.channels", 2000, sortChannels),
|
||||
policy("Nip28.messages", 10000, sortBy(prop("created_at"))),
|
||||
policy("Nip04.contacts", 1000, sortContacts),
|
||||
policy("Nip04.messages", 10000, sortBy(prop("created_at"))),
|
||||
policy("Nip24.channels", 1000, sortChannels),
|
||||
policy("Nip24.messages", 10000, sortBy(prop("created_at"))),
|
||||
policy("Content.topics", 1000, sortBy(prop("count"))),
|
||||
policy("Content.lists", 500, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Content.labels", 10000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Directory.profiles", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Events.cache", 5000, this.sortByPubkeyWhitelist(prop("created_at"))),
|
||||
policy("Nip02.graph", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Nip05.handles", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Nip57.zappers", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
policy("Nip65.relays", 2000, prop("count")),
|
||||
policy("Nip65.policies", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
|
||||
]
|
||||
|
||||
this.db = new IndexedDB(
|
||||
"nostr-engine/Storage",
|
||||
3,
|
||||
policies.map(({key}) => {
|
||||
const store = getStore(key, this.engine)
|
||||
|
||||
return {
|
||||
name: key,
|
||||
opts: {
|
||||
keyPath: store.pk,
|
||||
},
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
window.addEventListener("beforeunload", () => this.close())
|
||||
|
||||
await this.db.open()
|
||||
|
||||
for (const {key} of policies) {
|
||||
const store = getStore(key, this.engine)
|
||||
|
||||
store.set(await this.db.getAll(key))
|
||||
|
||||
store.subscribe(
|
||||
throttle(randomInt(3000, 5000), async <T>(rows: T) => {
|
||||
if (this.dead.get()) {
|
||||
return
|
||||
}
|
||||
|
||||
// Do it in small steps to avoid clogging stuff up
|
||||
for (const records of chunk(100, rows as any[])) {
|
||||
await this.db.bulkPut(key, records)
|
||||
await sleep(50)
|
||||
|
||||
if (this.dead.get()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Every so often randomly prune a store
|
||||
setInterval(() => {
|
||||
const {key, max, sort} = policies[Math.floor(policies.length * Math.random())]
|
||||
const store = getStore(key, this.engine)
|
||||
const data = store.get()
|
||||
|
||||
if (data.length < max * 1.1 || this.dead.get()) {
|
||||
return
|
||||
}
|
||||
|
||||
const [discard, keep] = splitAt(max, sort(data))
|
||||
|
||||
store.set(keep)
|
||||
this.db.bulkDelete(key, pluck(store.pk, discard))
|
||||
}, 30_000)
|
||||
}
|
||||
}
|
||||
}
|
@ -1,107 +0,0 @@
|
||||
// From https://gist.github.com/underground/d50e40170d54b8a0f8a3f4fdd466eee4
|
||||
export class IndexedDB {
|
||||
constructor(dbName, dbVersion, stores) {
|
||||
this.db
|
||||
this.dbName = dbName
|
||||
this.dbVersion = dbVersion
|
||||
this.stores = stores
|
||||
}
|
||||
|
||||
open() {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!window.indexedDB) {
|
||||
reject("Unsupported indexedDB")
|
||||
}
|
||||
|
||||
const request = window.indexedDB.open(this.dbName, this.dbVersion)
|
||||
|
||||
request.onsuccess = e => {
|
||||
this.db = request.result
|
||||
|
||||
resolve()
|
||||
}
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
|
||||
request.onupgradeneeded = e => {
|
||||
this.db = e.target.result
|
||||
|
||||
this.stores.forEach(o => {
|
||||
try {
|
||||
this.db.createObjectStore(o.name, o.opts)
|
||||
} catch (e) {
|
||||
console.warn(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
close() {
|
||||
return this.db.close()
|
||||
}
|
||||
|
||||
delete() {
|
||||
window.indexedDB.deleteDatabase(this.dbName)
|
||||
}
|
||||
|
||||
getAll(storeName) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const store = this.db.transaction(storeName).objectStore(storeName)
|
||||
const request = store.getAll()
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
request.onsuccess = e => resolve(e.target.result)
|
||||
})
|
||||
}
|
||||
|
||||
async bulkPut(storeName, data) {
|
||||
const transaction = this.db.transaction(storeName, "readwrite")
|
||||
const store = transaction.objectStore(storeName)
|
||||
|
||||
return Promise.all(
|
||||
data.map(row => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = store.put(row)
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
request.onsuccess = e => resolve(e.target.result)
|
||||
})
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
async bulkDelete(storeName, ids) {
|
||||
const transaction = this.db.transaction(storeName, "readwrite")
|
||||
const store = transaction.objectStore(storeName)
|
||||
|
||||
return Promise.all(
|
||||
ids.map(id => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = store.delete(id)
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
request.onsuccess = e => resolve(e.target.result)
|
||||
})
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
clear(storeName) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = this.db.transaction(storeName, "readwrite").objectStore(storeName).clear()
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
request.onsuccess = e => resolve(e.target.result)
|
||||
})
|
||||
}
|
||||
|
||||
count(storeName) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = this.db.transaction(storeName).objectStore(storeName).count()
|
||||
|
||||
request.onerror = e => reject(e.target.error)
|
||||
request.onsuccess = e => resolve(e.target.result)
|
||||
})
|
||||
}
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
export * from "./model"
|
||||
export * from "./state"
|
||||
export * from "./queries"
|
||||
export * from "./requests"
|
||||
export * from "./storage"
|
||||
export * from "./projections"
|
||||
export * from "./commands"
|
||||
|
@ -1,5 +1,8 @@
|
||||
export * from "./context"
|
||||
export * from "./cursor"
|
||||
export * from "./executor"
|
||||
export * from "./feed"
|
||||
export * from "./pubkeys"
|
||||
export * from "./publisher"
|
||||
export * from "./subscription"
|
||||
export * from "./cursor"
|
||||
export * from "./context"
|
||||
export * from "./thread"
|
||||
|
Loading…
Reference in New Issue
Block a user