nostr/coracle
1
0
Fork 0
mirror of https://github.com/coracle-social/coracle.git synced 2024-09-29 16:31:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove storage and pubkey loader

Browse Source
This commit is contained in:
Jonathan Staab 2023-09-09 12:56:38 -07:00
parent baf3108f3c
commit c9183acae0
15 changed files with 29 additions and 384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/app/App.svelte
View File

@ -11,8 +11,9 @@
import {isNil, find, last} from "ramda"
import {Storage, seconds, Fetch, shuffle} from "hurdak"
import {tryFetch, hexToBech32, bech32ToHex, now} from "src/util/misc"
import {storage} from "src/engine2"
import {default as engine} from "src/app/engine"
import {Keys, Nip65, user, Env, Network, Builder, Outbox, Settings, storage} from "src/app/engine"
import {Keys, Nip65, user, Env, Network, Builder, Outbox, Settings} from "src/app/engine"
import {loadAppData} from "src/app/state"
import {theme, getThemeVariables, appName, modal} from "src/partials/state"
import {logUsage} from "src/app/state"

2
src/app/Routes.svelte
View File

@ -22,7 +22,7 @@
import RelayList from "src/app/views/RelayList.svelte"
import UserProfile from "src/app/views/UserProfile.svelte"
import UserSettings from "src/app/views/UserSettings.svelte"
import {storage} from "src/app/engine"
import {storage} from "src/engine2"
const TypedRoute = Route as ComponentType<SvelteComponentTyped>

4
src/app/engine.ts
View File

@ -1,5 +1,5 @@
import {identity} from "ramda"
import {Engine, User, StorageAdapter, PubkeyLoader} from "src/engine"
import {Engine, User} from "src/engine"
const IMGPROXY_URL = import.meta.env.VITE_IMGPROXY_URL
@ -40,8 +40,6 @@ const engine = new Engine({
ENABLE_ZAPS,
})
export const storage = new StorageAdapter(engine)
export const pubkeyLoader = new PubkeyLoader(engine)
export const user = new User(engine)
export default engine

4
src/app/shared/NoteContentKind3.svelte
View File

@ -3,7 +3,7 @@
import Content from "src/partials/Content.svelte"
import PersonBadgeSmall from "src/app/shared/PersonBadgeSmall.svelte"
import NoteContentEllipsis from "src/app/shared/NoteContentEllipsis.svelte"
import {pubkeyLoader} from "src/app/engine"
import {loadPubkeys} from "src/engine2"
export let note
export let showEntire
@ -11,7 +11,7 @@
const limit = showEntire ? Infinity : 5
const pubkeys = Tags.from(note).type("p").values().all().slice(0, limit)
pubkeyLoader.load(pubkeys)
loadPubkeys(pubkeys)
</script>
<Content gap="gap-2" class="m-0">

5
src/app/shared/PersonList.svelte
View File

@ -5,7 +5,8 @@
import Content from "src/partials/Content.svelte"
import Spinner from "src/partials/Spinner.svelte"
import PersonSummary from "src/app/shared/PersonSummary.svelte"
import {Nip02, Nip65, Settings, Network, pubkeyLoader} from "src/app/engine"
import {loadPubkeys} from "src/engine2"
import {Nip02, Nip65, Settings, Network} from "src/app/engine"
export let type
export let pubkey
@ -22,7 +23,7 @@
onEvent: batch(500, events => {
const newPubkeys = pluck("pubkey", events)
pubkeyLoader.load(newPubkeys)
loadPubkeys(newPubkeys)
pubkeys = uniq(pubkeys.concat(newPubkeys))
}),

9
src/app/state.ts
View File

@ -10,8 +10,9 @@ import {warn} from "src/util/logger"
import {now} from "src/util/misc"
import {userKinds, noteKinds} from "src/util/nostr"
import {modal, toast} from "src/partials/state"
import {loadPubkeys} from "src/engine2"
import type {Event} from "src/engine/types"
import {pubkeyLoader, Events, Nip28, Env, Network, user, Settings, Keys} from "src/app/engine"
import {Events, Nip28, Env, Network, user, Settings, Keys} from "src/app/engine"
// Routing
@ -142,10 +143,10 @@ export const loadAppData = async () => {
const pubkey = Keys.pubkey.get()
// Make sure the user and their follows are loaded
await pubkeyLoader.load(pubkey, {force: true, kinds: userKinds})
await loadPubkeys(pubkey, {force: true, kinds: userKinds})
// Load their network
pubkeyLoader.load(user.getFollows())
loadPubkeys(user.getFollows())
// Start our listener
listenForNotifications()
@ -164,7 +165,7 @@ export const login = async (method: string, key: string | {pubkey: string; token
await Promise.all([
sleep(1500),
pubkeyLoader.load(Keys.pubkey.get(), {force: true, kinds: userKinds}),
loadPubkeys(Keys.pubkey.get(), {force: true, kinds: userKinds}),
])
navigate("/notes")

5
src/app/views/LoginConnect.svelte
View File

@ -12,7 +12,8 @@
import Anchor from "src/partials/Anchor.svelte"
import Modal from "src/partials/Modal.svelte"
import RelayCard from "src/app/shared/RelayCard.svelte"
import {Env, Nip65, user, Keys, pubkeyLoader, Network} from "src/app/engine"
import {loadPubkeys} from "src/engine2"
import {Env, Nip65, user, Keys, Network} from "src/app/engine"
import {loadAppData} from "src/app/state"
const pubkey = Keys.pubkey.get()
@ -56,7 +57,7 @@
// Wait a bit before removing the relay to smooth out the ui
Promise.all([
sleep(1500),
pubkeyLoader.load([pubkey], {
loadPubkeys([pubkey], {
force: true,
relays: [relay.url],
kinds: userKinds,

5
src/app/views/Onboarding.svelte
View File

@ -10,7 +10,8 @@
import OnboardingRelays from "src/app/views/OnboardingRelays.svelte"
import OnboardingFollows from "src/app/views/OnboardingFollows.svelte"
import OnboardingNote from "src/app/views/OnboardingNote.svelte"
import {Env, pubkeyLoader, Outbox, Builder, user, Keys} from "src/app/engine"
import {loadPubkeys} from "src/engine2"
import {Env, Outbox, Builder, user, Keys} from "src/app/engine"
import {listenForNotifications} from "src/app/state"
import {modal} from "src/partials/state"
@ -77,7 +78,7 @@
onMount(() => {
// Prime our database with some defaults
pubkeyLoader.load(Env.DEFAULT_FOLLOWS)
loadPubkeys(Env.DEFAULT_FOLLOWS)
})
</script>

5
src/app/views/PersonDetail.svelte
View File

@ -13,7 +13,8 @@
import PersonRelays from "src/app/shared/PersonRelays.svelte"
import PersonHandle from "src/app/shared/PersonHandle.svelte"
import PersonName from "src/app/shared/PersonName.svelte"
import {Env, Settings, pubkeyLoader, Directory, Nip65} from "src/app/engine"
import {loadPubkeys} from "src/engine2"
import {Env, Settings, Directory, Nip65} from "src/app/engine"
import PersonCircle from "src/app/shared/PersonCircle.svelte"
import PersonAbout from "src/app/shared/PersonAbout.svelte"
import PersonStats from "src/app/shared/PersonStats.svelte"
@ -39,7 +40,7 @@
info("Person", npub, $profile)
pubkeyLoader.load([pubkey], {force: true})
loadPubkeys([pubkey], {force: true})
document.title = Directory.displayProfile($profile)

1
src/engine/index.ts
View File

@ -23,7 +23,6 @@ export {FeedLoader} from "./util/FeedLoader"
export {ThreadLoader} from "./util/ThreadLoader"
export {ContextLoader} from "./util/ContextLoader"
export {Cursor, MultiCursor} from "./util/Cursor"
export {StorageAdapter} from "./util/StorageAdapter"
export {PubkeyLoader} from "./util/PubkeyLoader"
export {Subscription} from "./util/Subscription"
export {Worker} from "./util/Worker"

90
src/engine/util/PubkeyLoader.ts
View File

@ -1,90 +0,0 @@
import {without, pluck, uniq} from "ramda"
import {chunk, seconds, ensurePlural} from "hurdak"
import {personKinds, appDataKeys} from "src/util/nostr"
import {now} from "src/util/misc"
import type {Filter} from "src/engine/types"
import type {Engine} from "src/engine/Engine"
export type LoadPeopleOpts = {
relays?: string[]
kinds?: number[]
force?: boolean
}
export class PubkeyLoader {
engine: Engine
attemptedPubkeys = new Set()
constructor(engine: Engine) {
this.engine = engine
}
getStalePubkeys = (pubkeys: string[]) => {
const stale = new Set()
const since = now() - seconds(3, "hour")
for (const pubkey of pubkeys) {
if (stale.has(pubkey) || this.attemptedPubkeys.has(pubkey)) {
continue
}
this.attemptedPubkeys.add(pubkey)
if (this.engine.Directory.profiles.key(pubkey).get()?.updated_at || 0 > since) {
continue
}
stale.add(pubkey)
}
return Array.from(stale)
}
load = async (
pubkeyGroups: string | string[],
{relays, force, kinds = personKinds}: LoadPeopleOpts = {}
) => {
const rawPubkeys = ensurePlural(pubkeyGroups).reduce((a, b) => a.concat(b), [])
const pubkeys = force ? uniq(rawPubkeys) : this.getStalePubkeys(rawPubkeys)
const getChunkRelays = (chunk: string[]) => {
if (relays?.length > 0) {
return relays
}
const limit = this.engine.Settings.getSetting("relay_limit")
return this.engine.Nip65.mergeHints(
limit,
chunk.map(pubkey => this.engine.Nip65.getPubkeyHints(limit, pubkey, "write"))
)
}
const getChunkFilter = (chunk: string[]) => {
const filter = [] as Filter[]
filter.push({kinds: without([30078], kinds), authors: chunk})
// Add a separate filter for app data so we're not pulling down other people's stuff,
// or obsolete events of our own.
if (kinds.includes(30078)) {
filter.push({kinds: [30078], authors: chunk, "#d": Object.values(appDataKeys)})
}
return filter
}
await Promise.all(
pluck(
"result",
chunk(256, pubkeys).map((chunk: string[]) =>
this.engine.Network.subscribe({
relays: getChunkRelays(chunk),
filter: getChunkFilter(chunk),
timeout: 10_000,
})
)
)
)
}
}

165
src/engine/util/StorageAdapter.ts
View File

@ -1,165 +0,0 @@
import {prop, pluck, splitAt, path as getPath, sortBy} from "ramda"
import {sleep, defer, chunk, randomInt, throttle} from "hurdak"
import {Storage as LocalStorage} from "hurdak"
import type {Channel, Contact} from "src/engine/types"
import type {Engine} from "src/engine/Engine"
import {writable} from "src/engine/util/store"
import type {Collection} from "src/engine/util/store"
import {IndexedDB} from "src/engine/util/indexeddb"
const localStorageKeys = ["Keys.pubkey", "Keys.keyState", "Settings.settings"]
const sortChannels = sortBy((e: Channel) =>
e.joined ? 0 : -Math.max(e.last_checked || 0, e.last_sent || 0)
)
const sortContacts = sortBy((e: Contact) => -Math.max(e.last_checked || 0, e.last_sent || 0))
const policy = (key: string, max: number, sort: (xs: any[]) => any[]) => ({key, max, sort})
const getStore = (key: string, engine: Engine) => getPath(key.split("."), engine) as Collection<any>
export class StorageAdapter {
engine: Engine
db: IndexedDB
ready = defer()
dead = writable(false)
constructor(engine: Engine) {
this.engine = engine
Promise.all([this.syncToLocalStorage(), this.syncToIndexedDb()]).then(() => {
this.ready.resolve()
})
}
close = () => {
this.dead.set(true)
return this.db?.close()
}
clear = () => {
this.dead.set(true)
localStorage.clear()
return this.db?.delete()
}
getPubkeyWhitelist = () => {
const pubkeys = this.engine.Keys.keyState.get().map(prop("pubkey"))
return [new Set(pubkeys), this.engine.Nip02.getFollowsSet(pubkeys)]
}
sortByPubkeyWhitelist = (fallback: (x: any) => number) => (rows: Record<string, any>[]) => {
const [pubkeys, follows] = this.getPubkeyWhitelist()
return sortBy(x => {
if (pubkeys.has(x.pubkey)) {
return Number.MAX_SAFE_INTEGER
}
if (follows.has(x.pubkey)) {
return Number.MAX_SAFE_INTEGER - 1
}
return fallback(x)
}, rows)
}
syncToLocalStorage() {
for (const key of localStorageKeys) {
const store = getStore(key, this.engine)
if (Object.hasOwn(localStorage, key)) {
store.set(LocalStorage.getJson(key))
}
store.subscribe(throttle(300, $value => LocalStorage.setJson(key, $value)))
}
}
async syncToIndexedDb() {
if (window.indexedDB) {
const policies = [
policy("Nip28.channels", 2000, sortChannels),
policy("Nip28.messages", 10000, sortBy(prop("created_at"))),
policy("Nip04.contacts", 1000, sortContacts),
policy("Nip04.messages", 10000, sortBy(prop("created_at"))),
policy("Nip24.channels", 1000, sortChannels),
policy("Nip24.messages", 10000, sortBy(prop("created_at"))),
policy("Content.topics", 1000, sortBy(prop("count"))),
policy("Content.lists", 500, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Content.labels", 10000, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Directory.profiles", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Events.cache", 5000, this.sortByPubkeyWhitelist(prop("created_at"))),
policy("Nip02.graph", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Nip05.handles", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Nip57.zappers", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
policy("Nip65.relays", 2000, prop("count")),
policy("Nip65.policies", 5000, this.sortByPubkeyWhitelist(prop("updated_at"))),
]
this.db = new IndexedDB(
"nostr-engine/Storage",
3,
policies.map(({key}) => {
const store = getStore(key, this.engine)
return {
name: key,
opts: {
keyPath: store.pk,
},
}
})
)
window.addEventListener("beforeunload", () => this.close())
await this.db.open()
for (const {key} of policies) {
const store = getStore(key, this.engine)
store.set(await this.db.getAll(key))
store.subscribe(
throttle(randomInt(3000, 5000), async <T>(rows: T) => {
if (this.dead.get()) {
return
}
// Do it in small steps to avoid clogging stuff up
for (const records of chunk(100, rows as any[])) {
await this.db.bulkPut(key, records)
await sleep(50)
if (this.dead.get()) {
return
}
}
})
)
}
// Every so often randomly prune a store
setInterval(() => {
const {key, max, sort} = policies[Math.floor(policies.length * Math.random())]
const store = getStore(key, this.engine)
const data = store.get()
if (data.length < max * 1.1 || this.dead.get()) {
return
}
const [discard, keep] = splitAt(max, sort(data))
store.set(keep)
this.db.bulkDelete(key, pluck(store.pk, discard))
}, 30_000)
}
}
}

107
src/engine/util/indexeddb.js
View File

@ -1,107 +0,0 @@
// From https://gist.github.com/underground/d50e40170d54b8a0f8a3f4fdd466eee4
export class IndexedDB {
constructor(dbName, dbVersion, stores) {
this.db
this.dbName = dbName
this.dbVersion = dbVersion
this.stores = stores
}
open() {
return new Promise((resolve, reject) => {
if (!window.indexedDB) {
reject("Unsupported indexedDB")
}
const request = window.indexedDB.open(this.dbName, this.dbVersion)
request.onsuccess = e => {
this.db = request.result
resolve()
}
request.onerror = e => reject(e.target.error)
request.onupgradeneeded = e => {
this.db = e.target.result
this.stores.forEach(o => {
try {
this.db.createObjectStore(o.name, o.opts)
} catch (e) {
console.warn(e)
}
})
}
})
}
close() {
return this.db.close()
}
delete() {
window.indexedDB.deleteDatabase(this.dbName)
}
getAll(storeName) {
return new Promise((resolve, reject) => {
const store = this.db.transaction(storeName).objectStore(storeName)
const request = store.getAll()
request.onerror = e => reject(e.target.error)
request.onsuccess = e => resolve(e.target.result)
})
}
async bulkPut(storeName, data) {
const transaction = this.db.transaction(storeName, "readwrite")
const store = transaction.objectStore(storeName)
return Promise.all(
data.map(row => {
return new Promise((resolve, reject) => {
const request = store.put(row)
request.onerror = e => reject(e.target.error)
request.onsuccess = e => resolve(e.target.result)
})
})
)
}
async bulkDelete(storeName, ids) {
const transaction = this.db.transaction(storeName, "readwrite")
const store = transaction.objectStore(storeName)
return Promise.all(
ids.map(id => {
return new Promise((resolve, reject) => {
const request = store.delete(id)
request.onerror = e => reject(e.target.error)
request.onsuccess = e => resolve(e.target.result)
})
})
)
}
clear(storeName) {
return new Promise((resolve, reject) => {
const request = this.db.transaction(storeName, "readwrite").objectStore(storeName).clear()
request.onerror = e => reject(e.target.error)
request.onsuccess = e => resolve(e.target.result)
})
}
count(storeName) {
return new Promise((resolve, reject) => {
const request = this.db.transaction(storeName).objectStore(storeName).count()
request.onerror = e => reject(e.target.error)
request.onsuccess = e => resolve(e.target.result)
})
}
}

1
src/engine2/index.ts
View File

@ -1,6 +1,7 @@
export * from "./model"
export * from "./state"
export * from "./queries"
export * from "./requests"
export * from "./storage"
export * from "./projections"
export * from "./commands"

7
src/engine2/requests/index.ts
View File

@ -1,5 +1,8 @@
export * from "./context"
export * from "./cursor"
export * from "./executor"
export * from "./feed"
export * from "./pubkeys"
export * from "./publisher"
export * from "./subscription"
export * from "./cursor"
export * from "./context"
export * from "./thread"