From 6bec3d03e332b85ac41aee57c5960ea378a6d426 Mon Sep 17 00:00:00 2001 From: Jonathan Staab Date: Thu, 12 Jan 2023 17:57:41 -0800 Subject: [PATCH] Fix person feeds, move people to dexie for storage --- README.md | 1 + package-lock.json | Bin 243524 -> 242924 bytes package.json | 1 - src/agent/data.js | 70 ++++++++--------- src/agent/index.js | 6 ++ src/app/alerts.js | 12 ++- src/app/cmd.js | 2 +- src/app/index.js | 11 +-- src/app/loaders.js | 41 ++-------- src/app/query.js | 131 ++------------------------------ src/routes/Bech32Entity.svelte | 4 +- src/routes/Logout.svelte | 5 -- src/routes/NoteDetail.svelte | 29 ------- src/routes/Person.svelte | 27 ++----- src/util/misc.js | 7 +- src/views/notes/Network.svelte | 10 +-- src/views/person/Likes.svelte | 31 ++++---- src/views/person/Network.svelte | 38 ++++----- src/views/person/Notes.svelte | 29 +++---- 19 files changed, 121 insertions(+), 334 deletions(-) delete mode 100644 src/routes/NoteDetail.svelte diff --git a/README.md b/README.md index 3c2a2cb9..697d6414 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ If you like Coracle and want to support its development, you can donate sats via - [ ] Add no-relay gossip - Capture certain events in a local db - File import/export from db, NFC transfer +- [ ] Save user notes to db # Bugs diff --git a/package-lock.json b/package-lock.json index dc5685106c42ec5ef169791bb0f3ceb30620c884..8308d013095a80ce45b4afb216e0ae2ddd7941a1 100644 GIT binary patch delta 34 qcmX@|j_=J&z71~}o4J{`b2Bk=d$ou8GHws^Wzs#ny+)e(#B2ceOAR0Z delta 396 zcmaF!lJCenz71~}g>$NOi%W_!^U{^96qMo&^-T0knjbK3f56Bn?!^q0oSwLZQFi)- zB1V?UMeRnCaOL{C`f#=`P}_9D$BZhG2FckK=20F&0fk8h`JuTMRenXcX_hX5 zPG$xkhULD_SxzNsnPx`bImU_RUP&JLNoMJeMoG@v6+vZD`TjoIh8b0%-WFM@rLLZV zt`@~*u7S3;N|PD&q{QGBg4~*vnWqoLdRfyGl^Ipqzk4%o|L)DCdzNHJC(_K(Tna!? mtHU*&QI$z#de#y~@#(TYjBlqKC^Lyo4v^8`UL(a^Fbe?69D$qw diff --git a/package.json b/package.json index b7efa9aa..88006d7a 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,6 @@ "extract-urls": "^1.3.2", "fuse.js": "^6.6.2", "hurdak": "github:ConsignCloud/hurdak", - "lz-string": "^1.4.4", "nostr-tools": "^1.1.1", "ramda": "^0.28.0", "svelte-link-preview": "^0.3.3", diff --git a/src/agent/data.js b/src/agent/data.js index bef1de31..1e50c502 100644 --- a/src/agent/data.js +++ b/src/agent/data.js @@ -1,69 +1,61 @@ import Dexie from 'dexie' -import {ensurePlural, switcherFn} from 'hurdak/lib/hurdak' -import {synced, now, timedelta} from 'src/util/misc' +import {writable} from 'svelte/store' +import {ensurePlural, createMap, switcherFn} from 'hurdak/lib/hurdak' +import {now} from 'src/util/misc' import {personKinds} from 'src/util/nostr' export const db = new Dexie('agent/data/db') -db.version(7).stores({ +db.version(9).stores({ relays: '++url, name', - events: '++id, pubkey, created_at, loaded_at, kind, content, reply, root', - tags: '++key, event, value, created_at, loaded_at', - alerts: '++id', + alerts: '++id, created_at', + people: '++pubkey, updated_at', }) // Some things work better as observables than database tables +export const people = writable([]) -export const people = synced('agent/data/people', {}) +// Bootstrap our people observable +db.people.toArray().then($p => people.set(createMap('pubkey', $p))) +// Sync to a regular object so we have a synchronous interface let $people = {} people.subscribe($p => { $people = $p }) +// Our synchronous interface export const getPerson = (pubkey, fallback = false) => $people[pubkey] || (fallback ? {pubkey} : null) // Hooks export const processEvents = async events => { - const profileUpdates = ensurePlural(events) + const profileEvents = ensurePlural(events) .filter(e => personKinds.includes(e.kind)) - people.update($people => { - for (const event of profileUpdates) { - const {pubkey, kind, content, tags} = event - const putPerson = data => { - $people[pubkey] = { - ...$people[pubkey], - ...data, - pubkey, - updated_at: now(), - } - } - - switcherFn(kind, { - 0: () => putPerson(JSON.parse(content)), - 2: () => putPerson({relays: ($people[pubkey]?.relays || []).concat(content)}), - 3: () => putPerson({petnames: tags}), - 12165: () => putPerson({muffle: tags}), - 10001: () => putPerson({relays: tags.map(t => t[0])}), + const profileUpdates = {} + for (const e of profileEvents) { + profileUpdates[e.pubkey] = { + ...getPerson(e.pubkey, true), + ...profileUpdates[e.pubkey], + ...switcherFn(e.kind, { + 0: () => JSON.parse(e.content), + 2: () => ({relays: ($people[e.pubkey]?.relays || []).concat(e.content)}), + 3: () => ({petnames: e.tags}), + 12165: () => ({muffle: e.tags}), + 10001: () => ({relays: e.tags.map(t => t[0])}), default: () => { console.log(`Received unsupported event type ${event.kind}`) }, - }) + }), + updated_at: now(), } + } - return $people - }) + // Sync to our in memory copy + people.update($people => ({...$people, ...profileUpdates})) + + // Sync to our database + await db.people.bulkPut(Object.values(profileUpdates)) } - -// Periodicallly delete old event data -(function cleanup() { - const threshold = now() - timedelta(1, 'hours') - - db.events.where('loaded_at').below(threshold).delete() - db.tags.where('loaded_at').below(threshold).delete() - - setTimeout(cleanup, timedelta(15, 'minutes')) -})() diff --git a/src/agent/index.js b/src/agent/index.js index f3dcbe4a..6694be6a 100644 --- a/src/agent/index.js +++ b/src/agent/index.js @@ -31,6 +31,12 @@ export const getMuffle = () => { return getTagValues($user.muffle.filter(t => Math.random() < last(t))) } +export const getFollows = pubkey => { + const person = getPerson(pubkey) + + return getTagValues(person?.petnames || defaults.petnames) +} + export const getRelays = pubkey => { let relays = getPerson(pubkey)?.relays diff --git a/src/app/alerts.js b/src/app/alerts.js index 25d15745..a932afba 100644 --- a/src/app/alerts.js +++ b/src/app/alerts.js @@ -2,8 +2,9 @@ import {sortBy, pluck} from 'ramda' import {first} from 'hurdak/lib/hurdak' import {synced, batch, now, timedelta} from 'src/util/misc' import {isAlert} from 'src/util/nostr' -import {listen as _listen, getRelays} from 'src/agent' +import {listen as _listen, getMuffle, db} from 'src/agent' import loaders from 'src/app/loaders' +import query from 'src/app/query' let listener @@ -18,12 +19,15 @@ const listen = async (relays, pubkey) => { listener = await _listen( relays, - [{kinds: [1, 7], '#p': [pubkey], since: start}], - batch(300, events => { + {kinds: [1, 7], '#p': [pubkey], since: start}, + batch(300, async events => { events = events.filter(e => isAlert(e, pubkey)) if (events.length > 0) { - loaders.loadNotesContext(getRelays(), events) + const context = await loaders.loadContext(relays, events) + const notes = query.threadify(events, context, {muffle: getMuffle()}) + + await db.alerts.bulkPut(notes) latest.update( $latest => diff --git a/src/app/cmd.js b/src/app/cmd.js index cdd9bfab..d7266005 100644 --- a/src/app/cmd.js +++ b/src/app/cmd.js @@ -38,7 +38,7 @@ const createReaction = (relays, note, content) => { .concat([["p", note.pubkey, relay], ["e", note.id, relay, 'reply']]) ) - publishEvent(relays, 7, {content, tags}) + return publishEvent(relays, 7, {content, tags}) } const createReply = (relays, note, content, mentions = []) => { diff --git a/src/app/index.js b/src/app/index.js index bb327ac9..5b78c1c0 100644 --- a/src/app/index.js +++ b/src/app/index.js @@ -1,7 +1,7 @@ import {without} from 'ramda' import {updateIn, mergeRight} from 'hurdak/lib/hurdak' import {get} from 'svelte/store' -import {getPerson, getRelays, people, load, keys, db} from 'src/agent' +import {getPerson, getRelays, people, load, keys} from 'src/agent' import {toast, modal, settings} from 'src/app/ui' import cmd from 'src/app/cmd' import alerts from 'src/app/alerts' @@ -21,15 +21,6 @@ export const login = async ({privkey, pubkey}) => { await alerts.listen(getRelays(), pubkey) } -export const logout = async () => { - keys.clear() - - await Promise.all([ - db.tags.clear(), - db.events.clear(), - ]) -} - export const addRelay = async url => { const pubkey = get(keys.pubkey) const person = getPerson(pubkey) diff --git a/src/app/loaders.js b/src/app/loaders.js index 89b5f2df..08d10b5a 100644 --- a/src/app/loaders.js +++ b/src/app/loaders.js @@ -22,7 +22,9 @@ const loadPeople = (relays, pubkeys, {kinds = personKinds, force = false, ...opt pubkeys = getStalePubkeys(pubkeys) } - return pubkeys.length > 0 ? load(relays, {kinds, authors: pubkeys}, opts) : [] + return pubkeys.length > 0 + ? load(relays, {kinds, authors: pubkeys}, opts) + : Promise.resolve([]) } const loadNetwork = async (relays, pubkey) => { @@ -45,7 +47,8 @@ const loadNetwork = async (relays, pubkey) => { } const loadContext = async (relays, notes, {loadParents = true} = {}) => { - notes = ensurePlural(notes) + // TODO: remove this and batch context loading, or load less at a time + notes = ensurePlural(notes).slice(0, 256) if (notes.length === 0) { return notes @@ -76,36 +79,4 @@ const loadContext = async (relays, notes, {loadParents = true} = {}) => { return events.concat(await loadContext(relays, parents, {loadParents: false})) } -const loadNotesContext = async (relays, notes, {loadParents = false} = {}) => { - notes = ensurePlural(notes) - - if (notes.length === 0) { - return - } - - const authors = uniq(pluck('pubkey', notes)).filter(k => !getPerson(k)) - const parentIds = loadParents ? uniq(notes.map(findReply).filter(identity)) : [] - const filter = [{kinds: [1, 5, 7], '#e': pluck('id', notes)}] - - // Load authors if needed - if (authors.length > 0) { - filter.push({kinds: personKinds, authors}) - } - - // Load the note parents - if (parentIds.length > 0) { - filter.push({kinds: [1], ids: parentIds}) - } - - // Load the events - const events = await load(relays, filter) - const eventsById = createMap('id', events) - const parents = parentIds.map(id => eventsById[id]).filter(identity) - - // Load the parents' context as well - if (parents.length > 0) { - await loadNotesContext(relays, parents) - } -} - -export default {loadNotesContext, loadNetwork, loadPeople, personKinds, loadContext} +export default {loadNetwork, loadPeople, personKinds, loadContext} diff --git a/src/app/query.js b/src/app/query.js index 40f3ef27..c6f96958 100644 --- a/src/app/query.js +++ b/src/app/query.js @@ -1,132 +1,11 @@ import {get} from 'svelte/store' -import {intersection, sortBy, propEq, uniqBy, groupBy, concat, prop, isNil, identity} from 'ramda' -import {ensurePlural, createMap, ellipsize} from 'hurdak/lib/hurdak' +import {sortBy, identity} from 'ramda' +import {createMap, ellipsize} from 'hurdak/lib/hurdak' import {renderContent} from 'src/util/html' -import {Tags, displayPerson, getTagValues, findReply, findRoot} from 'src/util/nostr' -import {db, people, getPerson} from 'src/agent' +import {Tags, displayPerson, findReply} from 'src/util/nostr' +import {people, getPerson} from 'src/agent' import {routes} from "src/app/ui" -const filterEvents = async ({limit, ...filter}) => { - let events = db.events - - // Sorting is expensive, so prioritize that unless we have a filter that will dramatically - // reduce the number of results so we can do ordering in memory - if (filter.ids) { - events = await db.events.where('id').anyOf(ensurePlural(filter.ids)).reverse().sortBy('created') - } else if (filter.authors) { - events = await db.events.where('pubkey').anyOf(ensurePlural(filter.authors)).reverse().sortBy('created') - } else { - events = await events.orderBy('created_at').reverse().toArray() - } - - const result = [] - for (const e of events) { - if (filter.ids && !filter.ids.includes(e.id)) continue - if (filter.authors && !filter.authors.includes(e.pubkey)) continue - if (filter.muffle && filter.muffle.includes(e.pubkey)) continue - if (filter.kinds && !filter.kinds.includes(e.kind)) continue - if (filter.since && filter.since > e.created_at) continue - if (filter.until && filter.until < e.created_at) continue - if (filter['#p'] && intersection(filter['#p'], getTagValues(e.tags)).length === 0) continue - if (filter['#e'] && intersection(filter['#e'], getTagValues(e.tags)).length === 0) continue - if (!isNil(filter.content) && filter.content !== e.content) continue - if (filter.customFilter && !filter.customFilter(e)) continue - - result.push(e) - - if (result.length > limit) { - break - } - } - - return result -} - -const filterReplies = async (id, filter) => { - const events = await db.events.where('reply').equals(id).toArray() - - return events.filter(e => e.kind === 1) -} - -const filterReactions = async (id, filter) => { - const events = await db.events.where('reply').equals(id).toArray() - - return events.filter(e => e.kind === 7) -} - -const findNote = async (id, {showEntire = false, depth = 1} = {}) => { - const note = await db.events.get(id) - - if (!note) { - return null - } - - const reactions = await filterReactions(note.id) - const replies = await filterReplies(note.id) - const person = getPerson(note.pubkey) - const html = await renderNote(note, {showEntire}) - - let parent = null - const parentId = findReply(note) - if (parentId) { - parent = await db.events.get(parentId) - - if (parent) { - parent = { - ...parent, - reactions: await filterReactions(parent.id), - person: getPerson(parent.pubkey), - html: await renderNote(parent, {showEntire}), - } - } - } - - return { - ...note, reactions, person, html, parent, - repliesCount: replies.length, - replies: depth === 0 - ? [] - : await Promise.all( - sortBy(e => e.created_at, replies) - .slice(showEntire ? 0 : -3) - .map(r => findNote(r.id, {depth: depth - 1})) - ), - } -} - -const annotateChunk = async chunk => { - const ancestorIds = concat(chunk.map(findRoot), chunk.map(findReply)).filter(identity) - const ancestors = await filterEvents({kinds: [1], ids: ancestorIds}) - - const allNotes = uniqBy(prop('id'), chunk.concat(ancestors)) - const notesById = createMap('id', allNotes) - const notesByRoot = groupBy( - n => { - const rootId = findRoot(n) - const parentId = findReply(n) - - // Actually dereference the notes in case we weren't able to retrieve them - if (notesById[rootId]) { - return rootId - } - - if (notesById[parentId]) { - return parentId - } - - return n.id - }, - allNotes - ) - - const notes = await Promise.all(Object.keys(notesByRoot).map(findNote)) - - // Re-sort, since events come in order regardless of level in the hierarchy. - // This is really a hack, since a single like can bump an old note back up to the - // top of the feed. Also, discard non-notes (e.g. reactions) - return sortBy(e => -e.created_at, notes.filter(propEq('kind', 1))) -} - const renderNote = (note, {showEntire = false}) => { const shouldEllipsize = note.content.length > 500 && !showEntire const $people = get(people) @@ -202,4 +81,4 @@ const threadify = (events, context, {muffle = []} = {}) => { }) } -export default {filterEvents, filterReplies, filterReactions, annotateChunk, renderNote, findNote, threadify, annotate} +export default {renderNote, threadify, annotate} diff --git a/src/routes/Bech32Entity.svelte b/src/routes/Bech32Entity.svelte index 41f2dae4..4a797b1d 100644 --- a/src/routes/Bech32Entity.svelte +++ b/src/routes/Bech32Entity.svelte @@ -1,6 +1,6 @@ - -{#if !note} -
- Sorry, we weren't able to find this note. -
-{:else if note.pubkey} -
- -
-{:else} - -{/if} diff --git a/src/routes/Person.svelte b/src/routes/Person.svelte index 82769c71..d5d26c70 100644 --- a/src/routes/Person.svelte +++ b/src/routes/Person.svelte @@ -1,13 +1,12 @@ - + diff --git a/src/views/person/Network.svelte b/src/views/person/Network.svelte index 2edd04df..3727dc9d 100644 --- a/src/views/person/Network.svelte +++ b/src/views/person/Network.svelte @@ -1,33 +1,33 @@ - - + diff --git a/src/views/person/Notes.svelte b/src/views/person/Notes.svelte index c2016751..271802d4 100644 --- a/src/views/person/Notes.svelte +++ b/src/views/person/Notes.svelte @@ -1,30 +1,31 @@ - +