Fix person feeds, move people to dexie for storage

This commit is contained in:
Jonathan Staab 2023-01-12 17:57:41 -08:00
parent 2f818a561e
commit 6bec3d03e3
19 changed files with 121 additions and 348 deletions

View File

@ -37,6 +37,7 @@ If you like Coracle and want to support its development, you can donate sats via
- [ ] Add no-relay gossip
- Capture certain events in a local db
- File import/export from db, NFC transfer
- [ ] Save user notes to db
# Bugs

14
package-lock.json generated
View File

@ -16,7 +16,6 @@
"extract-urls": "^1.3.2",
"fuse.js": "^6.6.2",
"hurdak": "github:ConsignCloud/hurdak",
"lz-string": "^1.4.4",
"nostr-tools": "^1.1.1",
"ramda": "^0.28.0",
"svelte-link-preview": "^0.3.3",
@ -2345,14 +2344,6 @@
"tslib": "^2.0.3"
}
},
"node_modules/lz-string": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz",
"integrity": "sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==",
"bin": {
"lz-string": "bin/bin.js"
}
},
"node_modules/magic-string": {
"version": "0.26.7",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.26.7.tgz",
@ -5352,11 +5343,6 @@
"tslib": "^2.0.3"
}
},
"lz-string": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz",
"integrity": "sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ=="
},
"magic-string": {
"version": "0.26.7",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.26.7.tgz",

View File

@ -28,7 +28,6 @@
"extract-urls": "^1.3.2",
"fuse.js": "^6.6.2",
"hurdak": "github:ConsignCloud/hurdak",
"lz-string": "^1.4.4",
"nostr-tools": "^1.1.1",
"ramda": "^0.28.0",
"svelte-link-preview": "^0.3.3",

View File

@ -1,69 +1,61 @@
import Dexie from 'dexie'
import {ensurePlural, switcherFn} from 'hurdak/lib/hurdak'
import {synced, now, timedelta} from 'src/util/misc'
import {writable} from 'svelte/store'
import {ensurePlural, createMap, switcherFn} from 'hurdak/lib/hurdak'
import {now} from 'src/util/misc'
import {personKinds} from 'src/util/nostr'
export const db = new Dexie('agent/data/db')
db.version(7).stores({
db.version(9).stores({
relays: '++url, name',
events: '++id, pubkey, created_at, loaded_at, kind, content, reply, root',
tags: '++key, event, value, created_at, loaded_at',
alerts: '++id',
alerts: '++id, created_at',
people: '++pubkey, updated_at',
})
// Some things work better as observables than database tables
export const people = writable([])
export const people = synced('agent/data/people', {})
// Bootstrap our people observable
db.people.toArray().then($p => people.set(createMap('pubkey', $p)))
// Sync to a regular object so we have a synchronous interface
let $people = {}
people.subscribe($p => {
$people = $p
})
// Our synchronous interface
export const getPerson = (pubkey, fallback = false) =>
$people[pubkey] || (fallback ? {pubkey} : null)
// Hooks
export const processEvents = async events => {
const profileUpdates = ensurePlural(events)
const profileEvents = ensurePlural(events)
.filter(e => personKinds.includes(e.kind))
people.update($people => {
for (const event of profileUpdates) {
const {pubkey, kind, content, tags} = event
const putPerson = data => {
$people[pubkey] = {
...$people[pubkey],
...data,
pubkey,
updated_at: now(),
}
}
switcherFn(kind, {
0: () => putPerson(JSON.parse(content)),
2: () => putPerson({relays: ($people[pubkey]?.relays || []).concat(content)}),
3: () => putPerson({petnames: tags}),
12165: () => putPerson({muffle: tags}),
10001: () => putPerson({relays: tags.map(t => t[0])}),
const profileUpdates = {}
for (const e of profileEvents) {
profileUpdates[e.pubkey] = {
...getPerson(e.pubkey, true),
...profileUpdates[e.pubkey],
...switcherFn(e.kind, {
0: () => JSON.parse(e.content),
2: () => ({relays: ($people[e.pubkey]?.relays || []).concat(e.content)}),
3: () => ({petnames: e.tags}),
12165: () => ({muffle: e.tags}),
10001: () => ({relays: e.tags.map(t => t[0])}),
default: () => {
console.log(`Received unsupported event type ${event.kind}`)
},
})
}),
updated_at: now(),
}
}
return $people
})
// Sync to our in memory copy
people.update($people => ({...$people, ...profileUpdates}))
// Sync to our database
await db.people.bulkPut(Object.values(profileUpdates))
}
// Periodicallly delete old event data
(function cleanup() {
const threshold = now() - timedelta(1, 'hours')
db.events.where('loaded_at').below(threshold).delete()
db.tags.where('loaded_at').below(threshold).delete()
setTimeout(cleanup, timedelta(15, 'minutes'))
})()

View File

@ -31,6 +31,12 @@ export const getMuffle = () => {
return getTagValues($user.muffle.filter(t => Math.random() < last(t)))
}
export const getFollows = pubkey => {
const person = getPerson(pubkey)
return getTagValues(person?.petnames || defaults.petnames)
}
export const getRelays = pubkey => {
let relays = getPerson(pubkey)?.relays

View File

@ -2,8 +2,9 @@ import {sortBy, pluck} from 'ramda'
import {first} from 'hurdak/lib/hurdak'
import {synced, batch, now, timedelta} from 'src/util/misc'
import {isAlert} from 'src/util/nostr'
import {listen as _listen, getRelays} from 'src/agent'
import {listen as _listen, getMuffle, db} from 'src/agent'
import loaders from 'src/app/loaders'
import query from 'src/app/query'
let listener
@ -18,12 +19,15 @@ const listen = async (relays, pubkey) => {
listener = await _listen(
relays,
[{kinds: [1, 7], '#p': [pubkey], since: start}],
batch(300, events => {
{kinds: [1, 7], '#p': [pubkey], since: start},
batch(300, async events => {
events = events.filter(e => isAlert(e, pubkey))
if (events.length > 0) {
loaders.loadNotesContext(getRelays(), events)
const context = await loaders.loadContext(relays, events)
const notes = query.threadify(events, context, {muffle: getMuffle()})
await db.alerts.bulkPut(notes)
latest.update(
$latest =>

View File

@ -38,7 +38,7 @@ const createReaction = (relays, note, content) => {
.concat([["p", note.pubkey, relay], ["e", note.id, relay, 'reply']])
)
publishEvent(relays, 7, {content, tags})
return publishEvent(relays, 7, {content, tags})
}
const createReply = (relays, note, content, mentions = []) => {

View File

@ -1,7 +1,7 @@
import {without} from 'ramda'
import {updateIn, mergeRight} from 'hurdak/lib/hurdak'
import {get} from 'svelte/store'
import {getPerson, getRelays, people, load, keys, db} from 'src/agent'
import {getPerson, getRelays, people, load, keys} from 'src/agent'
import {toast, modal, settings} from 'src/app/ui'
import cmd from 'src/app/cmd'
import alerts from 'src/app/alerts'
@ -21,15 +21,6 @@ export const login = async ({privkey, pubkey}) => {
await alerts.listen(getRelays(), pubkey)
}
export const logout = async () => {
keys.clear()
await Promise.all([
db.tags.clear(),
db.events.clear(),
])
}
export const addRelay = async url => {
const pubkey = get(keys.pubkey)
const person = getPerson(pubkey)

View File

@ -22,7 +22,9 @@ const loadPeople = (relays, pubkeys, {kinds = personKinds, force = false, ...opt
pubkeys = getStalePubkeys(pubkeys)
}
return pubkeys.length > 0 ? load(relays, {kinds, authors: pubkeys}, opts) : []
return pubkeys.length > 0
? load(relays, {kinds, authors: pubkeys}, opts)
: Promise.resolve([])
}
const loadNetwork = async (relays, pubkey) => {
@ -45,7 +47,8 @@ const loadNetwork = async (relays, pubkey) => {
}
const loadContext = async (relays, notes, {loadParents = true} = {}) => {
notes = ensurePlural(notes)
// TODO: remove this and batch context loading, or load less at a time
notes = ensurePlural(notes).slice(0, 256)
if (notes.length === 0) {
return notes
@ -76,36 +79,4 @@ const loadContext = async (relays, notes, {loadParents = true} = {}) => {
return events.concat(await loadContext(relays, parents, {loadParents: false}))
}
const loadNotesContext = async (relays, notes, {loadParents = false} = {}) => {
notes = ensurePlural(notes)
if (notes.length === 0) {
return
}
const authors = uniq(pluck('pubkey', notes)).filter(k => !getPerson(k))
const parentIds = loadParents ? uniq(notes.map(findReply).filter(identity)) : []
const filter = [{kinds: [1, 5, 7], '#e': pluck('id', notes)}]
// Load authors if needed
if (authors.length > 0) {
filter.push({kinds: personKinds, authors})
}
// Load the note parents
if (parentIds.length > 0) {
filter.push({kinds: [1], ids: parentIds})
}
// Load the events
const events = await load(relays, filter)
const eventsById = createMap('id', events)
const parents = parentIds.map(id => eventsById[id]).filter(identity)
// Load the parents' context as well
if (parents.length > 0) {
await loadNotesContext(relays, parents)
}
}
export default {loadNotesContext, loadNetwork, loadPeople, personKinds, loadContext}
export default {loadNetwork, loadPeople, personKinds, loadContext}

View File

@ -1,132 +1,11 @@
import {get} from 'svelte/store'
import {intersection, sortBy, propEq, uniqBy, groupBy, concat, prop, isNil, identity} from 'ramda'
import {ensurePlural, createMap, ellipsize} from 'hurdak/lib/hurdak'
import {sortBy, identity} from 'ramda'
import {createMap, ellipsize} from 'hurdak/lib/hurdak'
import {renderContent} from 'src/util/html'
import {Tags, displayPerson, getTagValues, findReply, findRoot} from 'src/util/nostr'
import {db, people, getPerson} from 'src/agent'
import {Tags, displayPerson, findReply} from 'src/util/nostr'
import {people, getPerson} from 'src/agent'
import {routes} from "src/app/ui"
const filterEvents = async ({limit, ...filter}) => {
let events = db.events
// Sorting is expensive, so prioritize that unless we have a filter that will dramatically
// reduce the number of results so we can do ordering in memory
if (filter.ids) {
events = await db.events.where('id').anyOf(ensurePlural(filter.ids)).reverse().sortBy('created')
} else if (filter.authors) {
events = await db.events.where('pubkey').anyOf(ensurePlural(filter.authors)).reverse().sortBy('created')
} else {
events = await events.orderBy('created_at').reverse().toArray()
}
const result = []
for (const e of events) {
if (filter.ids && !filter.ids.includes(e.id)) continue
if (filter.authors && !filter.authors.includes(e.pubkey)) continue
if (filter.muffle && filter.muffle.includes(e.pubkey)) continue
if (filter.kinds && !filter.kinds.includes(e.kind)) continue
if (filter.since && filter.since > e.created_at) continue
if (filter.until && filter.until < e.created_at) continue
if (filter['#p'] && intersection(filter['#p'], getTagValues(e.tags)).length === 0) continue
if (filter['#e'] && intersection(filter['#e'], getTagValues(e.tags)).length === 0) continue
if (!isNil(filter.content) && filter.content !== e.content) continue
if (filter.customFilter && !filter.customFilter(e)) continue
result.push(e)
if (result.length > limit) {
break
}
}
return result
}
const filterReplies = async (id, filter) => {
const events = await db.events.where('reply').equals(id).toArray()
return events.filter(e => e.kind === 1)
}
const filterReactions = async (id, filter) => {
const events = await db.events.where('reply').equals(id).toArray()
return events.filter(e => e.kind === 7)
}
const findNote = async (id, {showEntire = false, depth = 1} = {}) => {
const note = await db.events.get(id)
if (!note) {
return null
}
const reactions = await filterReactions(note.id)
const replies = await filterReplies(note.id)
const person = getPerson(note.pubkey)
const html = await renderNote(note, {showEntire})
let parent = null
const parentId = findReply(note)
if (parentId) {
parent = await db.events.get(parentId)
if (parent) {
parent = {
...parent,
reactions: await filterReactions(parent.id),
person: getPerson(parent.pubkey),
html: await renderNote(parent, {showEntire}),
}
}
}
return {
...note, reactions, person, html, parent,
repliesCount: replies.length,
replies: depth === 0
? []
: await Promise.all(
sortBy(e => e.created_at, replies)
.slice(showEntire ? 0 : -3)
.map(r => findNote(r.id, {depth: depth - 1}))
),
}
}
const annotateChunk = async chunk => {
const ancestorIds = concat(chunk.map(findRoot), chunk.map(findReply)).filter(identity)
const ancestors = await filterEvents({kinds: [1], ids: ancestorIds})
const allNotes = uniqBy(prop('id'), chunk.concat(ancestors))
const notesById = createMap('id', allNotes)
const notesByRoot = groupBy(
n => {
const rootId = findRoot(n)
const parentId = findReply(n)
// Actually dereference the notes in case we weren't able to retrieve them
if (notesById[rootId]) {
return rootId
}
if (notesById[parentId]) {
return parentId
}
return n.id
},
allNotes
)
const notes = await Promise.all(Object.keys(notesByRoot).map(findNote))
// Re-sort, since events come in order regardless of level in the hierarchy.
// This is really a hack, since a single like can bump an old note back up to the
// top of the feed. Also, discard non-notes (e.g. reactions)
return sortBy(e => -e.created_at, notes.filter(propEq('kind', 1)))
}
const renderNote = (note, {showEntire = false}) => {
const shouldEllipsize = note.content.length > 500 && !showEntire
const $people = get(people)
@ -202,4 +81,4 @@ const threadify = (events, context, {muffle = []} = {}) => {
})
}
export default {filterEvents, filterReplies, filterReactions, annotateChunk, renderNote, findNote, threadify, annotate}
export default {renderNote, threadify, annotate}

View File

@ -1,6 +1,6 @@
<script>
import {nip19} from 'nostr-tools'
import NoteDetail from 'src/routes/NoteDetail.svelte'
import NoteDetail from 'src/views/NoteDetail.svelte'
export let entity
@ -9,7 +9,7 @@
<div class="py-4 max-w-xl m-auto">
{#if type === "nevent"}
<NoteDetail {...data} />
<NoteDetail note={{id: data.id}} relays={data.relays} />
{/if}
</div>

View File

@ -1,15 +1,10 @@
<script>
import {fly} from 'svelte/transition'
import {db} from 'src/agent'
setTimeout(async () => {
// Clear localstorage
localStorage.clear()
// Keep relays around, but delete events/tags
await db.events.clear()
await db.tags.clear()
// do a hard refresh so everything gets totally cleared
window.location = '/login'
}, 300)

View File

@ -1,29 +0,0 @@
<script>
import {fly} from 'svelte/transition'
import {loadNote} from 'src/app'
import Note from 'src/partials/Note.svelte'
import Spinner from 'src/partials/Spinner.svelte'
export let id
export let relays
let note = {id}
console.log(id, relays)
loadNote(relays, id).then(found => {
note = found
})
</script>
{#if !note}
<div class="p-4 text-center text-white" in:fly={{y: 20}}>
Sorry, we weren't able to find this note.
</div>
{:else if note.pubkey}
<div in:fly={{y: 20}}>
<Note invertColors anchorId={note.id} note={note} depth={2} />
</div>
{:else}
<Spinner />
{/if}

View File

@ -1,13 +1,12 @@
<script>
import {find, propEq, reject} from 'ramda'
import {find, reject} from 'ramda'
import {onMount, onDestroy} from 'svelte'
import {nip19} from 'nostr-tools'
import {first} from 'hurdak/lib/hurdak'
import {fly} from 'svelte/transition'
import {navigate} from 'svelte-routing'
import {now, batch} from 'src/util/misc'
import {renderContent} from 'src/util/html'
import {displayPerson, personKinds} from 'src/util/nostr'
import {displayPerson} from 'src/util/nostr'
import Tabs from "src/partials/Tabs.svelte"
import Button from "src/partials/Button.svelte"
import Notes from "src/views/person/Notes.svelte"
@ -30,24 +29,12 @@
let person = getPerson(pubkey, true)
onMount(async () => {
subs.push(await listen(
getRelays(pubkey),
[{kinds: [1, 5, 7], authors: [pubkey], since: now()},
{kinds: personKinds, authors: [pubkey]}],
batch(300, events => {
const profiles = events.filter(propEq('kind', 0))
const notes = events.filter(propEq('kind', 1))
if (profiles.length > 0) {
person = getPerson(pubkey, true)
}
if (notes.length > 0) {
loaders.loadNoteContext(notes)
}
})
))
// Refresh our person if needed
loaders.loadPeople(getRelays(pubkey), [pubkey]).then(() => {
person = getPerson(pubkey, true)
})
// Get our followers count
subs.push(await listen(
getRelays(pubkey),
[{kinds: [3], '#p': [pubkey]}],

View File

@ -1,4 +1,3 @@
import LZ from 'lz-string'
import {debounce} from 'throttle-debounce'
import {pluck, sortBy} from "ramda"
import Fuse from "fuse.js/dist/fuse.min.js"
@ -17,7 +16,7 @@ export const hash = s =>
export const getLocalJson = (k) => {
try {
return JSON.parse(LZ.decompress(localStorage.getItem(k)))
return JSON.parse(localStorage.getItem(k))
} catch (e) {
console.warn(`Unable to parse ${k}: ${e}`)
@ -25,9 +24,9 @@ export const getLocalJson = (k) => {
}
}
export const setLocalJson = (k, v, compressed = false) => {
export const setLocalJson = (k, v) => {
try {
localStorage.setItem(k, LZ.compress(JSON.stringify(v)))
localStorage.setItem(k, JSON.stringify(v))
} catch (e) {
console.warn(`Unable to set ${k}: ${e}`)
}

View File

@ -1,18 +1,10 @@
<script>
import Notes from "src/partials/Notes.svelte"
import {now, timedelta, shuffle, batch, Cursor} from 'src/util/misc'
import {getTagValues} from 'src/util/nostr'
import {user, getRelays, getMuffle, getPerson, listen, load} from 'src/agent'
import defaults from 'src/agent/defaults'
import {user, getRelays, getFollows, getMuffle, listen, load} from 'src/agent'
import loaders from 'src/app/loaders'
import query from 'src/app/query'
const getFollows = pubkey => {
const person = getPerson(pubkey)
return getTagValues(person?.petnames || defaults.petnames)
}
// Get first- and second-order follows. shuffle and slice network so we're not
// sending too many pubkeys. This will also result in some variety.
const relays = getRelays()

View File

@ -1,32 +1,31 @@
<script>
import Notes from "src/partials/Notes.svelte"
import {timedelta, Cursor} from 'src/util/misc'
import {getTagValues} from 'src/util/nostr'
import {user, load, getRelays} from 'src/agent'
import {timedelta, now, batch, Cursor} from 'src/util/misc'
import {load, listen, getRelays, getMuffle} from 'src/agent'
import loaders from 'src/app/loaders'
import query from 'src/app/query'
export let pubkey
const relays = getRelays(pubkey)
const filter = {kinds: [7], authors: [pubkey]}
const cursor = new Cursor(timedelta(1, 'days'))
const listenForNotes = onNotes =>
listen(relays, {...filter, since: now()}, batch(300, async notes => {
const context = await loaders.loadContext(relays, notes)
onNotes(query.threadify(notes, context, {muffle: getMuffle()}))
}))
const loadNotes = async () => {
const [since, until] = cursor.step()
const filter = {kinds: [7], authors: [pubkey], since, until}
const notes = await load(getRelays(pubkey), filter)
const notes = await load(relays, {...filter, since, until})
const context = await loaders.loadContext(relays, notes)
await loaders.loadNotesContext(getRelays(pubkey), notes, {loadParents: true})
}
const queryNotes = () => {
return query.filterEvents({
kinds: [7],
since: cursor.since,
authors: [pubkey],
muffle: getTagValues($user?.muffle || []),
})
return query.threadify(notes, context, {muffle: getMuffle()})
}
</script>
<Notes shouldMuffle {loadNotes} {queryNotes} />
<Notes {listenForNotes} {loadNotes} />

View File

@ -1,33 +1,33 @@
<script>
import Notes from "src/partials/Notes.svelte"
import {timedelta, Cursor} from 'src/util/misc'
import {getTagValues} from 'src/util/nostr'
import {load, user, getRelays} from 'src/agent'
import {now, timedelta, shuffle, batch, Cursor} from 'src/util/misc'
import {getRelays, getFollows, getMuffle, listen, load} from 'src/agent'
import loaders from 'src/app/loaders'
import query from 'src/app/query'
export let person
export let pubkey
const relays = getRelays(pubkey)
const follows = getFollows(pubkey)
const network = shuffle(follows.flatMap(getFollows)).slice(0, 50)
const authors = follows.concat(network)
const filter = {kinds: [1, 7], authors}
const cursor = new Cursor(timedelta(1, 'hours'))
const listenForNotes = onNotes =>
listen(relays, {...filter, since: now()}, batch(300, async notes => {
const context = await loaders.loadContext(relays, notes)
onNotes(query.threadify(notes, context, {muffle: getMuffle()}))
}))
const loadNotes = async () => {
const [since, until] = cursor.step()
const authors = getTagValues(person.petnames)
const filter = {since, until, kinds: [1], authors}
const events = await load(getRelays(person.pubkey), filter)
const notes = await load(relays, {...filter, since, until})
const context = await loaders.loadContext(relays, notes)
await loaders.loadNotesContext(getRelays(person.pubkey), events, {loadParents: true})
}
const queryNotes = () => {
return query.filterEvents({
kinds: [1],
since: cursor.since,
authors: getTagValues(person.petnames),
muffle: getTagValues($user?.muffle || []),
})
return query.threadify(notes, context, {muffle: getMuffle()})
}
</script>
<Notes shouldMuffle {loadNotes} {queryNotes} />
<Notes {listenForNotes} {loadNotes} />

View File

@ -1,30 +1,31 @@
<script>
import Notes from "src/partials/Notes.svelte"
import {timedelta, Cursor} from 'src/util/misc'
import {load, getRelays} from 'src/agent'
import {timedelta, now, batch, Cursor} from 'src/util/misc'
import {load, listen, getRelays, getMuffle} from 'src/agent'
import loaders from 'src/app/loaders'
import query from 'src/app/query'
export let pubkey
const relays = getRelays(pubkey)
const filter = {kinds: [1], authors: [pubkey]}
const cursor = new Cursor(timedelta(1, 'days'))
const listenForNotes = onNotes =>
listen(relays, {...filter, since: now()}, batch(300, async notes => {
const context = await loaders.loadContext(relays, notes)
onNotes(query.threadify(notes, context, {muffle: getMuffle()}))
}))
const loadNotes = async () => {
const [since, until] = cursor.step()
const filter = {kinds: [1], authors: [pubkey], since, until}
const notes = await load(getRelays(pubkey), filter)
const notes = await load(relays, {...filter, since, until})
const context = await loaders.loadContext(relays, notes)
await loaders.loadNotesContext(getRelays(pubkey), notes, {loadParents: true})
}
const queryNotes = () => {
return query.filterEvents({
kinds: [1],
since: cursor.since,
authors: [pubkey],
})
return query.threadify(notes, context, {muffle: getMuffle()})
}
</script>
<Notes shouldMuffle {loadNotes} {queryNotes} />
<Notes {listenForNotes} {loadNotes} />