From 3097b9e7e83d2c9408a8f5b52f03ff33e88d355d Mon Sep 17 00:00:00 2001 From: Jonathan Staab Date: Fri, 30 Dec 2022 13:29:06 -0800 Subject: [PATCH] Batch load context on feeds. This makes loading faster, and provides more context --- README.md | 12 +++++++--- src/relay/index.js | 42 ++++++++++++++++++++------------- src/routes/Alerts.svelte | 31 +++++++++--------------- src/routes/Person.svelte | 2 +- src/views/NoteDetail.svelte | 2 +- src/views/notes/Global.svelte | 12 ++++------ src/views/notes/Network.svelte | 12 ++++------ src/views/person/Likes.svelte | 8 +++++-- src/views/person/Network.svelte | 12 +++++----- src/views/person/Notes.svelte | 9 +++---- 10 files changed, 74 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 498a9912..6b7c192c 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,26 @@ If you like Coracle and want to support its development, you can donate sats via - [ ] An actual readme - [ ] Server discovery and relay publishing - https://github.com/nostr-protocol/nips/pull/32/files - [ ] Support invoices https://twitter.com/jb55/status/1604131336247476224 -- [ ] Expand/collapse large threads - [ ] NIP 05 +- [ ] Lightning tips +- [ ] Direct messages +- [ ] Rooms/groups # Bugs +- [ ] Follow fiatjaf's vision of clients being smart and connecting to recommended relays to fetch content - [ ] Add alerts for replies to posts the user liked - [ ] Stack views so scroll position isn't lost on navigation -- [ ] Add notification for slow relays +- [ ] Add notification for slow relays, suggest relays based on network - [ ] Separating events table into notes/reactions/etc would effectively give us a second index on kind. - [ ] Clicking on a badge in the popover falls through, and might also crash -- [ ] Add a slider in settings so users can decide whether to go with fast relays, or wait for everyone to complete their queries. Most relevant for NoteDetail # Changelog +## 0.2.5 + +- [x] Batch load context for feeds + ## 0.2.4 - [x] Fix reactions - livequery is required in order to listen for changes diff --git a/src/relay/index.js b/src/relay/index.js index fe9dd2e9..a99bb85c 100644 --- a/src/relay/index.js +++ b/src/relay/index.js @@ -1,7 +1,7 @@ import {liveQuery} from 'dexie' import extractUrls from 'extract-urls' import {get} from 'svelte/store' -import {intersection, find, sortBy, propEq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda' +import {uniq, pluck, intersection, sortBy, propEq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda' import {ensurePlural, first, createMap, ellipsize} from 'hurdak/lib/hurdak' import {escapeHtml} from 'src/util/html' import {filterTags, getTagValues, findReply, findRoot} from 'src/util/nostr' @@ -221,28 +221,36 @@ const unfollow = async pubkey => { // Methods that wil attempt to load from the database and fall back to the network. // This is intended only for bootstrapping listeners -const loadNoteContext = async (note, {loadParent = false} = {}) => { - const $people = get(people) - const filter = [{kinds: [1, 5, 7], '#e': [note.id]}] +const loadNotesContext = async (notes, {loadParents = false} = {}) => { + notes = ensurePlural(notes) - // Load the author if needed - if (!$people[note.pubkey]) { - filter.push({kinds: [0], authors: [note.pubkey]}) + if (notes.length === 0) { + return } - // Load the note's parent - const parentId = findReply(note) - if (loadParent && parentId) { - filter.push({kinds: [1], ids: [parentId]}) + const $people = get(people) + const authors = uniq(pluck('pubkey', notes)).filter(k => !$people[k]) + const parentIds = loadParents ? uniq(notes.map(findReply).filter(identity)) : [] + const filter = [{kinds: [1, 5, 7], '#e': pluck('id', notes)}] + + // Load authors if needed + if (authors.length > 0) { + filter.push({kinds: [0], authors}) + } + + // Load the note parents + if (parentIds.length > 0) { + filter.push({kinds: [1], ids: parentIds}) } // Load the events const events = await pool.loadEvents(filter) + const eventsById = createMap('id', events) + const parents = parentIds.map(id => eventsById[id]).filter(identity) - // Load the note's context as well - const parent = find(propEq('id', parentId), events) - if (loadParent && parent) { - await loadNoteContext(parent) + // Load the parents' context as well + if (parents.length > 0) { + await loadNotesContext(parents) } } @@ -254,7 +262,7 @@ const getOrLoadNote = async id => { const note = await db.events.get(id) if (note) { - await loadNoteContext(note, {loadParent: true}) + await loadNotesContext([note], {loadParent: true}) } return note @@ -296,5 +304,5 @@ export const connections = db.connections export default { db, pool, cmd, lq, filterEvents, getOrLoadNote, filterReplies, findNote, annotateChunk, renderNote, login, addRelay, removeRelay, - follow, unfollow, loadNoteContext, + follow, unfollow, loadNotesContext, } diff --git a/src/routes/Alerts.svelte b/src/routes/Alerts.svelte index 740e29c5..e6033bf0 100644 --- a/src/routes/Alerts.svelte +++ b/src/routes/Alerts.svelte @@ -16,7 +16,7 @@ onMount(() => { const scroller = createScroller(async () => { - notes = notes.concat(await loadNotes()) + notes = uniqBy(prop('id'), notes.concat(await loadNotes())) }) return () => scroller.stop() @@ -24,26 +24,16 @@ const loadNotes = async () => { const [since, until] = cursor.step() + const filter = {kinds: [1, 7], '#p': [$user.pubkey], since, until} - await relay.pool.loadEvents( - [{kinds: [1, 7], '#p': [$user.pubkey], since, until}], - e => { - if (e.kind === 1) { - relay.loadNoteContext(e) - } - - if (e.kind === 7) { - const replyId = findReply(e) - - if (replyId) { - relay.getOrLoadNote(replyId) - } - } - - alerts.set({since: now()}) - } + // Load all our alerts and their context + await relay.loadNotesContext( + await relay.pool.loadEvents(filter), + {loadParents: true} ) + alerts.set({since: now()}) + const events = await relay.filterEvents({ since, until, @@ -80,7 +70,6 @@ // Combine likes of a single note. Remove grandchild likes const likesById = {} - const alerts = notes.filter(e => e.pubkey !== $user.pubkey) for (const reaction of reactions.filter(e => e.parent?.pubkey === $user.pubkey)) { if (!likesById[reaction.parent.id]) { likesById[reaction.parent.id] = {...reaction.parent, people: []} @@ -91,7 +80,9 @@ return sortBy( e => -e.created_at, - uniqBy(prop('id'), alerts.concat(Object.values(likesById))) + notes + .filter(e => e.pubkey !== $user.pubkey) + .concat(Object.values(likesById)) ) } diff --git a/src/routes/Person.svelte b/src/routes/Person.svelte index 692b30ad..45a8730a 100644 --- a/src/routes/Person.svelte +++ b/src/routes/Person.svelte @@ -22,7 +22,7 @@ sub = await relay.pool.listenForEvents( 'routes/Person', [{kinds: [0, 1, 5, 7], authors: [pubkey], since: now()}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), relay.loadNotesContext) ) }) diff --git a/src/views/NoteDetail.svelte b/src/views/NoteDetail.svelte index 53f133e7..f2a42878 100644 --- a/src/views/NoteDetail.svelte +++ b/src/views/NoteDetail.svelte @@ -21,7 +21,7 @@ sub = await relay.pool.listenForEvents( 'routes/NoteDetail', [{kinds: [1, 5, 7], '#e': [note.id], since: now()}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), relay.loadNotesContext) ) } }) diff --git a/src/views/notes/Global.svelte b/src/views/notes/Global.svelte index 6e140701..41974ed4 100644 --- a/src/views/notes/Global.svelte +++ b/src/views/notes/Global.svelte @@ -12,7 +12,7 @@ sub = await relay.pool.listenForEvents( 'views/notes/Global', [{kinds: [1, 5, 7], since: cursor.since}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), relay.loadNotesContext) ) }) @@ -24,13 +24,11 @@ const cursor = new Cursor(timedelta(1, 'minutes')) - const loadNotes = () => { + const loadNotes = async () => { const [since, until] = cursor.step() - - return relay.pool.loadEvents( - [{kinds: [1, 5, 7], since, until}], - when(propEq('kind', 1), relay.loadNoteContext) - ) + const filter = {kinds: [1], since, until} + const notes = await relay.pool.loadEvents(filter) + await relay.loadNotesContext(notes, {loadParents: true}) } const queryNotes = () => { diff --git a/src/views/notes/Network.svelte b/src/views/notes/Network.svelte index 4ce362cd..7eac2348 100644 --- a/src/views/notes/Network.svelte +++ b/src/views/notes/Network.svelte @@ -16,7 +16,7 @@ sub = await relay.pool.listenForEvents( 'views/notes/Network', [{kinds: [1, 5, 7], authors: $network, since: cursor.since}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), relay.loadNotesContext) ) }) }) @@ -31,13 +31,11 @@ const cursor = new Cursor(timedelta(10, 'minutes')) - const loadNotes = () => { + const loadNotes = async () => { const [since, until] = cursor.step() - - return relay.pool.loadEvents( - [{kinds: [1, 5, 7], authors: $network, since, until}], - when(propEq('kind', 1), relay.loadNoteContext) - ) + const filter = {kinds: [1, 7], authors: $network, since, until} + const notes = await relay.pool.loadEvents(filter) + await relay.loadNotesContext(notes, {loadParents: true}) } const queryNotes = () => { diff --git a/src/views/person/Likes.svelte b/src/views/person/Likes.svelte index 2d65cc76..4cb09a22 100644 --- a/src/views/person/Likes.svelte +++ b/src/views/person/Likes.svelte @@ -8,10 +8,14 @@ const cursor = new Cursor(timedelta(1, 'days')) - const loadNotes = () => { + const loadNotes = async () => { const [since, until] = cursor.step() + const filter = {kinds: [7], authors: [pubkey], since, until} - return relay.pool.loadEvents({kinds: [7], authors: [pubkey], since, until}) + await relay.loadEventsContext( + await relay.pool.loadEvents(filter), + {loadParents: true} + ) } const queryNotes = () => { diff --git a/src/views/person/Network.svelte b/src/views/person/Network.svelte index 5306bbb2..d3136e58 100644 --- a/src/views/person/Network.svelte +++ b/src/views/person/Network.svelte @@ -10,13 +10,13 @@ const loadNotes = async () => { const [since, until] = cursor.step() + const authors = getTagValues(person.petnames) + const filter = {since, until, kinds: [1], authors} - return relay.pool.loadEvents({ - since, - until, - kinds: [1], - authors: getTagValues(person.petnames), - }) + await relay.loadEventsContext( + await relay.pool.loadEvents(filter), + {loadParents: true} + ) } const queryNotes = () => { diff --git a/src/views/person/Notes.svelte b/src/views/person/Notes.svelte index a61973fa..f25420f1 100644 --- a/src/views/person/Notes.svelte +++ b/src/views/person/Notes.svelte @@ -7,12 +7,13 @@ const cursor = new Cursor(timedelta(1, 'days')) - const loadNotes = () => { + const loadNotes = async () => { const [since, until] = cursor.step() + const filter = {kinds: [1], authors: [pubkey], since, until} - return relay.pool.loadEvents( - [{kinds: [1], authors: [pubkey], since, until}], - relay.loadNoteContext + await relay.loadNotesContext( + await relay.pool.loadEvents(filter), + {loadParents: true} ) }