From 7a3338eeaa55771e2565ed7098bfcaca0d38dcfa Mon Sep 17 00:00:00 2001 From: Jonathan Staab Date: Mon, 26 Dec 2022 09:43:55 -0800 Subject: [PATCH] Switch from cache-first to cache-last by always attempting to retrieve messages from the network with an aggressive timeout. --- README.md | 9 ++++++ src/App.svelte | 2 +- src/partials/Notes.svelte | 36 ++++++++++++++++++------ src/relay/index.js | 38 +++++++++++++------------ src/relay/pool.js | 39 ++++++++++++++++++++------ src/routes/Alerts.svelte | 7 ++--- src/routes/Person.svelte | 44 ++++++----------------------- src/routes/RelayList.svelte | 4 ++- src/util/html.js | 2 -- src/util/misc.js | 19 ++++++++++--- src/views/NoteDetail.svelte | 2 +- src/views/notes/Global.svelte | 41 +++++++++++++-------------- src/views/notes/Network.svelte | 49 +++++++++++++++------------------ src/views/person/Likes.svelte | 23 ++++++++++++++++ src/views/person/Network.svelte | 24 ++++++++++++++++ src/views/person/Notes.svelte | 21 ++++++++++++++ 16 files changed, 229 insertions(+), 131 deletions(-) create mode 100644 src/views/person/Likes.svelte create mode 100644 src/views/person/Network.svelte create mode 100644 src/views/person/Notes.svelte diff --git a/README.md b/README.md index bcf599cc..0895cd11 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,15 @@ If you like Coracle and want to support its development, you can donate sats via # Changelog +## 0.2.2 + +- [x] Show notification for new notes rather than automatically adding them to the feed +- [x] Improve slow relay pruning by using a timeout for each relay +- [x] Re-work feed loading - go to network first and fall back to cache to ensure results that are as complete as possible +- [x] Slightly improved context fetching to reduce subscriptions +- [x] Split person feeds out into separate components +- [x] Add timeout in scroller to keep polling for new results + ## 0.2.1 - [x] Exclude people from search who have no profile data available diff --git a/src/App.svelte b/src/App.svelte index 8ef8cd35..4571d924 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -10,7 +10,7 @@ import {Router, Route, links, navigate} from "svelte-routing" import {globalHistory} from "svelte-routing/src/history" import {hasParent} from 'src/util/html' - import {timedelta, getLastSync, now} from 'src/util/misc' + import {timedelta, now} from 'src/util/misc' import {store as toast} from "src/state/toast" import {modal, settings, alerts} from "src/state/app" import relay, {user, connections} from 'src/relay' diff --git a/src/partials/Notes.svelte b/src/partials/Notes.svelte index e3124944..3c5af31a 100644 --- a/src/partials/Notes.svelte +++ b/src/partials/Notes.svelte @@ -1,32 +1,52 @@ -{#if notes} -{/if} diff --git a/src/relay/index.js b/src/relay/index.js index c7f73428..d5e6c06e 100644 --- a/src/relay/index.js +++ b/src/relay/index.js @@ -1,7 +1,7 @@ import {liveQuery} from 'dexie' import extractUrls from 'extract-urls' import {get} from 'svelte/store' -import {intersection, pluck, sortBy, uniq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda' +import {intersection, find, sortBy, propEq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda' import {ensurePlural, first, createMap, ellipsize} from 'hurdak/lib/hurdak' import {escapeHtml} from 'src/util/html' import {filterTags, getTagValues, findReply, findRoot} from 'src/util/nostr' @@ -103,7 +103,7 @@ const findNote = async (id, {showEntire = false, depth = 1} = {}) => { ? [] : await Promise.all( sortBy(e => -e.created_at, replies) - .slice(0, showEntire ? Infinity : 5) + .slice(0, showEntire ? Infinity : 3) .map(r => findNote(r.id, {depth: depth - 1})) ), } @@ -134,10 +134,12 @@ const annotateChunk = async chunk => { allNotes ) + const notes = await Promise.all(Object.keys(notesByRoot).map(findNote)) + // Re-sort, since events come in order regardless of level in the hierarchy. // This is really a hack, since a single like can bump an old note back up to the - // top of the feed - return sortBy(e => -e.created_at, await Promise.all(Object.keys(notesByRoot).map(findNote))) + // top of the feed. Also, discard non-notes (e.g. reactions) + return sortBy(e => -e.created_at, notes.filter(propEq('kind', 1))) } const renderNote = async (note, {showEntire = false}) => { @@ -218,27 +220,27 @@ const unfollow = async pubkey => { // This is intended only for bootstrapping listeners const loadNoteContext = async (note, {loadParent = false} = {}) => { - // Load note context - this assumes that we are looking at a feed, and so - // we already have the note's parent and its likes loaded. + const $people = get(people) const filter = [{kinds: [1, 5, 7], '#e': [note.id]}] - if (!prop(note.pubkey, get(db.people))) { + // Load the author if needed + if (!$people[note.pubkey]) { filter.push({kinds: [0], authors: [note.pubkey]}) } - // Load the events + // Load the note's parent + const parentId = findReply(note) + if (loadParent && parentId) { + filter.push({kinds: [1], ids: [parentId]}) + } + + // Load the events const events = await pool.loadEvents(filter) - // Load any related people we're missing - const $people = get(people) - await pool.loadPeople( - uniq(pluck('pubkey', events)).filter(k => !$people[k]) - ) - - // Load the note's parent - const replyId = findReply(note) - if (loadParent && replyId) { - await getOrLoadNote(replyId) + // Load the note's context as well + const parent = find(propEq('id', parentId), events) + if (loadParent && parent) { + await loadNoteContext(parent) } } diff --git a/src/relay/pool.js b/src/relay/pool.js index 814e21ce..c7ae6a8d 100644 --- a/src/relay/pool.js +++ b/src/relay/pool.js @@ -36,8 +36,18 @@ class Channel { // Start our subscription, wait for only our fastest relays to eose before calling it done. // We were waiting for all before, but that made the slowest relay a bottleneck. Waiting for // only one meant we might be settling for very incomplete data + const start = new Date().valueOf() + const lastEvent = {} const eoseRelays = [] - const sub = pool.sub({filter, cb: onEvent}, this.name, r => { + + // Keep track of when we last heard from each relay, and close unresponsive ones + const cb = (e, r) => { + lastEvent[r] = new Date().valueOf() + onEvent(e) + } + + // If we have lots of relays, ignore the slowest ones + const onRelayEose = r => { eoseRelays.push(r) // If we have only a few, wait for all of them, otherwise ignore the slowest 1/5 @@ -45,21 +55,30 @@ class Channel { if (eoseRelays.length >= relays.length - threshold) { onEose() } - }) + } + // Create our subscription + const sub = pool.sub({filter, cb}, this.name, onRelayEose) + + // Watch for relays that are slow to respond and give up on them + const interval = !opts.timeout ? null : setInterval(() => { + for (const r of relays) { + if ((lastEvent[r] || start) < new Date().valueOf() - opts.timeout) { + onRelayEose(r) + } + } + }, 300) + + // Clean everything up when we're done const done = () => { if (this.status === 'busy') { sub.unsub() } + clearInterval(interval) this.release() } - // If the relay takes to long, just give up - if (opts.timeout) { - setTimeout(done, opts.timeout) - } - return {unsub: done} } all(filter, opts = {}) { @@ -75,7 +94,7 @@ class Channel { resolve(uniqBy(prop('id'), result)) }, - {timeout: 30000, ...opts}, + {timeout: 3000, ...opts}, ) }) } @@ -159,6 +178,8 @@ const listenForEvents = async (key, filter, onEvent) => { onEvent(e) } }) + + return listenForEvents.subs[key] } listenForEvents.subs = {} @@ -177,7 +198,7 @@ const syncNetwork = async () => { let pubkeys = [] if ($user) { // Get this user's profile to start with - await loadPeople([$user.pubkey], {timeout: null}) + await loadPeople([$user.pubkey]) // Get our refreshed person const people = get(db.people) diff --git a/src/routes/Alerts.svelte b/src/routes/Alerts.svelte index 8e9183a3..197f7253 100644 --- a/src/routes/Alerts.svelte +++ b/src/routes/Alerts.svelte @@ -5,7 +5,7 @@ import {alerts} from 'src/state/app' import {findReply} from 'src/util/nostr' import relay, {people, user} from 'src/relay' - import {now, timedelta, createScroller, Cursor, getLastSync} from 'src/util/misc' + import {now, timedelta, createScroller, Cursor} from 'src/util/misc' import Spinner from "src/partials/Spinner.svelte" import Note from 'src/partials/Note.svelte' import Like from 'src/partials/Like.svelte' @@ -15,10 +15,7 @@ let notes let limit = 0 - const cursor = new Cursor( - getLastSync('routes/Alerts'), - timedelta(1, 'days') - ) + const cursor = new Cursor(timedelta(1, 'hours')) onMount(async () => { sub = await relay.pool.listenForEvents( diff --git a/src/routes/Person.svelte b/src/routes/Person.svelte index cc20cfea..8585a2b4 100644 --- a/src/routes/Person.svelte +++ b/src/routes/Person.svelte @@ -1,13 +1,14 @@ {#if !note} -
+
Sorry, we weren't able to find this note.
{:else if $observable} diff --git a/src/views/notes/Global.svelte b/src/views/notes/Global.svelte index 48d79f9e..b616c469 100644 --- a/src/views/notes/Global.svelte +++ b/src/views/notes/Global.svelte @@ -2,19 +2,21 @@ import {when, propEq} from 'ramda' import {onMount, onDestroy} from 'svelte' import Notes from "src/partials/Notes.svelte" - import {timedelta, now, Cursor} from 'src/util/misc' + import {timedelta, Cursor} from 'src/util/misc' import {getTagValues} from 'src/util/nostr' import relay, {user} from 'src/relay' - let sub - - const cursor = new Cursor(now(), timedelta(1, 'minutes')) + let notes, sub onMount(async () => { sub = await relay.pool.listenForEvents( 'views/notes/Global', [{kinds: [1, 5, 7], since: cursor.since}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), async e => { + await relay.loadNoteContext(e) + + notes.addNewNotes([e]) + }) ) }) @@ -24,24 +26,23 @@ } }) - const loadNotes = async limit => { - const notes = await relay.filterEvents({ - limit, + const cursor = new Cursor(timedelta(1, 'minutes')) + + const loadNotes = async () => { + const [since, until] = cursor.step() + + await relay.pool.loadEvents( + [{kinds: [1, 5, 7], since, until}], + when(propEq('kind', 1), relay.loadNoteContext) + ) + + return relay.filterEvents({ + since, + until, kinds: [1], muffle: getTagValues($user?.muffle || []), }) - - if (notes.length <= limit) { - const [since, until] = cursor.step() - - relay.pool.loadEvents( - [{kinds: [1, 5, 7], since, until}], - when(propEq('kind', 1), relay.loadNoteContext) - ) - } - - return relay.annotateChunk(notes.slice(0, limit)) } - + diff --git a/src/views/notes/Network.svelte b/src/views/notes/Network.svelte index f2d755ed..1f3e0b26 100644 --- a/src/views/notes/Network.svelte +++ b/src/views/notes/Network.svelte @@ -2,17 +2,11 @@ import {when, propEq} from 'ramda' import {onMount, onDestroy} from 'svelte' import Notes from "src/partials/Notes.svelte" - import {timedelta, Cursor, getLastSync} from 'src/util/misc' + import {timedelta, Cursor} from 'src/util/misc' import {getTagValues} from 'src/util/nostr' import relay, {user, network} from 'src/relay' - let sub - let networkUnsub - - const cursor = new Cursor( - getLastSync('views/notes/Network'), - timedelta(1, 'hours') - ) + let notes, sub, networkUnsub onMount(() => { // We need to re-create the sub when network changes, since this is where @@ -22,7 +16,11 @@ sub = await relay.pool.listenForEvents( 'views/notes/Network', [{kinds: [1, 5, 7], authors: $network, since: cursor.since}], - when(propEq('kind', 1), relay.loadNoteContext) + when(propEq('kind', 1), async e => { + await relay.loadNoteContext(e) + + notes.addNewNotes([e]) + }) ) }) }) @@ -35,28 +33,25 @@ } }) - const loadNotes = async limit => { - const notes = await relay.filterEvents({ - limit, + const cursor = new Cursor(timedelta(10, 'minutes')) + + const loadNotes = async () => { + const [since, until] = cursor.step() + + await relay.pool.loadEvents( + [{kinds: [1, 5, 7], authors: $network, since, until}], + when(propEq('kind', 1), relay.loadNoteContext) + ) + + return relay.filterEvents({ + since, + until, kinds: [1], authors: $network.concat($user.pubkey), muffle: getTagValues($user?.muffle || []), }) - - if (notes.length <= limit) { - const [since, until] = cursor.step() - - relay.pool.loadEvents( - [{kinds: [1, 5, 7], authors: $network, since, until}], - when(propEq('kind', 1), relay.loadNoteContext) - ) - } - - return relay.annotateChunk(notes.slice(0, limit)) } - -{#key $network.map(n => n[0]).join('')} - -{/key} + + diff --git a/src/views/person/Likes.svelte b/src/views/person/Likes.svelte new file mode 100644 index 00000000..78c87ce1 --- /dev/null +++ b/src/views/person/Likes.svelte @@ -0,0 +1,23 @@ + + + + diff --git a/src/views/person/Network.svelte b/src/views/person/Network.svelte new file mode 100644 index 00000000..fbc4300b --- /dev/null +++ b/src/views/person/Network.svelte @@ -0,0 +1,24 @@ + + + + diff --git a/src/views/person/Notes.svelte b/src/views/person/Notes.svelte new file mode 100644 index 00000000..03711a51 --- /dev/null +++ b/src/views/person/Notes.svelte @@ -0,0 +1,21 @@ + + + +