Switch from cache-first to cache-last by always attempting to retrieve messages from the network with an aggressive timeout.

README.md

@@ -36,6 +36,15 @@ If you like Coracle and want to support its development, you can donate sats via
 
 # Changelog
 
+## 0.2.2
+
+- [x] Show notification for new notes rather than automatically adding them to the feed
+- [x] Improve slow relay pruning by using a timeout for each relay
+- [x] Re-work feed loading - go to network first and fall back to cache to ensure results that are as complete as possible
+- [x] Slightly improved context fetching to reduce subscriptions
+- [x] Split person feeds out into separate components
+- [x] Add timeout in scroller to keep polling for new results
+
 ## 0.2.1
 
 - [x] Exclude people from search who have no profile data available

src/App.svelte

@@ -10,7 +10,7 @@
   import {Router, Route, links, navigate} from "svelte-routing"
   import {globalHistory} from "svelte-routing/src/history"
   import {hasParent} from 'src/util/html'
-  import {timedelta, getLastSync, now} from 'src/util/misc'
+  import {timedelta, now} from 'src/util/misc'
   import {store as toast} from "src/state/toast"
   import {modal, settings, alerts} from "src/state/app"
   import relay, {user, connections} from 'src/relay'

src/partials/Notes.svelte

@@ -1,32 +1,52 @@
 <script>
+  import {sortBy, uniqBy, reject, prop} from 'ramda'
   import {onDestroy} from 'svelte'
+  import {slide} from 'svelte/transition'
+  import {quantify} from 'hurdak/lib/hurdak'
   import {createScroller} from 'src/util/misc'
+  import {findReply} from 'src/util/nostr'
   import Spinner from 'src/partials/Spinner.svelte'
   import Note from "src/partials/Note.svelte"
   import relay from 'src/relay'
 
   export let loadNotes
+  export const addNewNotes = xs => {
+    newNotes = newNotes.concat(xs)
+  }
 
-  let notes
-  let limit = 0
+  let notes = []
+  let newNotes = []
+  let newNotesLength = 0
+
+  $: newNotesLength = reject(findReply, newNotes).length
 
   const scroller = createScroller(async () => {
-    limit += 20
-
-    notes = relay.lq(() => loadNotes(limit))
+    addNotes(await loadNotes())
   })
 
+  const addNotes = async xs => {
+    const chunk = await relay.annotateChunk(xs)
+
+    notes = sortBy(e => -e.created_at, uniqBy(prop('id'), notes.concat(chunk)))
+  }
+
   onDestroy(() => {
     scroller.stop()
   })
 </script>
 
-{#if notes}
 <ul class="py-4 flex flex-col gap-2 max-w-xl m-auto">
-  {#each ($notes || []) as n (n.id)}
+  {#if newNotesLength > 0}
+  <div
+    transition:slide
+    class="mb-2 cursor-pointer text-center underline text-light"
+    on:click={() => { addNotes(newNotes); newNotes = [] }}>
+    Load {quantify(newNotesLength, 'new note')}
+  </div>
+  {/if}
+  {#each notes as n (n.id)}
     <li><Note note={n} depth={2} /></li>
   {/each}
 </ul>
-{/if}
 
 <Spinner />

src/relay/index.js

@@ -1,7 +1,7 @@
 import {liveQuery} from 'dexie'
 import extractUrls from 'extract-urls'
 import {get} from 'svelte/store'
-import {intersection, pluck, sortBy, uniq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda'
+import {intersection, find, sortBy, propEq, uniqBy, groupBy, concat, without, prop, isNil, identity} from 'ramda'
 import {ensurePlural, first, createMap, ellipsize} from 'hurdak/lib/hurdak'
 import {escapeHtml} from 'src/util/html'
 import {filterTags, getTagValues, findReply, findRoot} from 'src/util/nostr'
@@ -103,7 +103,7 @@ const findNote = async (id, {showEntire = false, depth = 1} = {}) => {
       ? []
       : await Promise.all(
           sortBy(e => -e.created_at, replies)
-          .slice(0, showEntire ? Infinity : 5)
+          .slice(0, showEntire ? Infinity : 3)
           .map(r => findNote(r.id, {depth: depth - 1}))
       ),
   }
@@ -134,10 +134,12 @@ const annotateChunk = async chunk => {
     allNotes
   )
 
+  const notes = await Promise.all(Object.keys(notesByRoot).map(findNote))
+
   // Re-sort, since events come in order regardless of level in the hierarchy.
   // This is really a hack, since a single like can bump an old note back up to the
-  // top of the feed
-  return sortBy(e => -e.created_at, await Promise.all(Object.keys(notesByRoot).map(findNote)))
+  // top of the feed. Also, discard non-notes (e.g. reactions)
+  return sortBy(e => -e.created_at, notes.filter(propEq('kind', 1)))
 }
 
 const renderNote = async (note, {showEntire = false}) => {
@@ -218,27 +220,27 @@ const unfollow = async pubkey => {
 // This is intended only for bootstrapping listeners
 
 const loadNoteContext = async (note, {loadParent = false} = {}) => {
-  // Load note context - this assumes that we are looking at a feed, and so
-  // we already have the note's parent and its likes loaded.
+  const $people = get(people)
   const filter = [{kinds: [1, 5, 7], '#e': [note.id]}]
 
-  if (!prop(note.pubkey, get(db.people))) {
+  // Load the author if needed
+  if (!$people[note.pubkey]) {
     filter.push({kinds: [0], authors: [note.pubkey]})
   }
 
-  // Load  the events
+  // Load the note's parent
+  const parentId = findReply(note)
+  if (loadParent && parentId) {
+    filter.push({kinds: [1], ids: [parentId]})
+  }
+
+  // Load the events
   const events = await pool.loadEvents(filter)
 
-  // Load any related people we're missing
-  const $people = get(people)
-  await pool.loadPeople(
-    uniq(pluck('pubkey', events)).filter(k => !$people[k])
-  )
-
-  // Load the note's parent
-  const replyId = findReply(note)
-  if (loadParent && replyId) {
-    await getOrLoadNote(replyId)
+  // Load the note's context as well
+  const parent = find(propEq('id', parentId), events)
+  if (loadParent && parent) {
+    await loadNoteContext(parent)
   }
 }
 

src/relay/pool.js

@@ -36,8 +36,18 @@ class Channel {
     // Start our subscription, wait for only our fastest relays to eose before calling it done.
     // We were waiting for all before, but that made the slowest relay a bottleneck. Waiting for
     // only one meant we might be settling for very incomplete data
+    const start = new Date().valueOf()
+    const lastEvent = {}
     const eoseRelays = []
-    const sub = pool.sub({filter, cb: onEvent}, this.name, r => {
+
+    // Keep track of when we last heard from each relay, and close unresponsive ones
+    const cb = (e, r) => {
+      lastEvent[r] = new Date().valueOf()
+      onEvent(e)
+    }
+
+    // If we have lots of relays, ignore the slowest ones
+    const onRelayEose = r => {
       eoseRelays.push(r)
 
       // If we have only a few, wait for all of them, otherwise ignore the slowest 1/5
@@ -45,21 +55,30 @@ class Channel {
       if (eoseRelays.length >= relays.length - threshold) {
         onEose()
       }
-    })
+    }
 
+    // Create our subscription
+    const sub = pool.sub({filter, cb}, this.name, onRelayEose)
+
+    // Watch for relays that are slow to respond and give up on them
+    const interval = !opts.timeout ? null : setInterval(() => {
+      for (const r of relays) {
+        if ((lastEvent[r] || start) < new Date().valueOf() - opts.timeout) {
+          onRelayEose(r)
+        }
+      }
+    }, 300)
+
+    // Clean everything up when we're done
     const done = () => {
       if (this.status === 'busy') {
         sub.unsub()
       }
 
+      clearInterval(interval)
       this.release()
     }
 
-    // If the relay takes to long, just give up
-    if (opts.timeout) {
-      setTimeout(done, opts.timeout)
-    }
-
     return {unsub: done}
   }
   all(filter, opts = {}) {
@@ -75,7 +94,7 @@ class Channel {
 
           resolve(uniqBy(prop('id'), result))
         },
-        {timeout: 30000, ...opts},
+        {timeout: 3000, ...opts},
       )
     })
   }
@@ -159,6 +178,8 @@ const listenForEvents = async (key, filter, onEvent) => {
       onEvent(e)
     }
   })
+
+  return listenForEvents.subs[key]
 }
 
 listenForEvents.subs = {}
@@ -177,7 +198,7 @@ const syncNetwork = async () => {
   let pubkeys = []
   if ($user) {
     // Get this user's profile to start with
-    await loadPeople([$user.pubkey], {timeout: null})
+    await loadPeople([$user.pubkey])
 
     // Get our refreshed person
     const people = get(db.people)

src/routes/Alerts.svelte

@@ -5,7 +5,7 @@
   import {alerts} from 'src/state/app'
   import {findReply} from 'src/util/nostr'
   import relay, {people, user} from 'src/relay'
-  import {now, timedelta, createScroller, Cursor, getLastSync} from 'src/util/misc'
+  import {now, timedelta, createScroller, Cursor} from 'src/util/misc'
   import Spinner from "src/partials/Spinner.svelte"
   import Note from 'src/partials/Note.svelte'
   import Like from 'src/partials/Like.svelte'
@@ -15,10 +15,7 @@
   let notes
   let limit = 0
 
-  const cursor = new Cursor(
-    getLastSync('routes/Alerts'),
-    timedelta(1, 'days')
-  )
+  const cursor = new Cursor(timedelta(1, 'hours'))
 
   onMount(async () => {
     sub = await relay.pool.listenForEvents(

src/routes/Person.svelte

@@ -1,13 +1,14 @@
 <script>
-  import {find, take, when, propEq} from 'ramda'
+  import {find, when, propEq} from 'ramda'
   import {onMount, onDestroy} from 'svelte'
   import {fly} from 'svelte/transition'
   import {navigate} from 'svelte-routing'
-  import {getLastSync} from 'src/util/misc'
-  import {getTagValues} from 'src/util/nostr'
+  import {now, timedelta} from 'src/util/misc'
   import Tabs from "src/partials/Tabs.svelte"
   import Button from "src/partials/Button.svelte"
-  import Notes from "src/partials/Notes.svelte"
+  import Notes from "src/views/person/Notes.svelte"
+  import Likes from "src/views/person/Likes.svelte"
+  import Network from "src/views/person/Network.svelte"
   import {modal} from "src/state/app"
   import relay, {user, people} from 'src/relay'
 
@@ -16,12 +17,11 @@
 
   let sub = null
   let following = $user && find(t => t[1] === pubkey, $user.petnames)
-  let since = getLastSync(['Person', pubkey])
 
   onMount(async () => {
     sub = await relay.pool.listenForEvents(
       'routes/Person',
-      [{kind: [0, 1, 5, 7], authors: [pubkey], since}],
+      [{kinds: [0, 1, 5, 7], authors: [pubkey], since: now()}],
       when(propEq('kind', 1), relay.loadNoteContext)
     )
   })
@@ -34,32 +34,6 @@
 
   const getPerson = () => $people[pubkey]
 
-  const loadNotes = async limit => {
-    const filter = {kinds: [1], authors: [pubkey]}
-
-    return relay.annotateChunk(take(limit, await relay.filterEvents(filter)))
-  }
-
-  const loadLikes = async limit => {
-    const events = await relay.annotateChunk(
-      take(limit, await relay.filterEvents({
-        kinds: [7],
-        authors: [pubkey],
-        muffle: getTagValues($user?.muffle || []),
-      }))
-    )
-
-    return events.filter(e => e.kind === 1)
-  }
-
-  const loadNetwork = async limit => {
-    return relay.annotateChunk(take(limit, await relay.filterEvents({
-      kinds: [1],
-      authors: getTagValues(getPerson().petnames),
-      muffle: getTagValues($user?.muffle || []),
-    })))
-  }
-
   const setActiveTab = tab => navigate(`/people/${pubkey}/${tab}`)
 
   const follow = () => {
@@ -115,12 +89,12 @@
 
 <Tabs tabs={['notes', 'likes', 'network']} {activeTab} {setActiveTab} />
 {#if activeTab === 'notes'}
-<Notes loadNotes={loadNotes} />
+<Notes {pubkey} />
 {:else if activeTab === 'likes'}
-<Notes loadNotes={loadLikes} />
+<Likes {pubkey} />
 {:else if activeTab === 'network'}
 {#if getPerson()}
-<Notes shouldMuffle loadNotes={loadNetwork} />
+<Network person={getPerson()} />
 {:else}
 <div class="py-16 max-w-xl m-auto flex justify-center">
   Unable to show network for this person.

src/routes/RelayList.svelte

@@ -77,7 +77,9 @@
         </div>
         {/if}
       {/each}
-      <small class="text-center">Found {($knownRelays || []).length} known relays</small>
+      <small class="text-center">
+        Showing {Math.min(($knownRelays || []).length, 50)} of {($knownRelays || []).length} known relays
+      </small>
     </div>
   </div>
 </div>

src/util/html.js

@@ -1,5 +1,3 @@
-import {first} from 'hurdak/lib/hurdak'
-
 export const copyToClipboard = text => {
   const {activeElement} = document
   const input = document.createElement("textarea")

src/util/misc.js

@@ -53,6 +53,7 @@ export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 export const createScroller = loadMore => {
   let done = false
   let didLoad = false
+  let timeout = null
   const check = async () => {
     // While we have empty space, fill it
     const {scrollY, innerHeight} = window
@@ -61,7 +62,15 @@ export const createScroller = loadMore => {
 
     // Only trigger loading the first time we reach the threshhold
     if (shouldLoad && !didLoad) {
+      clearTimeout(timeout)
+
       await loadMore()
+
+      // If nothing loads, the page doesn't reflow and we get stuck.
+      // Give it a generous timeout from last time something did load
+      timeout = setTimeout(() => {
+        didLoad = false
+      }, 5000)
     }
 
     didLoad = shouldLoad
@@ -72,6 +81,7 @@ export const createScroller = loadMore => {
     if (!done) {
       requestAnimationFrame(check)
     }
+
   }
 
   requestAnimationFrame(check)
@@ -99,14 +109,15 @@ export const getLastSync = (k, fallback = 0) => {
 }
 
 export class Cursor {
-  constructor(since, delta) {
-    this.since = (since || now()) - delta,
+  constructor(delta) {
+    this.since = now()
+    this.until = now()
     this.delta = delta
   }
   step() {
-    const until = this.since
+    this.until = this.since
     this.since -= this.delta
 
-    return [this.since, until]
+    return [this.since, this.until]
   }
 }

src/views/NoteDetail.svelte

@@ -44,7 +44,7 @@
 </script>
 
 {#if !note}
-<div class="text-white" in:fly={{y: 20}}>
+<div class="p-4 text-center text-white" in:fly={{y: 20}}>
   Sorry, we weren't able to find this note.
 </div>
 {:else if $observable}

src/views/notes/Global.svelte

@@ -2,19 +2,21 @@
   import {when, propEq} from 'ramda'
   import {onMount, onDestroy} from 'svelte'
   import Notes from "src/partials/Notes.svelte"
-  import {timedelta, now, Cursor} from 'src/util/misc'
+  import {timedelta, Cursor} from 'src/util/misc'
   import {getTagValues} from 'src/util/nostr'
   import relay, {user} from 'src/relay'
 
-  let sub
-
-  const cursor = new Cursor(now(), timedelta(1, 'minutes'))
+  let notes, sub
 
   onMount(async () => {
     sub = await relay.pool.listenForEvents(
       'views/notes/Global',
       [{kinds: [1, 5, 7], since: cursor.since}],
-      when(propEq('kind', 1), relay.loadNoteContext)
+      when(propEq('kind', 1), async e => {
+        await relay.loadNoteContext(e)
+
+        notes.addNewNotes([e])
+      })
     )
   })
 
@@ -24,24 +26,23 @@
     }
   })
 
-  const loadNotes = async limit => {
-    const notes = await relay.filterEvents({
-      limit,
+  const cursor = new Cursor(timedelta(1, 'minutes'))
+
+  const loadNotes = async () => {
+    const [since, until] = cursor.step()
+
+    await relay.pool.loadEvents(
+      [{kinds: [1, 5, 7], since, until}],
+      when(propEq('kind', 1), relay.loadNoteContext)
+    )
+
+    return relay.filterEvents({
+      since,
+      until,
       kinds: [1],
       muffle: getTagValues($user?.muffle || []),
     })
-
-    if (notes.length <= limit) {
-      const [since, until] = cursor.step()
-
-      relay.pool.loadEvents(
-        [{kinds: [1, 5, 7], since, until}],
-        when(propEq('kind', 1), relay.loadNoteContext)
-      )
-    }
-
-    return relay.annotateChunk(notes.slice(0, limit))
   }
 </script>
 
-<Notes shouldMuffle loadNotes={loadNotes} />
+<Notes bind:this={notes} shouldMuffle {loadNotes} />

src/views/notes/Network.svelte

@@ -2,17 +2,11 @@
   import {when, propEq} from 'ramda'
   import {onMount, onDestroy} from 'svelte'
   import Notes from "src/partials/Notes.svelte"
-  import {timedelta, Cursor, getLastSync} from 'src/util/misc'
+  import {timedelta, Cursor} from 'src/util/misc'
   import {getTagValues} from 'src/util/nostr'
   import relay, {user, network} from 'src/relay'
 
-  let sub
-  let networkUnsub
-
-  const cursor = new Cursor(
-    getLastSync('views/notes/Network'),
-    timedelta(1, 'hours')
-  )
+  let notes, sub, networkUnsub
 
   onMount(() => {
     // We need to re-create the sub when network changes, since this is where
@@ -22,7 +16,11 @@
       sub = await relay.pool.listenForEvents(
         'views/notes/Network',
         [{kinds: [1, 5, 7], authors: $network, since: cursor.since}],
-        when(propEq('kind', 1), relay.loadNoteContext)
+        when(propEq('kind', 1), async e => {
+          await relay.loadNoteContext(e)
+
+          notes.addNewNotes([e])
+        })
       )
     })
   })
@@ -35,28 +33,25 @@
     }
   })
 
-  const loadNotes = async limit => {
-    const notes = await relay.filterEvents({
-      limit,
+  const cursor = new Cursor(timedelta(10, 'minutes'))
+
+  const loadNotes = async () => {
+    const [since, until] = cursor.step()
+
+    await relay.pool.loadEvents(
+      [{kinds: [1, 5, 7], authors: $network, since, until}],
+      when(propEq('kind', 1), relay.loadNoteContext)
+    )
+
+    return relay.filterEvents({
+      since,
+      until,
       kinds: [1],
       authors: $network.concat($user.pubkey),
       muffle: getTagValues($user?.muffle || []),
     })
-
-    if (notes.length <= limit) {
-      const [since, until] = cursor.step()
-
-      relay.pool.loadEvents(
-        [{kinds: [1, 5, 7], authors: $network, since, until}],
-        when(propEq('kind', 1), relay.loadNoteContext)
-      )
-    }
-
-    return relay.annotateChunk(notes.slice(0, limit))
   }
 </script>
 
-<!-- hack to reload notes when our network initiall loads, see onMount -->
-{#key $network.map(n => n[0]).join('')}
-<Notes shouldMuffle loadNotes={loadNotes} />
-{/key}
+<Notes bind:this={notes} shouldMuffle {loadNotes} />
+

src/views/person/Likes.svelte

@@ -0,0 +1,23 @@
+<script>
+  import Notes from "src/partials/Notes.svelte"
+  import {timedelta, Cursor} from 'src/util/misc'
+  import {getTagValues} from 'src/util/nostr'
+  import relay, {user} from 'src/relay'
+
+  export let pubkey
+
+  const cursor = new Cursor(timedelta(1, 'days'))
+
+  const loadNotes = async () => {
+    const [since, until] = cursor.step()
+    const filter = {kinds: [7], authors: [pubkey], since, until}
+    const muffle = getTagValues($user?.muffle || [])
+
+    await relay.pool.loadEvents(filter)
+
+    return relay.filterEvents({...filter, muffle})
+  }
+</script>
+
+<Notes shouldMuffle {loadNotes} />
+

src/views/person/Network.svelte

@@ -0,0 +1,24 @@
+<script>
+  import Notes from "src/partials/Notes.svelte"
+  import {timedelta, Cursor} from 'src/util/misc'
+  import {getTagValues} from 'src/util/nostr'
+  import relay, {user} from 'src/relay'
+
+  export let person
+
+  const cursor = new Cursor(timedelta(1, 'hours'))
+
+  const loadNotes = async () => {
+    const [since, until] = cursor.step()
+    const authors = getTagValues(person.petnames)
+    const filter = {kinds: [1], authors, since, until}
+    const muffle = getTagValues($user?.muffle || [])
+
+    await relay.pool.loadEvents(filter)
+
+    return relay.filterEvents({...filter, muffle})
+  }
+</script>
+
+<Notes shouldMuffle {loadNotes} />
+

src/views/person/Notes.svelte

@@ -0,0 +1,21 @@
+<script>
+  import Notes from "src/partials/Notes.svelte"
+  import {timedelta, Cursor} from 'src/util/misc'
+  import relay from 'src/relay'
+
+  export let pubkey
+
+  const cursor = new Cursor(timedelta(1, 'days'))
+
+  const loadNotes = async () => {
+    const [since, until] = cursor.step()
+    const filter = {kinds: [1], authors: [pubkey], since, until}
+
+    await relay.pool.loadEvents(filter, relay.loadNoteContext)
+
+    return relay.filterEvents(filter)
+  }
+</script>
+
+<Notes shouldMuffle {loadNotes} />
+