Fix scrolling bug, figure out pattern for listen/load

README.md

@@ -40,9 +40,9 @@ Coracle is currently in _alpha_ - expect bugs, slow loading times, and rough edg
 # Current update
 
 - [ ] Re-implement muffle
+  - Don't store muffled events, when muffle changes delete them
 - [ ] Delete old events
 - [ ] Sync account updates to user for e.g. muffle settings
-- [ ] Test nos2x
 - [ ] Make sure login/out, no user usage works
 - [ ] Add a re-sync/clear cache button
 - https://vitejs.dev/guide/features.html#web-workers

src/App.svelte

@@ -12,7 +12,7 @@
   import {hasParent} from 'src/util/html'
   import {timedelta} from 'src/util/misc'
   import {store as toast} from "src/state/toast"
-  import {modal, alerts} from "src/state/app"
+  import {modal, alerts, settings} from "src/state/app"
   import relay, {user, connections} from 'src/relay'
   import Anchor from 'src/partials/Anchor.svelte'
   import NoteDetail from "src/views/NoteDetail.svelte"
@@ -53,6 +53,7 @@
     // Give any animations a moment to finish
     setTimeout(() => {
       const $connections = get(connections)
+      const $settings = get(settings)
 
       localStorage.clear()
 
@@ -60,8 +61,9 @@
       relay.db.events.clear()
       relay.db.tags.clear()
 
-      // Remember the user's relay selection
+      // Remember the user's relay selection and settings
       connections.set($connections)
+      settings.set($settings)
 
       // Do a hard refresh so everything gets totally cleared
       window.location = '/login'

src/relay/index.js

@@ -8,6 +8,7 @@ import {db} from 'src/relay/db'
 import pool from 'src/relay/pool'
 
 // Livequery appears to swallow errors
+
 const lq = f => liveQuery(async () => {
   try {
     return await f()
@@ -31,36 +32,7 @@ export const buildNoteContextFilter = async (note, extra = {}) => {
     return filter
 }
 
-// Context getters attempt to retrieve from the db and fall back to the network
-
-const ensurePerson = async ({pubkey}) => {
-  await pool.syncPersonInfo({...prop(pubkey, get(db.people)), pubkey})
-}
-
-const ensureContext = async events => {
-  const promises = []
-  const people = uniq(pluck('pubkey', events)).map(objOf('pubkey'))
-  const ids = events.flatMap(e => filterTags({tag: "e"}, e).concat(e.id))
-
-  if (people.length > 0) {
-    for (const p of people.map(ensurePerson)) {
-      promises.push(p)
-    }
-  }
-
-  if (ids.length > 0) {
-    promises.push(
-      pool.loadEvents([
-        {kinds: [1, 5, 7], '#e': ids},
-        {kinds: [1, 5], ids},
-      ])
-    )
-  }
-
-  await Promise.all(promises)
-}
-
-// Utils for qurying dexie
+// Utils for querying dexie - these return collections, not arrays
 
 const prefilterEvents = filter => {
   if (filter.ids) {
@@ -78,7 +50,7 @@ const prefilterEvents = filter => {
   return db.events
 }
 
-// Utils for filtering db
+// Utils for filtering db - nothing below should load events from the network
 
 const filterEvents = filter => {
   return prefilterEvents(filter)
@@ -110,22 +82,6 @@ const filterReactions = async (id, filter) => {
   return reactions
 }
 
-const findReaction = async (id, filter) =>
-  first(await filterReactions(id, filter))
-
-const countReactions = async (id, filter) =>
-  (await filterReactions(id, filter)).length
-
-const getOrLoadNote = async (id, {showEntire = false} = {}) => {
-  const note = await db.events.get(id)
-
-  if (!note) {
-    return first(await pool.loadEvents({kinds: [1], ids: [id]}))
-  }
-
-  return note
-}
-
 const findNote = async (id, {showEntire = false} = {}) => {
   const note = await db.events.get(id)
 
@@ -251,6 +207,19 @@ const unfollow = async pubkey => {
   db.network.update($network => $network.concat(pubkey))
 }
 
+// Methods that wil attempt to load from the database and fall back to the network.
+// This is intended only for bootstrapping listeners
+
+const getOrLoadNote = async (id, {showEntire = false} = {}) => {
+  const note = await db.events.get(id)
+
+  if (!note) {
+    return first(await pool.loadEvents({kinds: [1], ids: [id]}))
+  }
+
+  return note
+}
+
 // Initialization
 
 db.user.subscribe($user => {
@@ -277,14 +246,15 @@ db.connections.subscribe($connections => {
   }
 })
 
+// Export stores on their own for convenience
+
 export const user = db.user
 export const people = db.people
 export const network = db.network
 export const connections = db.connections
 
 export default {
-  db, pool, lq, buildNoteContextFilter, ensurePerson, ensureContext, filterEvents,
-  filterReactions, getOrLoadNote,
-  countReactions, findReaction, filterReplies, findNote, annotateChunk, renderNote,
-  filterAlerts, login, addRelay, removeRelay, follow, unfollow,
+  db, pool, lq, buildNoteContextFilter, filterEvents, getOrLoadNote,
+  filterReplies, findNote, annotateChunk, renderNote, filterAlerts,
+  login, addRelay, removeRelay, follow, unfollow,
 }

src/relay/pool.js

@@ -1,8 +1,7 @@
-import {uniqBy, prop, uniq} from 'ramda'
+import {uniqBy, find, propEq, prop, uniq} from 'ramda'
 import {get} from 'svelte/store'
 import {relayPool, getPublicKey} from 'nostr-tools'
-import {noop, range} from 'hurdak/lib/hurdak'
-import {now, timedelta, randomChoice, getLocalJson, setLocalJson} from "src/util/misc"
+import {noop, range, sleep} from 'hurdak/lib/hurdak'
 import {getTagValues, filterTags} from "src/util/nostr"
 import {db} from 'src/relay/db'
 
@@ -14,9 +13,15 @@ const pool = relayPool()
 class Channel {
   constructor(name) {
     this.name = name
-    this.p = Promise.resolve()
+    this.status = 'idle'
   }
-  async sub(filter, onEvent, onEose = noop, timeout = 30000) {
+  claim() {
+    this.status = 'busy'
+  }
+  release() {
+    this.status = 'idle'
+  }
+  sub(filter, onEvent, onEose = noop, opts = {}) {
     // If we don't have any relays, we'll wait forever for an eose, but
     // we already know we're done. Use a timeout since callers are
     // expecting this to be async and we run into errors otherwise.
@@ -26,17 +31,6 @@ class Channel {
       return {unsub: noop}
     }
 
-    // Grab our spot in the queue, save resolve for later
-    let resolve
-    let p = this.p
-    this.p = new Promise(r => {
-      resolve = r
-    })
-
-    // Make sure callers have to wait for the previous sub to be done
-    // before they can get a new one.
-    await p
-
     // Start our subscription, wait for only one relay to eose before
     // calling it done. We were waiting for all before, but that made
     // the slowest relay a bottleneck
@@ -45,22 +39,22 @@ class Channel {
     const done = () => {
       sub.unsub()
 
-      resolve()
+      this.release()
     }
 
     // If the relay takes to long, just give up
-    if (timeout) {
-      setTimeout(done, 1000)
+    if (opts.timeout) {
+      setTimeout(done, opts.timeout)
     }
 
     return {unsub: done}
   }
-  all(filter) {
+  all(filter, opts = {}) {
     /* eslint no-async-promise-executor: 0 */
     return new Promise(async resolve => {
       const result = []
 
-      const sub = await this.sub(
+      const sub = this.sub(
         filter,
         e => result.push(e),
         r => {
@@ -68,6 +62,7 @@ class Channel {
 
           resolve(uniqBy(prop('id'), result))
         },
+        {timeout: 30000, ...opts},
       )
     })
   }
@@ -75,8 +70,25 @@ class Channel {
 
 export const channels = range(0, 10).map(i => new Channel(i.toString()))
 
-const req = (...args) => randomChoice(channels).all(...args)
-const sub = (...args) => randomChoice(channels).sub(...args)
+const getChannel = async () => {
+  /*eslint no-constant-condition: 0*/
+
+  // Find a channel that isn't busy, or wait for one to become available
+  while (true) {
+    const channel = find(propEq('status', 'idle'), channels)
+
+    if (channel) {
+      channel.claim()
+
+      return channel
+    }
+
+    await sleep(300)
+  }
+}
+
+const req = async (...args) => (await getChannel()).all(...args)
+const sub = async (...args) => (await getChannel()).sub(...args)
 
 const getPubkey = () => {
   return pool._pubkey || getPublicKey(pool._privkey)
@@ -122,14 +134,12 @@ const loadEvents = async filter => {
   return events
 }
 
-const subs = {}
-
 const listenForEvents = async (key, filter, onEvent) => {
-  if (subs[key]) {
-    subs[key].unsub()
+  if (listenForEvents.subs[key]) {
+    listenForEvents.subs[key].unsub()
   }
 
-  subs[key] = await sub(filter, e => {
+  listenForEvents.subs[key] = await sub(filter, e => {
     db.events.process(e)
 
     if (onEvent) {
@@ -138,8 +148,14 @@ const listenForEvents = async (key, filter, onEvent) => {
   })
 }
 
-const loadPeople = pubkeys => {
-  return pubkeys.length ? loadEvents({kinds: [0, 3, 12165], authors: pubkeys}) : []
+listenForEvents.subs = {}
+
+const loadPeople = (pubkeys, opts = {}) => {
+  if (pubkeys.length === 0) {
+    return []
+  }
+
+  return loadEvents({kinds: [0, 3, 12165], authors: pubkeys}, opts)
 }
 
 const syncNetwork = async () => {
@@ -148,7 +164,7 @@ const syncNetwork = async () => {
   let pubkeys = []
   if ($user) {
     // Get this user's profile to start with
-    await loadPeople([$user.pubkey])
+    await loadPeople([$user.pubkey], {timeout: null})
 
     // Get our refreshed person
     const people = get(db.people)

src/routes/Login.svelte

@@ -7,7 +7,7 @@
   import Anchor from "src/partials/Anchor.svelte"
   import Input from "src/partials/Input.svelte"
   import toast from "src/state/toast"
-  import relay from 'src/relay'
+  import relay, {connections} from 'src/relay'
 
   let privkey = ''
   let hasExtension = false
@@ -30,10 +30,14 @@
     toast.show("info", "Your private key has been re-generated.")
   }
 
-  const logIn = ({privkey, pubkey}) => {
+  const logIn = async ({privkey, pubkey}) => {
     relay.login({privkey, pubkey})
 
-    navigate('/relays')
+    if ($connections.length === 0) {
+      navigate('/relays')
+    } else {
+      navigate('/notes/network')
+    }
   }
 
   const logInWithExtension = async () => {

src/routes/Notes.svelte

@@ -4,69 +4,14 @@
   import {findReply} from 'src/util/nostr'
   import Anchor from "src/partials/Anchor.svelte"
   import Tabs from "src/partials/Tabs.svelte"
-  import Notes from "src/views/Notes.svelte"
+  import Network from "src/views/notes/Network.svelte"
+  import Global from "src/views/notes/Global.svelte"
   import {now, timedelta} from 'src/util/misc'
-  import relay, {network, connections} from 'src/relay'
+  import relay, {connections} from 'src/relay'
 
   export let activeTab
 
-  let sub
-  let delta = timedelta(1, 'minutes')
-  let since = now() - delta
-
-  onMount(async () => {
-    sub = await subscribe(now())
-  })
-
-  onDestroy(() => {
-    if (sub) {
-      sub.unsub()
-    }
-  })
-
   const setActiveTab = tab => navigate(`/notes/${tab}`)
-
-  const subscribe = until =>
-    relay.pool.listenForEvents(
-      'routes/Notes',
-      [{kinds: [1, 5, 7], since, until}],
-      async e => {
-        if (e.kind === 1) {
-          const filter = await relay.buildNoteContextFilter(e, {since})
-
-          await relay.pool.loadEvents(filter)
-        }
-
-        if (e.kind === 7) {
-          const replyId = findReply(e)
-
-          if (replyId && !await relay.db.events.get(replyId)) {
-            await relay.pool.loadEvents({kinds: [1], ids: [replyId]})
-          }
-
-        }
-      }
-    )
-
-  const loadNetworkNotes = async limit => {
-    const filter = {kinds: [1], authors: $network}
-    const notes = await relay.filterEvents(filter).reverse().sortBy('created_at')
-
-    return relay.annotateChunk(notes.slice(0, limit))
-  }
-
-  const loadGlobalNotes = async limit => {
-    const filter = {kinds: [1], since}
-    const notes = await relay.filterEvents(filter).reverse().sortBy('created_at')
-
-    if (notes.length < limit) {
-      since -= delta
-
-      sub = await subscribe(since + delta)
-    }
-
-    return relay.annotateChunk(notes.slice(0, limit))
-  }
 </script>
 
 {#if $connections.length === 0}
@@ -82,7 +27,7 @@
 {#if activeTab === 'network'}
 <Network />
 {:else}
-<Notes shouldMuffle loadNotes={loadGlobalNotes} />
+<Global />
 {/if}
 <div class="fixed bottom-0 right-0 p-8">
   <a

src/util/misc.js

@@ -51,23 +51,23 @@ export const formatTimestamp = ts => {
 export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 export const createScroller = loadMore => {
-  /* eslint no-constant-condition: 0 */
-
   let done = false
-
+  let didLoad = false
   const check = async () => {
     // While we have empty space, fill it
     const {scrollY, innerHeight} = window
     const {scrollHeight} = document.body
+    const shouldLoad = scrollY + innerHeight + 300 > scrollHeight
 
-    if (scrollY + innerHeight + 2000 > scrollHeight) {
+    // Only trigger loading the first time we reach the threshhold
+    if (shouldLoad && !didLoad) {
       await loadMore()
     }
 
-    // This is a gross hack, basically, keep loading if the user doesn't scroll again,
-    // but wait a long time because otherwise we'll send off multiple concurrent requests
-    // that will clog up our channels and stall the app.
-    await sleep(30000)
+    didLoad = shouldLoad
+
+    // No need to check all that often
+    await sleep(300)
 
     if (!done) {
       requestAnimationFrame(check)
@@ -91,3 +91,16 @@ export const getLastSync = (k, fallback) => {
 
   return lastSync
 }
+
+export class Cursor {
+  constructor(since, delta) {
+    this.since = since || now() - delta,
+    this.delta = delta
+  }
+  step() {
+    const until = this.since
+    this.since -= this.delta
+
+    return [this.since, until]
+  }
+}

src/views/Network.svelte

@@ -1,63 +0,0 @@
-<script>
-  import {onMount, onDestroy} from 'svelte'
-  import {navigate} from 'svelte-routing'
-  import {findReply} from 'src/util/nostr'
-  import Anchor from "src/partials/Anchor.svelte"
-  import Tabs from "src/partials/Tabs.svelte"
-  import Notes from "src/views/Notes.svelte"
-  import {now, timedelta} from 'src/util/misc'
-  import relay, {network, connections} from 'src/relay'
-
-  let sub
-  let since = getLastSync('views/Network')
-
-  onMount(async () => {
-    sub = await subscribe(now())
-  })
-
-  onDestroy(() => {
-    if (sub) {
-      sub.unsub()
-    }
-  })
-
-  const setActiveTab = tab => navigate(`/notes/${tab}`)
-
-  const subscribe = until =>
-    relay.pool.listenForEvents(
-      'views/Network',
-      [{kinds: [1, 5, 7], authors: $network, since, until}],
-      async e => {
-        if (e.kind === 1) {
-          const filter = await relay.buildNoteContextFilter(e, {since})
-
-          await relay.pool.loadEvents(filter)
-        }
-
-        if (e.kind === 7) {
-          const replyId = findReply(e)
-
-          if (replyId && !await relay.db.events.get(replyId)) {
-            await relay.pool.loadEvents({kinds: [1], ids: [replyId]})
-          }
-
-        }
-      }
-    )
-
-  const loadNotes = async limit => {
-    const filter = {kinds: [1], authors: $network}
-    const notes = await relay.filterEvents(filter).reverse().sortBy('created_at')
-
-    if (notes.length < limit) {
-      until = notes.reduce((t, n) => Math.min(n.created_at), since)
-      since = until - timedelta(1, 'hours')
-
-      sub = await subscribe(since)
-    }
-
-    return relay.annotateChunk(notes.slice(0, limit))
-  }
-</script>
-
-<Notes shouldMuffle loadNotes={loadNotes} />

src/views/Note.svelte

@@ -50,10 +50,30 @@
     } else {
       navigate('/login')
     }
+
+    if (content === '+') {
+      like = true
+      likes += 1
+    }
+
+    if (content === '-') {
+      flag = true
+      flags += 1
+    }
   }
 
   const deleteReaction = e => {
     dispatch('event/delete', [e.id])
+
+    if (e.content === '+') {
+      like = false
+      likes -= 1
+    }
+
+    if (e.content === '-') {
+      flag = false
+      flags -= 1
+    }
   }
 
   const startReply = () => {

src/views/NoteDetail.svelte

@@ -8,7 +8,6 @@
   export let note
 
   let observable, sub
-  let since = getLastSync(['NoteDetail', note.id])
 
   onMount(async () => {
     note = await relay.getOrLoadNote(note.id)
@@ -16,7 +15,7 @@
     if (note) {
       sub = await relay.pool.listenForEvents(
         'routes/NoteDetail',
-        await relay.buildNoteContextFilter(note, {since})
+        await relay.buildNoteContextFilter(note)
       )
     }
   })

src/views/notes/Global.svelte

@@ -0,0 +1,59 @@
+<script>
+  import {onMount, onDestroy} from 'svelte'
+  import {findReply} from 'src/util/nostr'
+  import Notes from "src/views/Notes.svelte"
+  import {now, timedelta, Cursor, getLastSync} from 'src/util/misc'
+  import relay from 'src/relay'
+
+  let sub
+
+  const cursor = new Cursor(
+    getLastSync('views/notes/Global'),
+    timedelta(1, 'minutes')
+  )
+
+  const onEvent = async e => {
+    if (e.kind === 1) {
+      const filter = await relay.buildNoteContextFilter(e)
+
+      await relay.pool.loadEvents(filter)
+    }
+
+    if (e.kind === 7) {
+      const replyId = findReply(e)
+
+      if (replyId && !await relay.db.events.get(replyId)) {
+        await relay.pool.loadEvents({kinds: [1], ids: [replyId]})
+      }
+
+    }
+  }
+
+  onMount(async () => {
+    sub = await relay.pool.listenForEvents(
+      'views/notes/Global',
+      [{kinds: [1, 5, 7], since: cursor.since}],
+      onEvent
+    )
+  })
+
+  onDestroy(() => {
+    if (sub) {
+      sub.unsub()
+    }
+  })
+
+  const loadNotes = async limit => {
+    const notes = await relay.filterEvents({kinds: [1]}).reverse().sortBy('created_at')
+
+    if (notes.length < limit) {
+      const [since, until] = cursor.step()
+
+      relay.pool.loadEvents([{kinds: [1, 5, 7], since, until}], onEvent)
+    }
+
+    return relay.annotateChunk(notes.slice(0, limit))
+  }
+</script>
+
+<Notes shouldMuffle loadNotes={loadNotes} />

src/views/notes/Network.svelte

@@ -0,0 +1,63 @@
+<script>
+  import {onMount, onDestroy} from 'svelte'
+  import {findReply} from 'src/util/nostr'
+  import Notes from "src/views/Notes.svelte"
+  import {now, timedelta, Cursor, getLastSync} from 'src/util/misc'
+  import relay, {network} from 'src/relay'
+
+  let sub
+
+  const cursor = new Cursor(
+    getLastSync('views/notes/Network'),
+    timedelta(1, 'hours')
+  )
+
+  const onEvent = async e => {
+    if (e.kind === 1) {
+      const filter = await relay.buildNoteContextFilter(e)
+
+      await relay.pool.loadEvents(filter)
+    }
+
+    if (e.kind === 7) {
+      const replyId = findReply(e)
+
+      if (replyId && !await relay.db.events.get(replyId)) {
+        await relay.pool.loadEvents({kinds: [1], ids: [replyId]})
+      }
+
+    }
+  }
+
+  onMount(async () => {
+    sub = await relay.pool.listenForEvents(
+      'views/notes/Network',
+      [{kinds: [1, 5, 7], authors: $network, since: cursor.since}],
+      onEvent
+    )
+  })
+
+  onDestroy(() => {
+    if (sub) {
+      sub.unsub()
+    }
+  })
+
+  const loadNotes = async limit => {
+    const filter = {kinds: [1], authors: $network}
+    const notes = await relay.filterEvents(filter).reverse().sortBy('created_at')
+
+    if (notes.length < limit) {
+      const [since, until] = cursor.step()
+
+      relay.pool.loadEvents(
+        [{kinds: [1, 5, 7], authors: $network, since, until}],
+        onEvent
+      )
+    }
+
+    return relay.annotateChunk(notes.slice(0, limit))
+  }
+</script>
+
+<Notes shouldMuffle loadNotes={loadNotes} />