Remove event cache, livequery

README.md

@@ -34,6 +34,9 @@ If you like Coracle and want to support its development, you can donate sats via
   - https://github.com/nbd-wtf/nostr-tools/blob/master/nip26.ts
 - [ ] Add relay selector when publishing a note
 - [ ] Add keyword mutes
+- [ ] Add no-relay gossip
+  - Capture certain events in a local db
+  - File import/export from db, NFC transfer
 
 # Bugs
 
@@ -69,6 +72,7 @@ If you like Coracle and want to support its development, you can donate sats via
   - Load feeds by setting a listener since now, paginating using limit (of 2 maybe), and awaiting context for each page. Listener appends to "newNotes", cursor appends to "notes", load more moves new notes into notes. Use the note originally loaded as the anchor, don't re-process the whole list
 - [ ] Close connections that haven't been used in a while
 - [ ] Load feeds from network rather than user relays? This could make global feed more useful: global for _my_ relays
+- [ ] Release to android with https://svelte-native.technology/docs
 
 ## 0.2.6
 

src/agent/data.js

@@ -1,11 +1,7 @@
 import Dexie from 'dexie'
-import {matchFilter} from 'nostr-tools'
-import {get} from 'svelte/store'
-import {groupBy, prop, flatten, pick} from 'ramda'
 import {ensurePlural, switcherFn} from 'hurdak/lib/hurdak'
 import {synced, now, timedelta} from 'src/util/misc'
-import {Tags, personKinds, findReply, findRoot} from 'src/util/nostr'
-import keys from 'src/agent/keys'
+import {personKinds} from 'src/util/nostr'
 
 export const db = new Dexie('agent/data/db')
 
@@ -31,50 +27,9 @@ export const getPerson = (pubkey, fallback = false) =>
 // Hooks
 
 export const processEvents = async events => {
-  // Only persist ones we care about, the rest can be ephemeral and used to update people etc
-  const pubkey = get(keys.pubkey)
-  const eventsByKind = groupBy(prop('kind'), ensurePlural(events))
-  const notesAndReactions = flatten(Object.values(pick([1, 7], eventsByKind)))
-    .map(e => ({...e, root: findRoot(e), reply: findReply(e), loaded_at: now()}))
-  const alerts = notesAndReactions.filter(e => matchFilter({kinds: [1, 7], '#p': [pubkey]}, e))
-  const profileUpdates = flatten(Object.values(pick(personKinds, eventsByKind)))
-  const deletions = eventsByKind[5] || []
+  const profileUpdates = ensurePlural(events)
+    .filter(e => personKinds.includes(e.kind))
 
-  // Persist notes and reactions
-  if (notesAndReactions.length > 0) {
-    db.events.bulkPut(notesAndReactions)
-
-    db.tags.bulkPut(
-      notesAndReactions
-        .flatMap(e =>
-          e.tags.map(
-            tag => ({
-              id: [e.id, ...tag.slice(0, 2)].join(':'),
-              event: e.id,
-              type: tag[0],
-              value: tag[1],
-              relay: tag[2],
-              mark: tag[3],
-              loaded_at: now(),
-            })
-          )
-        )
-    )
-  }
-
-  if (alerts.length > 0) {
-    db.alerts.bulkPut(alerts)
-  }
-
-  // Delete stuff that needs to be deleted
-  if (deletions.length > 0) {
-    const eventIds = Tags.from(deletions).type("e").values().all()
-
-    db.events.where('id').anyOf(eventIds).delete()
-    db.tags.where('event').anyOf(eventIds).delete()
-  }
-
-  // Update our people
   people.update($people => {
     for (const event of profileUpdates) {
       const {pubkey, kind, content, tags} = event

src/agent/defaults.js

@@ -8,7 +8,6 @@ export default {
     ["p", "85080d3bad70ccdcd7f74c29a44f55bb85cbcd3dd0cbb957da1d215bdb931204", "preston",      "wss://relay.damus.io"],
   ],
   relays: [
-    'wss://nostr.rocks',
     'wss://relay.damus.io',
     'wss://nostr.zebedee.cloud',
     'wss://nostr-pub.wellorder.net',

src/agent/index.js

@@ -1,4 +1,6 @@
+import {last} from 'ramda'
 import {derived, get} from 'svelte/store'
+import {getTagValues} from 'src/util/nostr'
 import pool from 'src/agent/pool'
 import keys from 'src/agent/keys'
 import defaults from 'src/agent/defaults'
@@ -19,6 +21,16 @@ export const user = derived(
   }
 )
 
+export const getMuffle = () => {
+  const $user = get(user)
+
+  if (!$user?.muffle) {
+    return []
+  }
+
+  return getTagValues($user.muffle.filter(t => Math.random() < last(t)))
+}
+
 export const getRelays = pubkey => {
   let relays = getPerson(pubkey)?.relays
 

src/app/alerts.js

@@ -1,4 +1,3 @@
-import {get} from 'svelte/store'
 import {sortBy, pluck} from 'ramda'
 import {first} from 'hurdak/lib/hurdak'
 import {synced, batch, now, timedelta} from 'src/util/misc'
@@ -9,7 +8,6 @@ import loaders from 'src/app/loaders'
 let listener
 
 const start = now() - timedelta(30, 'days')
-
 const since = synced("app/alerts/since", start)
 const latest = synced("app/alerts/latest", start)
 
@@ -18,8 +16,6 @@ const listen = async (relays, pubkey) => {
     listener.unsub()
   }
 
-  console.log(get(since))
-
   listener = await _listen(
     relays,
     [{kinds: [1, 7], '#p': [pubkey], since: start}],

src/app/loaders.js

@@ -1,17 +1,35 @@
 import {uniq, pluck, groupBy, identity} from 'ramda'
 import {ensurePlural, createMap} from 'hurdak/lib/hurdak'
-import {findReply, personKinds} from 'src/util/nostr'
+import {findReply, personKinds, Tags, getTagValues} from 'src/util/nostr'
+import {now, timedelta} from 'src/util/misc'
 import {load, db, getPerson} from 'src/agent'
 import defaults from 'src/agent/defaults'
 
-const loadPeople = (relays, pubkeys, {kinds = personKinds, ...opts} = {}) =>
-  pubkeys.length > 0 ? load(relays, {kinds, authors: pubkeys}, opts) : []
+const getStalePubkeys = pubkeys => {
+  // If we're not reloading, only get pubkeys we don't already know about
+  return uniq(pubkeys).filter(pubkey => {
+    const p = getPerson(pubkey)
+
+    return !p || p.updated_at < now() - timedelta(1, 'days')
+  })
+}
+
+const loadPeople = (relays, pubkeys, {kinds = personKinds, force = false, ...opts} = {}) => {
+  pubkeys = uniq(pubkeys)
+
+  // If we're not reloading, only get pubkeys we don't already know about
+  if (!force) {
+    pubkeys = getStalePubkeys(pubkeys)
+  }
+
+  return pubkeys.length > 0 ? load(relays, {kinds, authors: pubkeys}, opts) : []
+}
 
 const loadNetwork = async (relays, pubkey) => {
   // Get this user's profile to start with. This may update what relays
   // are available, so don't assign relays to a variable here.
-  let events = pubkey ? await loadPeople(relays, [pubkey]) : []
-  let petnames = events.filter(e => e.kind === 3).flatMap(e => e.tags.filter(t => t[0] === "p"))
+  let events = pubkey ? await loadPeople(relays, [pubkey], {force: true}) : []
+  let petnames = Tags.from(events.filter(e => e.kind === 3)).type("p").all()
 
   // Default to some cool guys we know
   if (petnames.length === 0) {
@@ -22,10 +40,42 @@ const loadNetwork = async (relays, pubkey) => {
   // relays to load our user's second-order follows in order to bootstrap our social graph
   await Promise.all(
     Object.entries(groupBy(t => t[2], petnames))
-      .map(([relay, petnames]) => loadPeople([relay], petnames.map(t => t[1])))
+      .map(([relay, petnames]) => loadPeople([relay], getTagValues(petnames)))
   )
 }
 
+const loadContext = async (relays, notes, {loadParents = true} = {}) => {
+  notes = ensurePlural(notes)
+
+  if (notes.length === 0) {
+    return notes
+  }
+
+  const authors = getStalePubkeys(pluck('pubkey', notes))
+  const parentTags = loadParents ? uniq(notes.map(findReply).filter(identity)) : []
+  const filter = [{kinds: [1, 7], '#e': pluck('id', notes)}]
+
+  if (authors.length > 0) {
+    filter.push({kinds: personKinds, authors})
+  }
+
+  if (parentTags.length > 0) {
+    filter.push({kinds: [1], ids: getTagValues(parentTags)})
+    relays = uniq(relays.concat(Tags.wrap(parentTags).relays()))
+  }
+
+  const events = await load(relays, filter)
+
+  if (parentTags.length === 0) {
+    return events
+  }
+
+  const eventsById = createMap('id', events)
+  const parents = getTagValues(parentTags).map(id => eventsById[id]).filter(identity)
+
+  return events.concat(await loadContext(relays, parents, {loadParents: false}))
+}
+
 const loadNotesContext = async (relays, notes, {loadParents = false} = {}) => {
   notes = ensurePlural(notes)
 
@@ -72,4 +122,4 @@ const getOrLoadNote = async (relays, id) => {
   return note
 }
 
-export default {getOrLoadNote, loadNotesContext, loadNetwork, loadPeople, personKinds}
+export default {getOrLoadNote, loadNotesContext, loadNetwork, loadPeople, personKinds, loadContext}

src/app/query.js

@@ -127,7 +127,7 @@ const annotateChunk = async chunk => {
   return sortBy(e => -e.created_at, notes.filter(propEq('kind', 1)))
 }
 
-const renderNote = async (note, {showEntire = false}) => {
+const renderNote = (note, {showEntire = false}) => {
   const shouldEllipsize = note.content.length > 500 && !showEntire
   const $people = get(people)
   const peopleByPubkey = createMap(
@@ -161,4 +161,45 @@ const renderNote = async (note, {showEntire = false}) => {
   return content
 }
 
-export default {filterEvents, filterReplies, filterReactions, annotateChunk, renderNote, findNote}
+const annotate = (note, context, {showEntire = false, depth = 1} = {}) => {
+  const reactions = context.filter(e => e.kind === 7 && findReply(e) === note.id)
+  const replies = context.filter(e => e.kind === 1 && findReply(e) === note.id)
+
+  return {
+    ...note, reactions,
+    html: renderNote(note, {showEntire}),
+    person: getPerson(note.pubkey),
+    repliesCount: replies.length,
+    replies: depth === 0
+      ? []
+      : sortBy(e => e.created_at, replies)
+          .slice(showEntire ? 0 : -3)
+          .map(r => annotate(r, context, {depth: depth - 1}))
+  }
+}
+
+const threadify = (events, context, {muffle = []} = {}) => {
+  const contextById = createMap('id', context)
+
+  // Show parents when possible. For reactions, if there's no parent,
+  // throw it away. Sort by created date descending
+  const notes = sortBy(
+    e => -e.created_at,
+    events
+      .map(e => contextById[findReply(e)] || (e.kind === 1 ? e : null))
+      .filter(e => e && !muffle.includes(e.pubkey))
+  )
+
+  // Annotate our feed with parents, reactions, replies
+  return notes.map(note => {
+    let parent = contextById[findReply(note)]
+
+    if (parent) {
+      parent = annotate(parent, context)
+    }
+
+    return annotate({...note, parent}, context)
+  })
+}
+
+export default {filterEvents, filterReplies, filterReactions, annotateChunk, renderNote, findNote, threadify, annotate}

src/partials/Note.svelte

@@ -18,7 +18,6 @@
   import cmd from 'src/app/cmd'
 
   export let note
-  export let until = Infinity
   export let depth = 0
   export let anchorId = null
   export let showParent = true
@@ -46,7 +45,7 @@
   }
 
   const goToParent = async () => {
-    modal.set({note: {id: findReply(note)}})
+    modal.set({note: {id: findReply(note)[1]}})
   }
 
   const react = async content => {
@@ -117,7 +116,7 @@
   <div class="ml-6 flex flex-col gap-2">
     {#if findReply(note) && showParent}
       <small class="text-light">
-        Reply to <Anchor on:click={goToParent}>{findReply(note).slice(0, 8)}</Anchor>
+        Reply to <Anchor on:click={goToParent}>{findReply(note)[1].slice(0, 8)}</Anchor>
       </small>
     {/if}
     {#if flag}
@@ -183,9 +182,7 @@
   </div>
   {/if}
   {#each note.replies as r (r.id)}
-  {#if r.created_at <= until}
   <svelte:self showParent={false} note={r} depth={depth - 1} {invertColors} {anchorId} />
-  {/if}
   {/each}
 </div>
 {/if}

src/partials/Notes.svelte

@@ -1,55 +1,58 @@
 <script>
-  import {liveQuery} from 'dexie'
-  import {sortBy, pluck, reject} from 'ramda'
   import {onMount} from 'svelte'
   import {slide} from 'svelte/transition'
   import {quantify} from 'hurdak/lib/hurdak'
-  import {createScroller, sleep, now} from 'src/util/misc'
-  import {findReply} from 'src/util/nostr'
+  import {createScroller} from 'src/util/misc'
   import Spinner from 'src/partials/Spinner.svelte'
   import Note from "src/partials/Note.svelte"
-  import query from 'src/app/query'
 
   export let loadNotes
-  export let queryNotes
-
-  const notes = liveQuery(async () => {
-    // Hacky way to wait for the loader to adjust the cursor so we have a nonzero duration
-    await sleep(100)
-
-    return sortBy(
-      e => -pluck('created_at', e.replies).concat(e.created_at).reduce((a, b) => Math.max(a, b)),
-      await query.annotateChunk(await queryNotes())
-    )
-  })
+  export let listenForNotes
 
   let depth = 2
-  let until = now()
+  let notes = []
   let newNotes = []
-  let newNotesLength = 0
 
-  $: newNotes = ($notes || []).filter(e => e.created_at > until)
-  $: newNotesLength = reject(findReply, newNotes).length
-  $: visibleNotes = ($notes || []).filter(e => e.created_at <= until)
+  // Make max notes sort of random so people don't know they're missing out
+  let maxNotes = 200 + Math.round(Math.random() * 100)
+
+  const showNewNotes = () => {
+    // Drop notes at the end if there are a lot
+    notes = newNotes.concat(notes).slice(0, maxNotes)
+    newNotes = []
+  }
 
   onMount(() => {
-    const scroller = createScroller(loadNotes)
+    const sub = listenForNotes(events => {
+      // Slice new notes so if someone leaves the tab open for a long time we don't get a bazillion
+      newNotes = events.concat(newNotes).slice(0, maxNotes)
+    })
 
-    return scroller.stop
+    const scroller = createScroller(async () => {
+      // Drop notes at the top if there are a lot
+      notes = notes.concat(await loadNotes()).slice(-maxNotes)
+    })
+
+    return async () => {
+      const {unsub} = await sub
+
+      scroller.stop()
+      unsub()
+    }
   })
 </script>
 
 <ul class="py-4 flex flex-col gap-2 max-w-xl m-auto">
-  {#if newNotesLength > 0}
+  {#if newNotes.length > 0}
   <div
     in:slide
     class="mb-2 cursor-pointer text-center underline text-light"
-    on:click={() => { until = now() }}>
-    Load {quantify(newNotesLength, 'new note')}
+    on:click={showNewNotes}>
+    Load {quantify(newNotes.length, 'new note')}
   </div>
   {/if}
-  {#each visibleNotes as note (note.id)}
-    <li><Note {until} {note} {depth} /></li>
+  {#each notes as note (note.id)}
+    <li><Note {note} {depth} /></li>
   {/each}
 </ul>
 

src/util/nostr.js

@@ -11,6 +11,9 @@ export class Tags {
   static from(events) {
     return new Tags(ensurePlural(events).flatMap(prop('tags')))
   }
+  static wrap(tags) {
+    return new Tags(tags)
+  }
   all() {
     return this.tags
   }
@@ -20,6 +23,11 @@ export class Tags {
   last() {
     return last(this.tags)
   }
+  relays() {
+    return this.tags
+      .map(t => t[3])
+      .filter(url => typeof url === 'string' && url.startsWith('ws'))
+  }
   values() {
     this.tags = this.tags.map(t => t[0])
 

src/views/notes/Global.svelte

@@ -1,44 +1,28 @@
 <script>
-  import {when, propEq} from 'ramda'
-  import {onMount} from 'svelte'
   import Notes from "src/partials/Notes.svelte"
-  import {timedelta, Cursor} from 'src/util/misc'
-  import {getTagValues} from 'src/util/nostr'
-  import {listen, user, load, getRelays} from 'src/agent'
+  import {timedelta, Cursor, now, batch} from 'src/util/misc'
+  import {getRelays, getMuffle, listen, load} from 'src/agent'
   import loaders from 'src/app/loaders'
   import query from 'src/app/query'
 
-  onMount(() => {
-    const sub = listen(
-      getRelays(),
-      [{kinds: [1, 5, 7], since: cursor.since}],
-      when(propEq('kind', 1), e => {
-        loaders.loadNotesContext(getRelays(), e)
-      })
-    )
-
-    return () => {
-      sub.then(s => s.unsub())
-    }
-  })
-
+  const relays = getRelays()
+  const filter = {kinds: [1, 5, 7]}
   const cursor = new Cursor(timedelta(1, 'minutes'))
 
+  const listenForNotes = onNotes =>
+    listen(relays, {...filter, since: now()}, batch(300, async notes => {
+      const context = await loaders.loadContext(relays, notes)
+
+      onNotes(query.threadify(notes, context, {muffle: getMuffle()}))
+    }))
+
   const loadNotes = async () => {
     const [since, until] = cursor.step()
-    const filter = {kinds: [1], since, until}
-    const notes = await load(getRelays(), filter)
+    const notes = await load(relays, {...filter, since, until})
+    const context = await loaders.loadContext(relays, notes)
 
-    await loaders.loadNotesContext(getRelays(), notes, {loadParents: true})
-  }
-
-  const queryNotes = () => {
-    return query.filterEvents({
-      kinds: [1],
-      since: cursor.since,
-      muffle: getTagValues($user?.muffle || []),
-    })
+    return query.threadify(notes, context, {muffle: getMuffle()})
   }
 </script>
 
-<Notes shouldMuffle {loadNotes} {queryNotes} />
+<Notes {listenForNotes} {loadNotes} />

src/views/notes/Network.svelte

@@ -1,10 +1,8 @@
 <script>
-  import {when, identity, nth, propEq} from 'ramda'
-  import {onMount} from 'svelte'
   import Notes from "src/partials/Notes.svelte"
-  import {timedelta, shuffle, Cursor} from 'src/util/misc'
+  import {now, timedelta, shuffle, batch, Cursor} from 'src/util/misc'
   import {getTagValues} from 'src/util/nostr'
-  import {user, getRelays, getPerson, listen, load} from 'src/agent'
+  import {user, getRelays, getMuffle, getPerson, listen, load} from 'src/agent'
   import defaults from 'src/agent/defaults'
   import loaders from 'src/app/loaders'
   import query from 'src/app/query'
@@ -17,43 +15,28 @@
 
   // Get first- and second-order follows. shuffle and slice network so we're not
   // sending too many pubkeys. This will also result in some variety.
+  const relays = getRelays()
   const follows = getFollows($user?.pubkey)
   const network = shuffle(follows.flatMap(getFollows)).slice(0, 50)
   const authors = follows.concat(network)
-
-  onMount(() => {
-    const sub = listen(
-      getRelays(),
-      [{kinds: [1, 5, 7], authors, since: cursor.since}],
-      when(propEq('kind', 1), e => {
-        loaders.loadNotesContext(getRelays(), e)
-      })
-    )
-
-    return () => {
-      sub.then(s => s.unsub())
-    }
-  })
-
+  const filter = {kinds: [1, 7], authors}
   const cursor = new Cursor(timedelta(20, 'minutes'))
 
+  const listenForNotes = onNotes =>
+    listen(relays, {...filter, since: now()}, batch(300, async notes => {
+      const context = await loaders.loadContext(relays, notes)
+
+      onNotes(query.threadify(notes, context, {muffle: getMuffle()}))
+    }))
+
   const loadNotes = async () => {
     const [since, until] = cursor.step()
-    const filter = {kinds: [1, 7], authors, since, until}
-    const notes = await load(getRelays(), filter)
+    const notes = await load(relays, {...filter, since, until})
+    const context = await loaders.loadContext(relays, notes)
 
-    await loaders.loadNotesContext(getRelays(), notes, {loadParents: true})
-  }
-
-  const queryNotes = () => {
-    return query.filterEvents({
-      kinds: [1],
-      since: cursor.since,
-      authors: authors.concat($user?.pubkey).filter(identity),
-      muffle: getTagValues($user?.muffle || []),
-    })
+    return query.threadify(notes, context, {muffle: getMuffle()})
   }
 </script>
 
-<Notes shouldMuffle {loadNotes} {queryNotes} />
+<Notes {listenForNotes} {loadNotes} />