Group feeds better

README.md

@@ -83,9 +83,6 @@ If you like Coracle and want to support its development, you can donate sats via
 - [ ] Deterministically calculate color for relays, show it on notes. User popper?
 - [ ] Likes list
 - [ ] Fix anon/new user experience
-- [ ] Stream likes rather than load, they're probably what is slowing things down. Figure out how to multiplex them, or store them in the database by count
-  - Stream likes and replies, only load parent up front
-  - Show full thread on detail view - fetch parent, load all descendants, highlight anchor
 - [ ] Likes are slow
 - [ ] Show loading on replies/notes
 
@@ -93,7 +90,7 @@ If you like Coracle and want to support its development, you can donate sats via
 
 ## 0.2.12
 
-
+- [x] Stream likes and replies in lazily
 
 ## 0.2.11
 

src/agent/database.ts

@@ -1,8 +1,8 @@
 import {debounce} from 'throttle-debounce'
-import {is, prop, find, without, pluck, all, identity} from 'ramda'
+import {filter, always, is, prop, find, without, pluck, all, identity} from 'ramda'
 import {writable, derived} from 'svelte/store'
 import {switcherFn, createMap, ensurePlural} from 'hurdak/lib/hurdak'
-import {defer, where, asyncIterableToArray} from 'src/util/misc'
+import {defer, where, now, timedelta, asyncIterableToArray} from 'src/util/misc'
 
 // Types
 
@@ -138,7 +138,13 @@ const iterate = (storeName, where = {}) => ({
 
 const registry = {}
 
-const defineTable = (name: string, pk: string): Table => {
+type TableOpts = {
+  isValid?: (x: any) => boolean
+  resetOnInit?: boolean
+}
+
+const defineTable = (name: string, pk: string, opts: TableOpts = {}): Table => {
+  const {isValid = always(true), resetOnInit = false} = opts
   let p = Promise.resolve()
   let listeners = []
   let data = {}
@@ -170,6 +176,7 @@ const defineTable = (name: string, pk: string): Table => {
       throw new Error(`Updates must be an object, not an array`)
     }
 
+    newData = filter(isValid, newData)
     setAndNotify({...data, ...newData})
 
     // Sync to storage, keeping updates in order
@@ -209,10 +216,18 @@ const defineTable = (name: string, pk: string): Table => {
   ;(async () => {
     const initialData = {}
     for await (const {k, v} of iterate(name)) {
-      initialData[k] = v
+      if (isValid(v)) {
+        initialData[k] = v
+      }
+    }
+
+    if (resetOnInit) {
+      await clear(name)
+      await bulkPut(initialData)
+    } else {
+      setAndNotify(initialData)
     }
 
-    setAndNotify(initialData)
     ready.set(true)
   })()
 
@@ -229,7 +244,11 @@ const rooms = defineTable('rooms', 'id')
 const messages = defineTable('messages', 'id')
 const alerts = defineTable('alerts', 'id')
 const relays = defineTable('relays', 'url')
-const routes = defineTable('routes', 'id')
+const routes = defineTable('routes', 'id', {
+  resetOnInit: true,
+  isValid: route =>
+    route.last_seen > now() - timedelta(7, 'days'),
+})
 
 // Helper to allow us to listen to changes of any given table
 

src/agent/sync.ts

@@ -167,11 +167,17 @@ const calculateRoute = (pubkey, url, type, mode, created_at) => {
   const id = hash([pubkey, url, mode].join('')).toString()
   const score = getWeight(type) * (1 - (now() - created_at) / timedelta(30, 'days'))
   const route = database.routes.get(id) || {id, pubkey, url, mode, score: 0, count: 0}
+
   const newTotalScore = route.score * route.count + score
   const newCount = route.count + 1
 
   if (score > 0) {
-    return {...route, count: newCount, score: newTotalScore / newCount}
+    return {
+      ...route,
+      count: newCount,
+      score: newTotalScore / newCount,
+      last_seen: Math.max(created_at, route.last_seen || 0),
+    }
   }
 }
 

src/app/index.ts

@@ -1,5 +1,5 @@
 import type {Person, DisplayEvent} from 'src/util/types'
-import {groupBy, whereEq, identity, when, assoc, reject} from 'ramda'
+import {assoc, omit, sortBy, whereEq, identity, when, reject} from 'ramda'
 import {navigate} from 'svelte-routing'
 import {createMap, ellipsize} from 'hurdak/lib/hurdak'
 import {get} from 'svelte/store'
@@ -128,11 +128,21 @@ export const asDisplayEvent = event =>
   ({children: [], replies: [], reactions: [], ...event})
 
 export const mergeParents = (notes: Array<DisplayEvent>) => {
-  const m = createMap('id', notes)
+  const notesById = createMap('id', notes)
+  const childIds = []
 
-  return Object.entries(groupBy(findReplyId, notes))
-    // Substiture parent and add notes as children
-    .flatMap(([p, children]) => m[p] ? [{...m[p], children}] : children)
-    // Remove replies where we failed to find a parent
-    .filter((note: DisplayEvent) => !findReplyId(note) || note.children.length > 0)
+  for (const note of notes) {
+    const parentId = findReplyId(note)
+
+    if (parentId) {
+      childIds.push(note.id)
+    }
+
+    // Add the current note to its parents children, but only if we found a parent
+    if (notesById[parentId]) {
+      notesById[parentId].children = notesById[parentId].children.concat(note)
+    }
+  }
+
+  return sortBy(e => -e.created_at, Object.values(omit(childIds, notesById)))
 }

src/partials/Note.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import cx from 'classnames'
   import {nip19} from 'nostr-tools'
-  import {whereEq, without, uniq, pluck, reject, propEq, find} from 'ramda'
+  import {always, whereEq, without, uniq, pluck, reject, propEq, find} from 'ramda'
   import {tweened} from 'svelte/motion'
   import {slide} from 'svelte/transition'
   import {navigate} from 'svelte-routing'
@@ -23,6 +23,7 @@
   export let anchorId = null
   export let showParent = true
   export let invertColors = false
+  export let shouldDisplay = always(true)
 
   const getDefaultReplyMentions = () =>
     Tags.from(note).type("p").values().all().concat(note.pubkey)
@@ -164,7 +165,7 @@
   }}
 />
 
-{#if $person}
+{#if $person && shouldDisplay(note)}
 <Card on:click={onClick} {interactive} {invertColors}>
   <div class="flex gap-4 items-center justify-between">
     <Badge person={$person} />
@@ -259,7 +260,7 @@
   </button>
   {/if}
   {#each note.children.slice(showEntire ? 0 : -3) as r (r.id)}
-  <svelte:self showParent={false} note={r} {invertColors} {anchorId} />
+  <svelte:self showParent={false} note={r} {invertColors} {anchorId} {shouldDisplay} />
   {/each}
 </div>
 {/if}

src/partials/Notes.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import {onMount} from 'svelte'
-  import {mergeRight, uniqBy, sortBy, prop} from 'ramda'
+  import {propEq, always, mergeRight, uniqBy, sortBy, prop} from 'ramda'
   import {slide} from 'svelte/transition'
   import {quantify} from 'hurdak/lib/hurdak'
   import {createScroller, now, Cursor} from 'src/util/misc'
@@ -13,6 +13,7 @@
 
   export let relays
   export let filter
+  export let shouldDisplay = always(true)
 
   let notes = []
   let notesBuffer = []
@@ -29,6 +30,7 @@
     const combined = uniqBy(
       prop('id'),
       newNotes
+        .filter(propEq('kind', 1))
         .concat(await network.loadParents(relays, newNotes))
         .map(mergeRight({replies: [], reactions: [], children: []}))
     )
@@ -48,7 +50,10 @@
 
   const loadBufferedNotes = () => {
     // Drop notes at the end if there are a lot
-    notes = uniqBy(prop('id'), notesBuffer.splice(0).concat(notes).slice(0, maxNotes))
+    notes = uniqBy(
+      prop('id'),
+      notesBuffer.splice(0).filter(shouldDisplay).concat(notes).slice(0, maxNotes)
+    )
   }
 
   onMount(() => {
@@ -93,18 +98,18 @@
 </script>
 
 <Content size="inherit" class="pt-6">
-  {#if notesBuffer.length > 0}
+  {#if notesBuffer.filter(shouldDisplay).length > 0}
   <button
     in:slide
     class="cursor-pointer text-center underline text-light"
     on:click={loadBufferedNotes}>
-    Load {quantify(notesBuffer.length, 'new note')}
+    Load {quantify(notesBuffer.filter(shouldDisplay).length, 'new note')}
   </button>
   {/if}
 
   <div>
     {#each notes as note (note.id)}
-    <Note {note} />
+    <Note {note} {shouldDisplay} />
     {/each}
   </div>
 

src/routes/Notes.svelte

@@ -4,7 +4,7 @@
   import Content from "src/partials/Content.svelte"
   import Tabs from "src/partials/Tabs.svelte"
   import Network from "src/views/notes/Network.svelte"
-  import Global from "src/views/notes/Global.svelte"
+  import Popular from "src/views/notes/Popular.svelte"
   import {user} from 'src/agent/helpers'
 
   export let activeTab
@@ -22,11 +22,11 @@
   {/if}
 
   <div>
-    <Tabs tabs={['network', 'global']} {activeTab} {setActiveTab} />
+    <Tabs tabs={['network', 'popular']} {activeTab} {setActiveTab} />
     {#if activeTab === 'network'}
     <Network />
     {:else}
-    <Global />
+    <Popular />
     {/if}
   </div>
 </Content>

src/routes/Person.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
   import {last, find, reject} from 'ramda'
-  import {onMount, onDestroy} from 'svelte'
+  import {onMount} from 'svelte'
+  import {tweened} from 'svelte/motion'
   import {nip19} from 'nostr-tools'
   import {fly} from 'svelte/transition'
   import {navigate} from 'svelte-routing'
@@ -25,11 +26,12 @@
   export let activeTab
   export let relays = []
 
-  let subs = []
+  const interpolate = (a, b) => t => a + Math.round((b - a) * t)
+
   let pubkey = nip19.decode(npub).data as string
   let following = false
   let followers = new Set()
-  let followersCount = 0
+  let followersCount = tweened(0, {interpolate, duration: 1000})
   let person = database.getPersonWithFallback(pubkey)
   let loading = true
 
@@ -45,24 +47,27 @@
       loading = false
     })
 
-    // Get our followers count
-    subs.push(
-      await network.listen(
-        relays,
-        [{kinds: [3], '#p': [pubkey]}],
-        e => {
-          followers.add(e.pubkey)
-          followersCount = followers.size
-        },
-        {shouldProcess: false},
-      )
-    )
-  })
+    // Prime our followers count
+    database.people.all().forEach(p => {
+      if (Tags.wrap(p.petnames).type("p").values().all().includes(pubkey)) {
+        followers.add(p.pubkey)
+        followersCount.set(followers.size)
+      }
+    })
 
-  onDestroy(() => {
-    for (const sub of subs) {
-      sub.unsub()
-    }
+    // Round it out
+    await network.listenUntilEose(
+      relays,
+      [{kinds: [3], '#p': [pubkey]}],
+      events => {
+        for (const e of events) {
+          followers.add(e.pubkey)
+        }
+
+        followersCount.set(followers.size)
+      },
+      {shouldProcess: false},
+    )
   })
 
   const setActiveTab = tab => navigate(routes.person(pubkey, tab))
@@ -157,7 +162,7 @@
           <strong>{person.petnames.length}</strong> following
         </button>
         <button on:click={showFollowers}>
-          <strong>{followersCount}</strong> followers
+          <strong>{$followersCount}</strong> followers
         </button>
       </div>
       {/if}

src/util/nostr.js

@@ -12,7 +12,7 @@ export class Tags {
     return new Tags(ensurePlural(events).flatMap(prop('tags')))
   }
   static wrap(tags) {
-    return new Tags(tags.filter(identity))
+    return new Tags((tags || []).filter(identity))
   }
   all() {
     return this.tags

src/views/notes/Global.svelte

@@ -1,9 +0,0 @@
-<script>
-  import Notes from "src/partials/Notes.svelte"
-  import {getUserRelays} from 'src/agent/helpers'
-
-  const relays = getUserRelays('read')
-  const filter = {kinds: [1, 5, 7]}
-</script>
-
-<Notes {relays} {filter} />

src/views/notes/Popular.svelte

@@ -0,0 +1,24 @@
+<script>
+  import {uniq} from 'ramda'
+  import Notes from "src/partials/Notes.svelte"
+  import {shuffle} from 'src/util/misc'
+  import {isLike} from 'src/util/nostr'
+  import {user, getTopRelays, getFollows} from 'src/agent/helpers'
+
+  // Get first- and second-order follows. shuffle and slice network so we're not
+  // sending too many pubkeys. This will also result in some variety.
+  const follows = shuffle(getFollows($user?.pubkey))
+  const others = shuffle(follows.flatMap(getFollows)).slice(0, 50)
+  const authors = uniq(follows.concat(others)).slice(0, 100)
+  const relays = getTopRelays(authors, 'write')
+  const filter = {kinds: [1, 7], authors}
+
+  const shouldDisplay = note => {
+    return (
+      note.reactions.filter(isLike).length > 2
+      || note.replies.length > 2
+    )
+  }
+</script>
+
+<Notes {relays} {filter} {shouldDisplay} />

src/views/notes/Top.svelte

@@ -0,0 +1,24 @@
+<script>
+  import {uniq} from 'ramda'
+  import Notes from "src/partials/Notes.svelte"
+  import {shuffle} from 'src/util/misc'
+  import {isLike} from 'src/util/nostr'
+  import {user, getTopRelays, getFollows} from 'src/agent/helpers'
+
+  // Get first- and second-order follows. shuffle and slice network so we're not
+  // sending too many pubkeys. This will also result in some variety.
+  const follows = shuffle(getFollows($user?.pubkey))
+  const others = shuffle(follows.flatMap(getFollows)).slice(0, 50)
+  const authors = uniq(follows.concat(others)).slice(0, 100)
+  const relays = getTopRelays(authors, 'write')
+  const filter = {kinds: [1, 7], authors}
+
+  const shouldDisplay = note => {
+    return (
+      note.reactions.filter(isLike).length > 2
+      || note.replies.length > 2
+    )
+  }
+</script>
+
+<Notes {relays} {filter} {shouldDisplay} />