Fix alerts

README.md

@@ -67,6 +67,12 @@ If you like Coracle and want to support its development, you can donate sats via
 - [ ] Clean up login page to prefer extension, make private key entry "advanced"
 - [ ] Do I need to implement re-connecting now?
 - [ ] handle localstorage limits https://stackoverflow.com/questions/2989284/what-is-the-max-size-of-localstorage-values
+- [ ] Clean up nostr utils
+- [ ] Improve login UX for bootstrap delay. Nostr facts?
+- [ ] Use bech32 entities
+- [ ] Revisit pagination. Use bigger timedelta + limit, set earliest seen timestamp when paginating? Handle no results on page.
+- [ ] We often get the root as the reply, figure out why that is
+- [ ] Alerts still aren't great. Maybe lazy load? We delete old events, so context will disappear and notes will become empty.
 
 ## 0.2.6
 

package.json

@@ -30,6 +30,7 @@
     "extract-urls": "^1.3.2",
     "fuse.js": "^6.6.2",
     "hurdak": "github:ConsignCloud/hurdak",
+    "lz-string": "^1.4.4",
     "nostr-tools": "^1.1.1",
     "ramda": "^0.28.0",
     "svelte-link-preview": "^0.3.3",

src/App.svelte

@@ -12,7 +12,7 @@
   import {displayPerson, isLike} from 'src/util/nostr'
   import {timedelta, now} from 'src/util/misc'
   import {user} from 'src/agent'
-  import {modal, toast, settings, alerts} from "src/app"
+  import {modal, toast, settings, alerts, getRelays} from "src/app"
   import Anchor from 'src/partials/Anchor.svelte'
   import NoteDetail from "src/views/NoteDetail.svelte"
   import PersonSettings from "src/views/PersonSettings.svelte"
@@ -44,6 +44,10 @@
   let {since, latest} = alerts
 
   onMount(() => {
+    if ($user) {
+      alerts.listen(getRelays(), $user.pubkey)
+    }
+
     // Close menu on click outside
     document.querySelector("html").addEventListener("click", e => {
       if (e.target !== menuIcon) {

src/agent/data.js

@@ -1,47 +1,51 @@
 import Dexie from 'dexie'
-import {writable} from 'svelte/store'
+import {matchFilter} from 'nostr-tools'
+import {writable, get} from 'svelte/store'
 import {groupBy, prop, flatten, pick} from 'ramda'
 import {ensurePlural, switcherFn} from 'hurdak/lib/hurdak'
-import {now, timedelta} from 'src/util/misc'
-import {filterTags, findReply, findRoot} from 'src/util/nostr'
+import {synced, now, timedelta} from 'src/util/misc'
+import {filterTags, personKinds, findReply, findRoot} from 'src/util/nostr'
+import keys from 'src/agent/keys'
 
 export const db = new Dexie('agent/data/db')
 
-db.version(6).stores({
+db.version(7).stores({
   relays: '++url, name',
   events: '++id, pubkey, created_at, loaded_at, kind, content, reply, root',
   tags: '++key, event, value, created_at, loaded_at',
+  alerts: '++id',
 })
 
 // Some things work better as observables than database tables
 
-export const people = writable({})
+export const people = synced('agent/data/people', {})
 
 let $people = {}
 people.subscribe($p => {
   $people = $p
 })
 
-export const getPerson = pubkey => $people[pubkey]
+export const getPerson = (pubkey, fallback = false) =>
+  $people[pubkey] || (fallback ? {pubkey} : null)
 
 // Hooks
 
 export const processEvents = async events => {
   // Only persist ones we care about, the rest can be ephemeral and used to update people etc
+  const pubkey = get(keys.pubkey)
   const eventsByKind = groupBy(prop('kind'), ensurePlural(events))
   const notesAndReactions = flatten(Object.values(pick([1, 7], eventsByKind)))
-  const profileUpdates = flatten(Object.values(pick([0, 3, 12165], eventsByKind)))
+    .map(e => ({...e, root: findRoot(e), reply: findReply(e), loaded_at: now()}))
+  const alerts = notesAndReactions.filter(e => matchFilter({kinds: [1, 7], '#p': [pubkey]}, e))
+  const profileUpdates = flatten(Object.values(pick(personKinds, eventsByKind)))
   const deletions = eventsByKind[5] || []
 
   // Persist notes and reactions
   if (notesAndReactions.length > 0) {
-    const persistentEvents = notesAndReactions
-      .map(e => ({...e, root: findRoot(e), reply: findReply(e), loaded_at: now()}))
-
-    db.events.bulkPut(persistentEvents)
+    db.events.bulkPut(notesAndReactions)
 
     db.tags.bulkPut(
-      persistentEvents
+      notesAndReactions
         .flatMap(e =>
           e.tags.map(
             tag => ({
@@ -58,6 +62,10 @@ export const processEvents = async events => {
     )
   }
 
+  if (alerts.length > 0) {
+    db.alerts.bulkPut(alerts)
+  }
+
   // Delete stuff that needs to be deleted
   if (deletions.length > 0) {
     const eventIds = deletions.flatMap(e => filterTags({tag: "e"}, e))
@@ -81,8 +89,10 @@ export const processEvents = async events => {
 
       switcherFn(kind, {
         0: () => putPerson(JSON.parse(content)),
+        2: () => putPerson({relays: ($people[pubkey]?.relays || []).concat(content)}),
         3: () => putPerson({petnames: tags}),
         12165: () => putPerson({muffle: tags}),
+        10001: () => putPerson({relays: tags}),
         default: () => {
           console.log(`Received unsupported event type ${event.kind}`)
         },

src/agent/index.js

@@ -9,11 +9,17 @@ export {pool, keys, db, people, getPerson}
 
 export const user = derived(
   [keys.pubkey, people],
-  ([pubkey, $people]) => $people[pubkey] || {pubkey}
+  ([pubkey, $people]) => {
+    if (!pubkey) {
+      return null
+    }
+
+    return $people[pubkey] || {pubkey}
+  }
 )
 
 export const publish = async (relays, event) => {
-  const signedEvent = keys.sign(event)
+  const signedEvent = await keys.sign(event)
 
   await Promise.all([
     pool.publish(relays, signedEvent),

src/agent/keys.js

@@ -37,4 +37,8 @@ const clear = () => {
   privkey.set(null)
 }
 
+
+// Init signing function by re-setting pubkey
+setPublicKey(get(pubkey))
+
 export default {pubkey, setPrivateKey, setPublicKey, sign, clear}

src/app/alerts.js

@@ -1,5 +1,7 @@
 import {get} from 'svelte/store'
-import {synced, now, timedelta} from 'src/util/misc'
+import {sortBy, pluck} from 'ramda'
+import {first} from 'hurdak/lib/hurdak'
+import {synced, batch, now, timedelta} from 'src/util/misc'
 import {isAlert} from 'src/util/nostr'
 import {listen as _listen} from 'src/agent'
 import {getRelays} from 'src/app'
@@ -9,25 +11,32 @@ let listener
 
 const start = now() - timedelta(30, 'days')
 
-export const since = synced("app/alerts/since", start)
-export const latest = synced("app/alerts/latest", start)
+const since = synced("app/alerts/since", start)
+const latest = synced("app/alerts/latest", start)
 
-export const listen = async (relays, pubkey) => {
+const listen = async (relays, pubkey) => {
   if (listener) {
     listener.unsub()
   }
 
+  console.log(get(since))
+
   listener = await _listen(
     relays,
-    [{kinds: [1, 7], '#p': [pubkey], since: get(since)}],
-    e => {
-      if (isAlert(e, pubkey)) {
-        loaders.loadNotesContext(getRelays(), [e])
+    [{kinds: [1, 7], '#p': [pubkey], since: start}],
+    batch(300, events => {
+      events = events.filter(e => isAlert(e, pubkey))
 
-        latest.set(Math.max(e.created_at, get(latest)))
+      if (events.length > 0) {
+        loaders.loadNotesContext(getRelays(), events)
+
+        latest.update(
+          $latest =>
+            Math.max(first(sortBy(t => -t, pluck('created_at', events))), $latest)
+        )
       }
-    }
+    })
   )
 }
 
-export default {latest, listen}
+export default {listen, since, latest}

src/app/defaults.js

@@ -8,9 +8,9 @@ export default {
     ["p", "85080d3bad70ccdcd7f74c29a44f55bb85cbcd3dd0cbb957da1d215bdb931204", "preston",      "wss://relay.damus.io"],
   ],
   relays: [
-    // 'wss://nostr.rocks',
-    // 'wss://astral.ninja',
-    // 'wss://relay.damus.io',
+    'wss://nostr.rocks',
+    'wss://relay.damus.io',
+    'wss://nostr.zebedee.cloud',
     'wss://nostr-pub.wellorder.net',
   ],
 }

src/app/index.js

@@ -1,7 +1,7 @@
 import {without, reject} from 'ramda'
-import {first} from 'hurdak/lib/hurdak'
+import {first, updateIn, mergeRight} from 'hurdak/lib/hurdak'
 import {get} from 'svelte/store'
-import {getPerson, keys, db} from 'src/agent'
+import {getPerson, people, keys, db} from 'src/agent'
 import {toast, modal, settings} from 'src/app/ui'
 import cmd from 'src/app/cmd'
 import alerts from 'src/app/alerts'
@@ -11,9 +11,17 @@ import defaults from 'src/app/defaults'
 export {toast, modal, settings, alerts}
 
 export const getRelays = pubkey => {
-  const person = getPerson(pubkey)
+  let relays = getPerson(pubkey)?.relays
 
-  return person && person.relays.length > 0 ? person.relays : defaults.relays
+  if (!relays?.length) {
+    relays = getPerson(get(keys.pubkey))?.relays
+  }
+
+  if (!relays?.length) {
+    relays = defaults.relays
+  }
+
+  return relays
 }
 
 export const getBestRelay = pubkey => {
@@ -47,9 +55,12 @@ export const logout = async () => {
 export const addRelay = async url => {
   const pubkey = get(keys.pubkey)
   const person = getPerson(pubkey)
-  const relays = person?.relays || []
+  const relays = (person?.relays || []).concat(url)
 
-  await cmd.setRelays(relays.concat(url))
+  // Persist to our local copy immediately so we can publish to the new one
+  people.update(updateIn(pubkey, mergeRight({pubkey, relays})))
+
+  await cmd.setRelays(relays)
   await loaders.loadNetwork(relays, pubkey)
   await alerts.listen(relays, pubkey)
 }

src/app/loaders.js

@@ -1,11 +1,9 @@
-import {uniq, pluck, groupBy, prop, identity} from 'ramda'
+import {uniq, pluck, groupBy, identity} from 'ramda'
 import {ensurePlural, createMap} from 'hurdak/lib/hurdak'
-import {filterTags, findReply} from 'src/util/nostr'
+import {findReply, personKinds} from 'src/util/nostr'
 import {load, db, getPerson} from 'src/agent'
 import defaults from 'src/app/defaults'
 
-const personKinds = [0, 2, 3, 10001, 12165]
-
 const loadPeople = (relays, pubkeys, {kinds = personKinds, ...opts} = {}) =>
   pubkeys.length > 0 ? load(relays, {kinds, authors: pubkeys}, opts) : []
 
@@ -74,4 +72,4 @@ const getOrLoadNote = async (relays, id) => {
   return note
 }
 
-export default {getOrLoadNote, loadNotesContext, loadNetwork, loadPeople}
+export default {getOrLoadNote, loadNotesContext, loadNetwork, loadPeople, personKinds}

src/app/query.js

@@ -57,7 +57,7 @@ const findNote = async (id, {showEntire = false, depth = 1} = {}) => {
   const note = await db.events.get(id)
 
   if (!note) {
-    return
+    return null
   }
 
   const reactions = await filterReactions(note.id)

src/partials/Note.svelte

@@ -14,7 +14,7 @@
   import Badge from "src/partials/Badge.svelte"
   import Compose from "src/partials/Compose.svelte"
   import Card from "src/partials/Card.svelte"
-  import {user} from 'src/agent'
+  import {user, getPerson} from 'src/agent'
   import cmd from 'src/app/cmd'
 
   export let note
@@ -111,7 +111,7 @@
 
 <Card on:click={onClick} {interactive} {invertColors}>
   <div class="flex gap-4 items-center justify-between">
-    <Badge person={{...note.person, pubkey: note.pubkey}} />
+    <Badge person={getPerson(note.pubkey, true)} />
     <p class="text-sm text-light">{formatTimestamp(note.created_at)}</p>
   </div>
   <div class="ml-6 flex flex-col gap-2">

src/partials/Preview.svelte

@@ -17,15 +17,15 @@
           'Content-Type': 'application/json',
         },
       })
+
+      const json = await res.json()
+
+      if (json.title || json.image) {
+        preview = json
+      }
     } catch (e) {
       return
     }
-
-    const json = await res.json()
-
-    if (json.title || json.image) {
-      preview = json
-    }
   })
 </script>
 

src/routes/Alerts.svelte

@@ -1,69 +1,62 @@
 <script>
-  import {propEq, sortBy} from 'ramda'
+  import {propEq, identity, uniq, uniqBy, prop, sortBy} from 'ramda'
   import {onMount} from 'svelte'
   import {fly} from 'svelte/transition'
-  import {now} from 'src/util/misc'
+  import {createMap} from 'hurdak/lib/hurdak'
+  import {now, createScroller, timedelta} from 'src/util/misc'
   import {findReply, isLike} from 'src/util/nostr'
-  import {getPerson, user} from 'src/agent'
+  import {getPerson, user, db} from 'src/agent'
   import {alerts} from 'src/app'
   import query from 'src/app/query'
   import Spinner from "src/partials/Spinner.svelte"
   import Note from 'src/partials/Note.svelte'
   import Like from 'src/partials/Like.svelte'
 
+  let limit = 0
   let annotatedNotes = []
 
   onMount(async () => {
     alerts.since.set(now())
 
-    const events = await query.filterEvents({
-      kinds: [1, 7],
-      '#p': [$user.pubkey],
-      customFilter: e => {
-        // Don't show people's own stuff
-        if (e.pubkey === $user.pubkey) {
-          return false
+    return createScroller(async () => {
+      limit += 10
+
+      const events = await db.alerts.toArray()
+      const parentIds = uniq(events.map(prop('reply')).filter(identity))
+      const parents = await Promise.all(parentIds.map(query.findNote))
+      const parentsById = createMap('id', parents.filter(identity))
+
+      const notes = await Promise.all(
+        events.filter(propEq('kind', 1)).map(n => query.findNote(n.id))
+      )
+
+      const reactions = await Promise.all(
+        events
+          .filter(e => e.kind === 7 && parentsById[e.reply])
+          .map(async e => ({
+            ...e,
+            person: getPerson(e.pubkey, true),
+            parent: parentsById[e.reply],
+          }))
+      )
+
+      // Combine likes of a single note. Remove grandchild likes
+      const likesById = {}
+      for (const reaction of reactions.filter(e => e.parent?.pubkey === $user.pubkey)) {
+        if (!likesById[reaction.parent.id]) {
+          likesById[reaction.parent.id] = {...reaction.parent, people: []}
         }
 
-        // Only notify users about positive reactions
-        if (e.kind === 7 && !isLike(e.content)) {
-          return false
-        }
-
-        return true
+        likesById[reaction.parent.id].people.push(reaction.person)
       }
+
+      annotatedNotes = sortBy(
+        e => -e.created_at,
+        notes
+          .filter(e => e.pubkey !== $user.pubkey)
+          .concat(Object.values(likesById))
+      ).slice(0, limit)
     })
-
-    const notes = await query.annotateChunk(
-      events.filter(propEq('kind', 1))
-    )
-
-    const reactions = await Promise.all(
-      events
-        .filter(e => e.kind === 7)
-        .map(async e => ({
-          ...e,
-          person: getPerson(e.pubkey),
-          parent: await query.findNote(findReply(e)),
-        }))
-    )
-
-    // Combine likes of a single note. Remove grandchild likes
-    const likesById = {}
-    for (const reaction of reactions.filter(e => e.parent?.pubkey === $user.pubkey)) {
-      if (!likesById[reaction.parent.id]) {
-        likesById[reaction.parent.id] = {...reaction.parent, people: []}
-      }
-
-      likesById[reaction.parent.id].people.push(reaction.person)
-    }
-
-    annotatedNotes = sortBy(
-      e => -e.created_at,
-      notes
-        .filter(e => e.pubkey !== $user.pubkey)
-        .concat(Object.values(likesById))
-    )
   })
 </script>
 
@@ -76,5 +69,3 @@
   {/if}
   {/each}
 </ul>
-
-<Spinner />

src/routes/Person.svelte

@@ -1,11 +1,11 @@
 <script>
-  import {find, when, propEq} from 'ramda'
+  import {find, propEq} from 'ramda'
   import {onMount, onDestroy} from 'svelte'
   import {fly} from 'svelte/transition'
   import {navigate} from 'svelte-routing'
-  import {now} from 'src/util/misc'
+  import {now, batch} from 'src/util/misc'
   import {renderContent} from 'src/util/html'
-  import {displayPerson} from 'src/util/nostr'
+  import {displayPerson, personKinds} from 'src/util/nostr'
   import Tabs from "src/partials/Tabs.svelte"
   import Button from "src/partials/Button.svelte"
   import Notes from "src/views/person/Notes.svelte"
@@ -23,22 +23,29 @@
   let following = find(t => t[1] === pubkey, $user?.petnames || [])
   let followers = new Set()
   let followersCount = 0
-  let person
-
-  $: {
-    person = getPerson(pubkey) || {pubkey}
-  }
+  let person = getPerson(pubkey, true)
 
   onMount(async () => {
     subs.push(await listen(
-      getRelays(),
+      getRelays(pubkey),
       [{kinds: [1, 5, 7], authors: [pubkey], since: now()},
-       {kinds: [0, 3, 12165], authors: [pubkey]}],
-      when(propEq('kind', 1), loaders.loadNoteContext)
+       {kinds: personKinds, authors: [pubkey]}],
+      batch(300, events => {
+        const profiles = events.filter(propEq('kind', 0))
+        const notes = events.filter(propEq('kind', 1))
+
+        if (profiles.length > 0) {
+          person = getPerson(pubkey, true)
+        }
+
+        if (notes.length > 0) {
+          loaders.loadNoteContext(notes)
+        }
+      })
     ))
 
     subs.push(await listen(
-      getRelays(),
+      getRelays(pubkey),
       [{kinds: [3], '#p': [pubkey]}],
       e => {
         followers.add(e.pubkey)
@@ -99,7 +106,7 @@
         <a href="/profile" class="cursor-pointer text-sm">
           <i class="fa-solid fa-edit" /> Edit
         </a>
-        {:else if $user.petnames}
+        {:else if $user?.petnames}
         <div class="flex flex-col items-end gap-2">
           {#if following}
           <Button on:click={unfollow}>Unfollow</Button>

src/routes/RelayList.svelte

@@ -1,17 +1,28 @@
 <script>
   import {liveQuery} from 'dexie'
   import {without} from 'ramda'
+  import {get} from 'svelte/store'
   import {fly} from 'svelte/transition'
   import {fuzzy} from "src/util/misc"
   import Input from "src/partials/Input.svelte"
   import Anchor from "src/partials/Anchor.svelte"
   import {db, user} from "src/agent"
-  import {modal, addRelay, removeRelay} from "src/app"
+  import {modal, addRelay, removeRelay, settings} from "src/app"
   import defaults from "src/app/defaults"
 
   let q = ""
   let search
-  let relays = $user?.relays || []
+  let relays = $user?.relays || defaults.relays
+
+  fetch(get(settings).dufflepudUrl + '/relay').then(r => r.json()).then(({relays}) => {
+    for (const url of relays) {
+      db.relays.put({url})
+    }
+  })
+
+  for (const url of defaults.relays) {
+     db.relays.put({url})
+  }
 
   const knownRelays = liveQuery(() => db.relays.toArray())
 

src/util/misc.js

@@ -1,3 +1,4 @@
+import LZ from 'lz-string'
 import {Buffer} from 'buffer'
 import {bech32} from 'bech32'
 import {debounce} from 'throttle-debounce'
@@ -16,9 +17,9 @@ export const fuzzy = (data, opts = {}) => {
 export const hash = s =>
   Math.abs(s.split("").reduce((a, b) => ((a << 5) - a + b.charCodeAt(0)) | 0, 0))
 
-export const getLocalJson = k => {
+export const getLocalJson = (k) => {
   try {
-    return JSON.parse(localStorage.getItem(k))
+    return JSON.parse(LZ.decompress(localStorage.getItem(k)))
   } catch (e) {
     console.warn(`Unable to parse ${k}: ${e}`)
 
@@ -26,9 +27,9 @@ export const getLocalJson = k => {
   }
 }
 
-export const setLocalJson = (k, v) => {
+export const setLocalJson = (k, v, compressed = false) => {
   try {
-    localStorage.setItem(k, JSON.stringify(v))
+    localStorage.setItem(k, LZ.compress(JSON.stringify(v)))
   } catch (e) {
     console.warn(`Unable to set ${k}: ${e}`)
   }
@@ -160,7 +161,7 @@ export const synced = (key, defaultValue = null) => {
       : (getLocalJson(key) || defaultValue)
   )
 
-  store.subscribe($value => setLocalJson(key, $value))
+  store.subscribe(debounce(1000, $value => setLocalJson(key, $value)))
 
   return store
 }

src/util/nostr.js

@@ -4,6 +4,8 @@ import {hexToBech32} from 'src/util/misc'
 
 export const epoch = 1633046400
 
+export const personKinds = [0, 2, 3, 10001, 12165]
+
 export const getTagValues = tags => tags.map(t => t[1])
 
 export const filterTags = (where, events) =>

src/views/person/Likes.svelte

@@ -14,9 +14,9 @@
   const loadNotes = async () => {
     const [since, until] = cursor.step()
     const filter = {kinds: [7], authors: [pubkey], since, until}
-    const notes = await load(getRelays(), filter)
+    const notes = await load(getRelays(pubkey), filter)
 
-    await loaders.loadNotesContext(getRelays(), notes, {loadParents: true})
+    await loaders.loadNotesContext(getRelays(pubkey), notes, {loadParents: true})
   }
 
   const queryNotes = () => {

src/views/person/Network.svelte

@@ -13,12 +13,11 @@
 
   const loadNotes = async () => {
     const [since, until] = cursor.step()
-    console.log(person)
     const authors = getTagValues(person.petnames)
     const filter = {since, until, kinds: [1], authors}
-    const events = await load(getRelays(), filter)
+    const events = await load(getRelays(person.pubkey), filter)
 
-    await loaders.loadNotesContext(getRelays(), events, {loadParents: true})
+    await loaders.loadNotesContext(getRelays(person.pubkey), events, {loadParents: true})
   }
 
   const queryNotes = () => {

src/views/person/Notes.svelte

@@ -13,9 +13,9 @@
   const loadNotes = async () => {
     const [since, until] = cursor.step()
     const filter = {kinds: [1], authors: [pubkey], since, until}
-    const notes = await load(getRelays(), filter)
+    const notes = await load(getRelays(pubkey), filter)
 
-    await loaders.loadNotesContext(getRelays(), notes, {loadParents: true})
+    await loaders.loadNotesContext(getRelays(pubkey), notes, {loadParents: true})
   }
 
   const queryNotes = () => {