optimize channel syncing

src/app/views/ChannelsList.svelte

@@ -1,7 +1,6 @@
 <script lang="ts">
-  import {whereEq, complement, prop, filter} from "ramda"
-  import {toTitle, seconds, batch} from "hurdak"
-  import {now} from "src/util/misc"
+  import {complement, prop, filter} from "ramda"
+  import {toTitle} from "hurdak"
   import {navigate} from "svelte-routing"
   import {modal} from "src/partials/state"
   import Tabs from "src/partials/Tabs.svelte"
@@ -12,20 +11,15 @@
   import ForegroundButtons from "src/partials/ForegroundButtons.svelte"
   import ChannelsListItem from "src/app/views/ChannelsListItem.svelte"
   import {
-    load,
-    session,
-    loadPubkeys,
-    channels,
+    nip24Channels,
     hasNewNip24Messages,
     sortChannels,
-    getUserRelayUrls,
     nip24MarkAllRead,
-    nip59,
+    loadAllNip24Messages,
   } from "src/engine2"
 
   export let activeTab = "conversations"
 
-  const nip24Channels = channels.derived(filter(whereEq({type: "nip24"})))
   const accepted = nip24Channels.derived(filter(prop("last_sent")))
   const requests = nip24Channels.derived(filter(complement(prop("last_sent"))))
 
@@ -40,19 +34,7 @@
 
   document.title = "Direct Messages"
 
-  load({
-    relays: getUserRelayUrls("read"),
-    filters: [{kinds: [1059], "#p": [$session.pubkey], since: now() - seconds(90, "day")}],
-    onEvent: batch(1000, events => {
-      const pubkeys = new Set<string>()
-
-      for (const event of events) {
-        $nip59.withUnwrappedEvent(event, $session.privkey, e => pubkeys.add(e.pubkey))
-      }
-
-      loadPubkeys(Array.from(pubkeys))
-    }),
-  })
+  loadAllNip24Messages()
 </script>
 
 <div class="bg-gray-7">

src/app/views/ChannelsListItem.svelte

@@ -1,13 +1,14 @@
 <script lang="ts">
   import {navigate} from "svelte-routing"
+  import {without} from "ramda"
   import {displayList} from "hurdak"
   import PersonCircles from "src/app/shared/PersonCircles.svelte"
   import Card from "src/partials/Card.svelte"
-  import {people, channels, displayPerson, loadPubkeys, hasNewMessages} from "src/engine2"
+  import {people, channels, displayPerson, loadPubkeys, hasNewMessages, session} from "src/engine2"
 
   export let channel
 
-  const pubkeys = channel.id.split(",")
+  const pubkeys = without([$session.pubkey], channel.id.split(",")) as string[]
   const showAlert = channels.key(channel.id).derived(hasNewMessages)
   const members = people.mapStore.derived($p => pubkeys.map(pk => $p.get(pk)))
 

src/app/views/MessagesList.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import {filter, whereEq, complement, prop} from "ramda"
+  import {filter, complement, prop} from "ramda"
   import {toTitle} from "hurdak"
   import {navigate} from "svelte-routing"
   import Tabs from "src/partials/Tabs.svelte"
@@ -7,7 +7,7 @@
   import Content from "src/partials/Content.svelte"
   import MessagesListItem from "src/app/views/MessagesListItem.svelte"
   import {
-    channels,
+    nip04Channels,
     hasNewNip04Messages,
     sortChannels,
     nip04MarkAllRead,
@@ -16,7 +16,6 @@
 
   export let activeTab = "conversations"
 
-  const nip04Channels = channels.derived(filter(whereEq({type: "nip04"})))
   const accepted = nip04Channels.derived(filter(prop("last_sent")))
   const requests = nip04Channels.derived(filter(complement(prop("last_sent"))))
 

src/engine2/model/index.ts

@@ -18,6 +18,7 @@ export enum EventKind {
   ChannelMessage = 42,
   ChannelHideMessage = 43,
   ChannelMuteUser = 44,
+  GiftWrap = 1059,
   FileMetadata = 1063,
   LiveChatMessage = 1311,
   Report = 1984,

src/engine2/projections/core.ts

@@ -1,4 +1,5 @@
 import {prop} from "ramda"
+import {batch} from "hurdak"
 import {Worker} from "src/engine2/util/worker"
 import type {Event} from "src/engine2/model"
 import {events, sessions} from "src/engine2/state"
@@ -28,8 +29,19 @@ export const updateRecord = (record, timestamp, updates) => {
 export const updateStore = (store, timestamp, updates) =>
   store.set(updateRecord(store.get(), timestamp, updates))
 
-projections.addGlobalHandler(e => {
-  if (sessions.get()[e.pubkey]) {
-    events.key(e.id).set(e)
-  }
-})
+projections.addGlobalHandler(
+  batch(500, chunk => {
+    const $sessions = sessions.get()
+    const userEvents = chunk.filter(e => $sessions[e.pubkey])
+
+    if (userEvents.length > 0) {
+      events.mapStore.update($events => {
+        for (const e of userEvents) {
+          $events.set(e.id, e)
+        }
+
+        return $events
+      })
+    }
+  })
+)

src/engine2/projections/index.ts

@@ -8,6 +8,7 @@ import "./nip28"
 import "./nip32"
 import "./nip51"
 import "./nip57"
+import "./nip59"
 import "./nip65"
 import "./nip78"
 import "./topics"

src/engine2/projections/nip24.ts

@@ -15,13 +15,18 @@ projections.addHandler(30078, async e => {
     await tryJson(async () => {
       const payload = JSON.parse(await nip04.get().decryptAsUser(e.content, e.pubkey))
 
-      for (const [id, ts] of Object.entries(payload) as [string, number][]) {
-        const channel = channels.key(id)
+      channels.mapStore.update($channels => {
+        for (const [id, ts] of Object.entries(payload) as [string, number][]) {
+          const channel = $channels.get(id)
 
-        channel.merge({
-          last_checked: Math.max(ts, channel.get()?.last_checked || 0),
-        })
-      }
+          $channels.set(id, {
+            ...channel,
+            last_checked: Math.max(ts, channel?.last_checked || 0),
+          })
+        }
+
+        return $channels
+      })
     })
   }
 })

src/engine2/projections/nip28.ts

@@ -69,22 +69,27 @@ projections.addHandler(30078, async (e: Event) => {
   }
 
   if (Tags.from(e).getMeta("d") === appDataKeys.NIP28_LAST_CHECKED) {
+    console.log(e)
     await tryJson(async () => {
       const payload = JSON.parse(await nip04.get().decryptAsUser(e.content, e.pubkey))
 
-      for (const key of Object.keys(payload)) {
-        // Backwards compat from when we used to prefix id/pubkey
-        const id = last(key.split("/"))
-        const channel = channels.key(id).get()
-        const last_checked = Math.max(payload[id], channel?.last_checked || 0)
+      channels.mapStore.update($channels => {
+        for (const key of Object.keys(payload)) {
+          // Backwards compat from when we used to prefix id/pubkey
+          const id = last(key.split("/"))
+          const channel = $channels.get(id)
+          const last_checked = Math.max(payload[id], channel?.last_checked || 0)
 
-        // A bunch of junk got added to this setting. Integer keys, settings, etc
-        if (isNaN(last_checked) || last_checked < 1577836800) {
-          continue
+          // A bunch of junk got added to this setting. Integer keys, settings, etc
+          if (isNaN(last_checked) || last_checked < 1577836800) {
+            continue
+          }
+
+          $channels.set(id, {...channel, id, last_checked})
         }
 
-        channels.key(id).merge({last_checked})
-      }
+        return $channels
+      })
     })
   }
 })

src/engine2/queries/channels.ts

@@ -3,6 +3,8 @@ import {fuzzy} from "src/util/misc"
 import type {Channel} from "src/engine2/model"
 import {channels} from "src/engine2/state"
 
+// Common
+
 export const sortChannels = sortBy(
   (c: Channel) => -Math.max(c.last_sent || 0, c.last_received || 0)
 )
@@ -10,25 +12,33 @@ export const sortChannels = sortBy(
 export const hasNewMessages = (c: Channel) =>
   c.last_received > Math.max(c.last_sent || 0, c.last_checked || 0)
 
-export const getNip24ChannelId = (pubkeys: string[]) => sortBy(identity, pubkeys).join(",")
-
-export const hasNewNip28Messages = channels.derived(
-  pipe(filter(path(["nip28", "joined"])), find(hasNewMessages))
-)
-
-export const hasNewNip04Messages = channels.derived(
-  pipe(filter(whereEq({type: "nip04"})), find(hasNewMessages))
-)
-
-export const hasNewNip24Messages = channels.derived(
-  pipe(filter(whereEq({type: "nip24"})), find(hasNewMessages))
-)
-
-export const nip28ChannelsWithMeta = channels.derived(
-  filter((c: Channel) => c.meta && c.type === "nip28")
-)
-
 export const getChannelSearch = $channels =>
   fuzzy($channels, {keys: ["meta.name", "meta.about"], threshold: 0.3})
 
+// Nip04
+
+export const nip04Channels = channels.throttle(300).derived(filter(whereEq({type: "nip04"})))
+
+export const hasNewNip04Messages = nip04Channels.derived(find(hasNewMessages))
+
+// Nip24
+
+export const nip24Channels = channels.throttle(300).derived(filter(whereEq({type: "nip24"})))
+
+export const hasNewNip24Messages = nip24Channels.derived(find(hasNewMessages))
+
+export const getNip24ChannelId = (pubkeys: string[]) => sortBy(identity, pubkeys).join(",")
+
+export const getNip24ChannelPubkeys = (id: string) => id.split(",")
+
+// Nip28
+
+export const nip28ChannelsWithMeta = channels
+  .throttle(300)
+  .derived(filter((c: Channel) => c.meta && c.type === "nip28"))
+
 export const searchNip28Channels = nip28ChannelsWithMeta.derived(getChannelSearch)
+
+export const hasNewNip28Messages = nip28ChannelsWithMeta.derived(
+  pipe(filter(path(["nip28", "joined"])), find(hasNewMessages))
+)

src/engine2/queries/session/nip59.ts

@@ -90,8 +90,8 @@ export class Nip59 {
     // Skip trying to parse the old version
     if (!wrap.content.includes("ciphertext")) {
       try {
-        const seal = await this.decrypt(wrap.content, sk)
-        const rumor = await this.decrypt(seal.content, sk)
+        const seal = await this.decrypt(wrap, sk)
+        const rumor = await this.decrypt(seal, sk)
 
         if (seal.pubkey === rumor.pubkey) {
           return Object.assign(rumor, {wrap, seen_on: wrap.seen_on})

src/engine2/requests/index.ts

@@ -10,6 +10,7 @@ export * from "./subscription"
 export * from "./thread"
 export * from "./nip04"
 export * from "./nip09"
+export * from "./nip24"
 export * from "./nip28"
 export * from "./nip57"
 export * from "./nip59"

src/engine2/requests/nip04.ts

@@ -2,7 +2,7 @@ import {pluck} from "ramda"
 import {batch, seconds} from "hurdak"
 import {now} from "src/util/misc"
 import {EventKind} from "src/engine2/model"
-import {session} from "src/engine2/state"
+import {session, nip04ChannelsLastChecked} from "src/engine2/state"
 import {getInboxHints, getUserRelayUrls} from "src/engine2/queries"
 import {load} from "./load"
 import {loadPubkeys} from "./pubkeys"
@@ -10,7 +10,9 @@ import {subscribe} from "./subscription"
 
 export function loadAllNip04Messages() {
   const {pubkey} = session.get()
-  const since = now() - seconds(90, "day")
+  const since = Math.max(0, nip04ChannelsLastChecked.get() - seconds(7, "day"))
+
+  nip04ChannelsLastChecked.set(now())
 
   load({
     relays: getUserRelayUrls("read"),

src/engine2/requests/nip24.ts

@@ -0,0 +1,25 @@
+import {seconds} from "hurdak"
+import {now} from "src/util/misc"
+import {EventKind} from "src/engine2/model"
+import {session, nip24ChannelsLastChecked} from "src/engine2/state"
+import {getUserRelayUrls, nip24Channels, getNip24ChannelPubkeys} from "src/engine2/queries"
+import {load} from "./load"
+import {loadPubkeys} from "./pubkeys"
+
+export function loadAllNip24Messages() {
+  const {pubkey} = session.get()
+  const since = Math.max(0, nip24ChannelsLastChecked.get() - seconds(7, "day"))
+
+  nip24ChannelsLastChecked.set(now())
+
+  // To avoid unwrapping everything twice, listen to channels and load pubkeys there
+  const unsubscribe = nip24Channels.throttle(1000).subscribe($channels => {
+    loadPubkeys($channels.flatMap(c => getNip24ChannelPubkeys(c.id)))
+  })
+
+  load({
+    relays: getUserRelayUrls("read"),
+    filters: [{kinds: [EventKind.GiftWrap], "#p": [pubkey], since}],
+    onClose: () => setTimeout(unsubscribe, 1000),
+  })
+}

src/engine2/state/index.ts

@@ -8,6 +8,8 @@ export const sessions = writable<Record<string, Session>>({})
 export const session = writable<Session | null>(null)
 export const env = writable<Record<string, any>>({})
 export const notificationsLastChecked = writable(0)
+export const nip04ChannelsLastChecked = writable(0)
+export const nip24ChannelsLastChecked = writable(0)
 
 // Async stores
 

src/engine2/storage/index.ts

@@ -27,6 +27,8 @@ export const storage = new Storage([
   new LocalStorageAdapter("sessions", state.sessions),
   new LocalStorageAdapter("session", state.session),
   new LocalStorageAdapter("notificationsLastChecked", state.notificationsLastChecked),
+  new LocalStorageAdapter("nip04ChannelsLastChecked", state.nip04ChannelsLastChecked),
+  new LocalStorageAdapter("nip24ChannelsLastChecked", state.nip24ChannelsLastChecked),
   new IndexedDBAdapter("events", state.events, {
     max: 10000,
     sort: sortByPubkeyWhitelist(prop("created_at")),