Move loadPeople to PubkeyLoader

.ackrc

@@ -1,5 +1,5 @@
 --ignore-dir=node_modules
 --ignore-dir=android
 --ignore-dir=dist
---ignore-file=match:package-lock.json
+--ignore-file=match:yarn.lock
 --ignore-file=match:.svg

src/agent/network.ts

@@ -1,39 +1,9 @@
-import {
-  max,
-  without,
-  mergeLeft,
-  fromPairs,
-  sortBy,
-  assoc,
-  uniq,
-  uniqBy,
-  prop,
-  propEq,
-  groupBy,
-  pluck,
-} from "ramda"
-import {personKinds, appDataKeys, findReplyId} from "src/util/nostr"
+import {max, mergeLeft, fromPairs, sortBy, assoc, uniqBy, prop, propEq, groupBy, pluck} from "ramda"
+import {findReplyId} from "src/util/nostr"
 import {chunk, ensurePlural} from "hurdak/lib/hurdak"
 import {batch, now, timedelta} from "src/util/misc"
-import {ENABLE_ZAPS, user, routing, directory, network} from "src/app/system"
-
-// If we ask for a pubkey and get nothing back, don't ask again this page load
-const attemptedPubkeys = new Set()
-
-const getStalePubkeys = pubkeys => {
-  // If we're not reloading, only get pubkeys we don't already know about
-  return uniq(pubkeys).filter(pubkey => {
-    if (attemptedPubkeys.has(pubkey)) {
-      return false
-    }
-
-    attemptedPubkeys.add(pubkey)
-
-    const profile = directory.profiles.get(pubkey)
-
-    return !profile || profile.updated_at < now() - timedelta(1, "days")
-  })
-}
+import {PubkeyLoader} from "src/system"
+import system, {ENABLE_ZAPS, user, routing, network} from "src/app/system"
 
 class Cursor {
   relays: string[]
@@ -137,46 +107,6 @@ class Cursor {
   }
 }
 
-const loadPeople = async (
-  pubkeys,
-  {
-    relays = null,
-    kinds = personKinds,
-    force = false,
-  }: {relays?: string[]; kinds?: number[]; force?: boolean} = {}
-) => {
-  pubkeys = uniq(pubkeys)
-
-  // If we're not reloading, only get pubkeys we don't already know about
-  if (!force) {
-    pubkeys = getStalePubkeys(pubkeys)
-  }
-
-  await Promise.all(
-    chunk(256, pubkeys).map(async chunk => {
-      const chunkRelays =
-        relays?.length > 0
-          ? relays
-          : routing.mergeHints(
-              user.getSetting("relay_limit"),
-              chunk.map(pubkey => routing.getPubkeyHints(3, pubkey))
-            )
-
-      const chunkFilter = [] as Array<Record<string, any>>
-
-      chunkFilter.push({kinds: without([30078], kinds), authors: chunk})
-
-      // Add a separate filter for app data so we're not pulling down other people's stuff,
-      // or obsolete events of our own.
-      if (kinds.includes(30078)) {
-        chunkFilter.push({kinds: [30078], authors: chunk, "#d": appDataKeys})
-      }
-
-      await network.load({relays: chunkRelays, filter: chunkFilter})
-    })
-  )
-}
-
 const streamContext = ({notes, onChunk, maxDepth = 2}) => {
   const seen = new Set()
   const kinds = ENABLE_ZAPS ? [1, 7, 9735] : [1, 7]
@@ -223,7 +153,7 @@ const streamContext = ({notes, onChunk, maxDepth = 2}) => {
     const pubkeys = pluck("pubkey", events)
 
     // Load any people we should know about
-    loadPeople(pubkeys)
+    new PubkeyLoader(system).loadPubkeys(pubkeys)
 
     // Load data prior to now for our new ids
     chunk(256, newIds).forEach(ids => {
@@ -274,7 +204,6 @@ const applyContext = (notes, context) => {
 
 export default {
   Cursor,
-  loadPeople,
   streamContext,
   applyContext,
 }

src/app/shared/PersonList.svelte

@@ -6,7 +6,7 @@
   import Spinner from "src/partials/Spinner.svelte"
   import PersonInfo from "src/app/shared/PersonInfo.svelte"
   import {social, routing, user, network} from "src/app/system"
-  import legacyNetwork from "src/agent/network"
+  import {pubkeyLoader} from "src/app/state"
 
   export let type
   export let pubkey
@@ -24,7 +24,7 @@
         onEvent: batch(500, events => {
           const newPubkeys = pluck("pubkey", events)
 
-          legacyNetwork.loadPeople(newPubkeys)
+          pubkeyLoader.loadPubkeys(newPubkeys)
 
           pubkeys = uniq(pubkeys.concat(newPubkeys))
         }),

src/app/state.ts

@@ -11,7 +11,8 @@ import {hash, timedelta, now, batch, shuffle, sleep, clamp} from "src/util/misc"
 import {userKinds, noteKinds} from "src/util/nostr"
 import {findReplyId} from "src/util/nostr"
 import {modal, toast} from "src/partials/state"
-import {
+import {PubkeyLoader} from "src/system"
+import system, {
   FORCE_RELAYS,
   DEFAULT_FOLLOWS,
   ENABLE_ZAPS,
@@ -23,7 +24,6 @@ import {
   outbox,
   user,
 } from "src/app/system"
-import legacyNetwork from "src/agent/network"
 
 // Routing
 
@@ -86,6 +86,8 @@ export const logUsage = async name => {
   }
 }
 
+export const pubkeyLoader = new PubkeyLoader(system)
+
 // Synchronization from events to state
 
 export const listen = async () => {
@@ -119,8 +121,8 @@ export const listen = async () => {
       {kinds, "#e": eventIds, since},
       {kinds: [42], "#e": channelIds, since},
     ],
-    onEvent: batch(500, events => {
-      legacyNetwork.loadPeople(pluck("pubkey", events))
+    onEvent: batch(3000, events => {
+      pubkeyLoader.loadPubkeys(pluck("pubkey", events))
     }),
   })
 }
@@ -157,8 +159,8 @@ export const loadAppData = async pubkey => {
     listen()
 
     // Make sure the user and their network is loaded
-    await legacyNetwork.loadPeople([pubkey], {force: true, kinds: userKinds})
-    await legacyNetwork.loadPeople(user.getFollows())
+    await pubkeyLoader.loadPubkeys([pubkey], {force: true, kinds: userKinds})
+    await pubkeyLoader.loadPubkeys(user.getFollows())
   }
 }
 
@@ -175,7 +177,7 @@ export const login = async (method, key) => {
 
     await Promise.all([
       sleep(1500),
-      legacyNetwork.loadPeople([user.getPubkey()], {force: true, kinds: userKinds}),
+      pubkeyLoader.loadPubkeys([user.getPubkey()], {force: true, kinds: userKinds}),
     ])
 
     navigate("/notes")

src/app/views/LoginConnect.svelte

@@ -14,8 +14,7 @@
   import RelayCard from "src/app/shared/RelayCard.svelte"
   import {DEFAULT_RELAYS, FORCE_RELAYS, routing, user, network} from "src/app/system"
   import {watch} from "src/util/loki"
-  import legacyNetwork from "src/agent/network"
-  import {loadAppData} from "src/app/state"
+  import {loadAppData, pubkeyLoader} from "src/app/state"
 
   let modal = null
   let customRelayUrl = null
@@ -53,27 +52,30 @@
       attemptedRelays.add(relay.url)
       currentRelays[i] = relay
 
-      legacyNetwork
-        .loadPeople([user.getPubkey()], {relays: [relay.url], force: true, kinds: userKinds})
-        .then(async () => {
-          // Wait a bit before removing the relay to smooth out the ui
-          await sleep(1000)
+      // Wait a bit before removing the relay to smooth out the ui
+      await Promise.all([
+        sleep(1500),
+        pubkeyLoader.loadPubkeys([user.getPubkey()], {
+          force: true,
+          relays: [relay.url],
+          kinds: userKinds,
+        }),
+      ])
 
-          currentRelays[i] = null
+      currentRelays[i] = null
 
-          if (searching && user.getRelayUrls().length > 0) {
-            searching = false
-            modal = "success"
+      if (searching && user.getRelayUrls().length > 0) {
+        searching = false
+        modal = "success"
 
-            // Reload everything, it's possible we didn't get their petnames if we got a match
-            // from something like purplepag.es. This helps us avoid nuking follow lists later
-            await Promise.all([loadAppData(user.getPubkey()), sleep(1500)])
+        // Reload everything, it's possible we didn't get their petnames if we got a match
+        // from something like purplepag.es. This helps us avoid nuking follow lists later
+        await Promise.all([loadAppData(user.getPubkey()), sleep(1500)])
 
-            navigate("/notes")
-          } else {
-            network.pool.remove(relay.url)
-          }
-        })
+        navigate("/notes")
+      } else {
+        network.pool.remove(relay.url)
+      }
     }
 
     // Wait for our relay list to load initially, then terminate when we've tried everything

src/app/views/Onboarding.svelte

@@ -9,9 +9,8 @@
   import OnboardingRelays from "src/app/views/OnboardingRelays.svelte"
   import OnboardingFollows from "src/app/views/OnboardingFollows.svelte"
   import OnboardingNote from "src/app/views/OnboardingNote.svelte"
-  import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, routing, builder, user} from "src/app/system"
-  import network from "src/agent/network"
-  import {loadAppData} from "src/app/state"
+  import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, builder, user} from "src/app/system"
+  import {loadAppData, pubkeyLoader} from "src/app/state"
   import {modal} from "src/partials/state"
 
   export let stage
@@ -45,9 +44,7 @@
 
   onMount(() => {
     // Prime our database with some defaults
-    network.loadPeople(DEFAULT_FOLLOWS, {
-      relays: routing.getPubkeyHints(user.getPubkey(), "read"),
-    })
+    pubkeyLoader.loadPubkeys(DEFAULT_FOLLOWS)
   })
 </script>
 

src/system/index.ts

@@ -14,3 +14,4 @@ export {Builder} from "src/system/builder"
 export {Meta} from "src/system/meta"
 export {Crypt, Keys, User} from "src/system/user"
 export {System} from "src/system/system"
+export {PubkeyLoader} from "src/system/util/PubkeyLoader"

src/system/types.ts

@@ -4,6 +4,17 @@ export type Event = NostrToolsEvent & {
   seen_on: string[]
 }
 
+export type Filter = {
+  ids?: string[]
+  kinds?: number[]
+  authors?: string[]
+  since?: number
+  until?: number
+  limit?: number
+  search?: string
+  [key: `#${string}`]: string[]
+}
+
 export type RelayInfo = {
   contact?: string
   description?: string

src/system/util/PubkeyLoader.ts

@@ -0,0 +1,84 @@
+import {without, uniq} from "ramda"
+import {chunk} from "hurdak/lib/hurdak"
+import {personKinds, appDataKeys} from "src/util/nostr"
+import {now, timedelta} from "src/util/misc"
+import type {Filter} from "src/system/types"
+import type {System} from "src/system/system"
+
+export type LoadPeopleOpts = {
+  relays?: string[]
+  kinds?: number[]
+  force?: boolean
+}
+
+export class PubkeyLoader {
+  system: System
+  attemptedPubkeys: Set<string>
+
+  constructor(system) {
+    this.system = system
+    this.attemptedPubkeys = new Set()
+  }
+
+  getStalePubkeys = pubkeys => {
+    const stale = new Set()
+    const since = now() - timedelta(3, "hours")
+
+    for (const pubkey of pubkeys) {
+      if (stale.has(pubkey) || this.attemptedPubkeys.has(pubkey)) {
+        continue
+      }
+
+      this.attemptedPubkeys.add(pubkey)
+
+      const profile = this.system.directory.profiles.get(pubkey)
+
+      if (profile?.updated_at > since) {
+        continue
+      }
+
+      stale.add(pubkey)
+    }
+
+    return stale
+  }
+
+  loadPubkeys = async (rawPubkeys, {relays, force, kinds = personKinds}: LoadPeopleOpts = {}) => {
+    const {network, routing, user} = this.system
+    const pubkeys = force ? uniq(rawPubkeys) : this.getStalePubkeys(rawPubkeys)
+
+    const getChunkRelays = chunk => {
+      if (relays?.length > 0) {
+        return relays
+      }
+
+      return routing.mergeHints(
+        user.getSetting("relay_limit"),
+        chunk.map(pubkey => routing.getPubkeyHints(3, pubkey))
+      )
+    }
+
+    const getChunkFilter = chunk => {
+      const filter = [] as Filter[]
+
+      filter.push({kinds: without([30078], kinds), authors: chunk})
+
+      // Add a separate filter for app data so we're not pulling down other people's stuff,
+      // or obsolete events of our own.
+      if (kinds.includes(30078)) {
+        filter.push({kinds: [30078], authors: chunk, "#d": appDataKeys})
+      }
+
+      return filter
+    }
+
+    await Promise.all(
+      chunk(256, pubkeys).map(async chunk => {
+        await network.load({
+          relays: getChunkRelays(chunk),
+          filter: getChunkFilter(chunk),
+        })
+      })
+    )
+  }
+}