throttle pubkey loading more

src/app/App.svelte

@@ -468,21 +468,23 @@
         .filter(r => (r.last_checked || 0) < lib.now() - seconds(7, "day"))
         .slice(0, 20)
 
-      misc.tryFetch(async () => {
-        const result = await Fetch.fetchJson(dufflepud("relay/info"), {
-          method: "POST",
-          body: JSON.stringify({urls: pluck("url", staleRelays)}),
-          headers: {
-            "Content-Type": "application/json",
-          },
+      if (staleRelays.length > 0) {
+        misc.tryFetch(async () => {
+          const result = await Fetch.fetchJson(dufflepud("relay/info"), {
+            method: "POST",
+            body: JSON.stringify({urls: pluck("url", staleRelays)}),
+            headers: {
+              "Content-Type": "application/json",
+            },
+          })
+
+          for (const {url: rawUrl, info} of result.data) {
+            const url = util.normalizeRelayUrl(rawUrl)
+
+            relays.key(url).merge({...info, url, last_checked: lib.now()})
+          }
         })
-
-        for (const {url: rawUrl, info} of result.data) {
-          const url = util.normalizeRelayUrl(rawUrl)
-
-          relays.key(url).merge({...info, url, last_checked: lib.now()})
-        }
-      })
+      }
     }, 30_000)
 
     return () => {

src/engine/projections.ts

@@ -263,11 +263,9 @@ projections.addHandler(RELAYS, (e: TrustedEvent) => {
     if (["r", "relay"].includes(key) && isShareableRelayUrl(value)) {
       relays.key(normalizeRelayUrl(value)).update($relay => ({
         url: value,
+        last_checked: 0,
         count: inc($relay?.count || 0),
         first_seen: $relay?.first_seen || e.created_at,
-        info: {
-          last_checked: 0,
-        },
       }))
     }
   }

src/engine/requests/pubkeys.ts

@@ -1,5 +1,5 @@
 import {seconds} from "hurdak"
-import {assoc, remove, now, inc} from "@welshman/lib"
+import {batch, addToMapKey, now, inc} from "@welshman/lib"
 import {
   RELAYS,
   PROFILE,
@@ -9,20 +9,23 @@ import {
   FOLLOWS,
   APP_DATA,
 } from "@welshman/util"
-import {appDataKeys} from "src/util/nostr"
+import {isHex, appDataKeys} from "src/util/nostr"
 import {LIST_KINDS} from "src/domain"
 import {getFreshness, setFreshness, withIndexers, load, hints} from "src/engine/state"
 
 const attempts = new Map<string, number>()
 
 const getStalePubkeys = (pubkeys: string[], key: string, delta: number) => {
+  const seen = new Set<string>()
   const result = new Set<string>()
 
   for (const pubkey of pubkeys) {
-    if (!pubkey?.match(/^[0-f]{64}$/)) {
+    if (!isHex(pubkey) || seen.has(pubkey)) {
       continue
     }
 
+    seen.add(pubkey)
+
     // If we've tried a few times, slow down the duplicate requests
     const thisAttempts = inc(attempts.get(pubkey))
     const thisDelta = delta * thisAttempts
@@ -37,6 +40,44 @@ const getStalePubkeys = (pubkeys: string[], key: string, delta: number) => {
   return Array.from(result)
 }
 
+const getFiltersForKey = (key: string, authors: string[]) => {
+  switch (key) {
+    case "pubkey/lists":
+      return [{authors, kinds: LIST_KINDS}]
+    case "pubkey/feeds":
+      return [{authors, kinds: [NAMED_BOOKMARKS, FEED]}]
+    case "pubkey/relays":
+      return [{authors, kinds: [RELAYS]}]
+    case "pubkey/profile":
+      return [{authors, kinds: [PROFILE, FOLLOWS, HANDLER_INFORMATION]}]
+    case "pubkey/user":
+      return [
+        {authors, kinds: [PROFILE, RELAYS, FOLLOWS, APP_DATA]},
+        {authors, kinds: [APP_DATA], "#d": Object.values(appDataKeys)},
+      ]
+  }
+}
+
+const loadPubkeysThrottled = batch(300, (requests: {key: string; pubkeys: string[]}[]) => {
+  const pubkeysByKey = new Map<string, Set<string>>()
+
+  for (const {key, pubkeys} of requests) {
+    for (const pubkey of pubkeys) {
+      addToMapKey(pubkeysByKey, key, pubkey)
+    }
+  }
+
+  for (const [key, pubkeys] of pubkeysByKey.entries()) {
+    const authors = Array.from(pubkeys)
+
+    load({
+      skipCache: true,
+      filters: getFiltersForKey(key, authors),
+      relays: withIndexers(hints.FromPubkeys(authors).getUrls()),
+    })
+  }
+})
+
 type LoadPubkeyOpts = {
   force?: boolean
   relays?: string[]
@@ -44,60 +85,33 @@ type LoadPubkeyOpts = {
 
 const loadPubkeyData = (
   key: string,
-  kinds: number[],
   rawPubkeys: string[],
   {force = false, relays = []}: LoadPubkeyOpts = {},
 ) => {
   const delta = force ? 5 : seconds(15, "minute")
   const pubkeys = getStalePubkeys(rawPubkeys, key, delta)
 
-  if (pubkeys.length === 0) {
-    return Promise.resolve([])
+  if (pubkeys.length > 0) {
+    loadPubkeysThrottled({key, pubkeys})
   }
-
-  // Add a separate filters for app data so we're not pulling down other people's stuff,
-  // or obsolete events of our own.
-  const filters = kinds.includes(APP_DATA)
-    ? [{kinds: [APP_DATA], "#d": Object.values(appDataKeys)}, {kinds: remove(APP_DATA, kinds)}]
-    : [{kinds}]
-
-  return Promise.all(
-    hints
-      .FromPubkeys(pubkeys)
-      .getSelections()
-      .map(({relay, values}) =>
-        load({
-          skipCache: true,
-          relays: withIndexers([relay]),
-          filters: filters.map(assoc("authors", values)),
-        }),
-      ),
-  )
 }
 
 export const loadPubkeyLists = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  loadPubkeyData("pubkey/lists", LIST_KINDS, pubkeys, opts)
+  loadPubkeyData("pubkey/lists", pubkeys, opts)
 
 export const loadPubkeyFeeds = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  loadPubkeyData("pubkey/feeds", [NAMED_BOOKMARKS, FEED], pubkeys, opts)
+  loadPubkeyData("pubkey/feeds", pubkeys, opts)
 
 export const loadPubkeyRelays = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  loadPubkeyData("pubkey/relays", [RELAYS], pubkeys, opts)
+  loadPubkeyData("pubkey/relays", pubkeys, opts)
 
 export const loadPubkeyProfiles = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  loadPubkeyData("pubkey/profile", [PROFILE, FOLLOWS, HANDLER_INFORMATION], pubkeys, opts)
+  loadPubkeyData("pubkey/profile", pubkeys, opts)
 
-export const loadPubkeys = async (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  // Load relays, then load profiles so we have a better chance of finding them. But also
-  // load profiles concurrently so that if we do find them it takes as little time as possible.
-  // Requests will be deduplicated by tracking freshness and within welshman
-  Promise.all([
-    loadPubkeyRelays(pubkeys, opts).then(() => loadPubkeyProfiles(pubkeys, opts)),
-    loadPubkeyProfiles(pubkeys, opts),
-  ])
+export const loadPubkeys = (pubkeys: string[], opts: LoadPubkeyOpts = {}) => {
+  loadPubkeyRelays(pubkeys, opts)
+  loadPubkeyProfiles(pubkeys, opts)
+}
 
 export const loadPubkeyUserData = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  loadPubkeyData("pubkey/user", [PROFILE, RELAYS, FOLLOWS, APP_DATA], pubkeys, {
-    force: true,
-    ...opts,
-  })
+  loadPubkeyData("pubkey/user", pubkeys, {force: true, ...opts})