Re-work load pubkeys

src/app/state.ts

@@ -1,7 +1,6 @@
 import Bugsnag from "@bugsnag/js"
 import {writable} from "@welshman/lib"
 import {Scope, makeScopeFeed} from "@welshman/feeds"
-import {userKinds} from "src/util/nostr"
 import {router} from "src/app/util/router"
 import type {Feed} from "src/domain"
 import {makeFeed} from "src/domain"
@@ -14,7 +13,7 @@ import {
   loadLabels,
   loadDeletes,
   loadHandlers,
-  loadPubkeys,
+  loadPubkeyUserData,
   loadGiftWrap,
   loadAllMessages,
   loadGroupMessages,
@@ -44,7 +43,7 @@ const redactErrorInfo = (info: any) =>
   JSON.parse(
     JSON.stringify(info || null)
       .replace(/\d+:{60}\w+:\w+/g, "[REDACTED]")
-      .replace(/\w{60}\w+/g, "[REDACTED]")
+      .replace(/\w{60}\w+/g, "[REDACTED]"),
   )
 
 // Wait for bugsnag to be started in main
@@ -93,10 +92,7 @@ export const loadAppData = () => {
 
 export const loadUserData = async () => {
   // Make sure the user and their follows are loaded
-  await loadPubkeys([pubkey.get()], {
-    force: true,
-    kinds: userKinds,
-  })
+  await loadPubkeyUserData([pubkey.get()])
 
   loadSeen()
   loadLabels()

src/app/views/Bech32Entity.svelte

@@ -1,5 +1,4 @@
 <script lang="ts">
-  import {nip19} from "nostr-tools"
   import {Address} from "@welshman/util"
   import Content from "src/partials/Content.svelte"
   import NoteDetail from "src/app/views/NoteDetail.svelte"
@@ -27,9 +26,9 @@
 {:else if type === "nrelay"}
   <RelayDetail url={data} />
 {:else if type === "nprofile"}
-  <PersonDetail npub={nip19.npubEncode(data.pubkey)} pubkey={data.pubkey} {relays} />
+  <PersonDetail pubkey={data.pubkey} {relays} />
 {:else if type === "npub"}
-  <PersonDetail npub={nip19.npubEncode(data)} pubkey={data} />
+  <PersonDetail pubkey={data} />
 {:else}
   <Content size="lg" class="text-center">
     <div>Sorry, we weren't able to find "{entity}".</div>

src/app/views/LoginConnect.svelte

@@ -1,7 +1,13 @@
 <script lang="ts">
   import {sleep} from "hurdak"
-  import {LOCAL_RELAY_URL, normalizeRelayUrl, isShareableRelayUrl} from "@welshman/util"
-  import {userKinds} from "src/util/nostr"
+  import {
+    RELAYS,
+    FOLLOWS,
+    PROFILE,
+    LOCAL_RELAY_URL,
+    normalizeRelayUrl,
+    isShareableRelayUrl,
+  } from "@welshman/util"
   import {showWarning} from "src/partials/Toast.svelte"
   import Modal from "src/partials/Modal.svelte"
   import Field from "src/partials/Field.svelte"
@@ -12,16 +18,20 @@
   import Anchor from "src/partials/Anchor.svelte"
   import {router} from "src/app/util/router"
   import {loadUserData} from "src/app/state"
-  import {env, loadPubkeys, session} from "src/engine"
+  import {env, loadPubkeyUserData, deriveEvents, session} from "src/engine"
 
   const t = Date.now()
 
+  const events = deriveEvents({
+    filters: [{kinds: [RELAYS, FOLLOWS, PROFILE], authors: [$session.pubkey]}],
+  })
+
   const skip = () => router.at("notes").push()
 
   const searchRelays = async relays => {
     failed = false
 
-    await loadPubkeys([$session.pubkey], {force: true, kinds: userKinds})
+    await loadPubkeyUserData([$session.pubkey], {relays})
 
     if (!found) {
       failed = true
@@ -59,11 +69,10 @@
   tryDefaultRelays()
 
   $: {
-    if (!found && $session.kind0 && ($session.kind3 || $session.kind10002)) {
+    if (!found && $events.length === 3) {
       found = true
 
-      // Reload everything, it's possible we didn't get their petnames if we got a match
-      // from something like purplepag.es. This helps us avoid nuking follow lists later
+      // Reload user data and pull in messages, notifications, etc
       loadUserData()
 
       // Show a success message once they've had time to read the intro message
@@ -117,7 +126,7 @@
       <Field label="Relay">
         <Input bind:value={customRelay} />
       </Field>
-      <div class="flex justify-center gap-2">
+      <div class="flex justify-between gap-2">
         <Anchor button on:click={closeModal}>Cancel</Anchor>
         <Anchor button accent on:click={confirmCustomRelay}>Search relay</Anchor>
       </div>

src/engine/commands.ts

@@ -9,8 +9,11 @@ import {
   Address,
   getIdAndAddress,
   isShareableRelayUrl,
+  isSignedEvent,
   normalizeRelayUrl,
   FOLLOWS,
+  RELAYS,
+  PROFILE,
 } from "@welshman/util"
 import {Fetch, chunk, createMapOf, randomId, seconds, sleep, tryFunc} from "hurdak"
 import {
@@ -270,9 +273,7 @@ export const joinRelay = async (url: string, claim?: string) => {
   }
 
   // Re-publish user meta to the new relay
-  if (canSign.get() && session.get().kind3) {
-    publish({event: session.get().kind3, relays: [url]})
-  }
+  broadcastUserData([url])
 
   return publishRelays([
     ...reject(whereEq({url}), relayPolicies.get()),
@@ -998,17 +999,13 @@ export const updateCurrentSession = f => {
 }
 
 export const broadcastUserData = (relays: string[]) => {
-  const {kind0, kind3, kind10002} = session.get() || {}
+  const authors = [pubkey.get()]
+  const kinds = [RELAYS, FOLLOWS, PROFILE]
+  const events = repository.query([{kinds, authors}])
 
-  if (kind0) {
-    publish({event: kind0, relays})
-  }
-
-  if (kind3) {
-    publish({event: kind3, relays})
-  }
-
-  if (kind10002) {
-    publish({event: kind10002, relays})
+  for (const event of events) {
+    if (isSignedEvent(event)) {
+      publish({event, relays})
+    }
   }
 }

src/engine/model.ts

@@ -1,5 +1,5 @@
 import type {Publish} from "@welshman/net"
-import type {SignedEvent, TrustedEvent, Zapper} from "@welshman/util"
+import type {TrustedEvent, Zapper} from "@welshman/util"
 
 export type RelayInfo = {
   contact?: string
@@ -155,12 +155,6 @@ export type Session = {
   connectKey?: string
   connectToken?: string
   connectHandler?: NostrConnectHandler
-  kind0?: SignedEvent
-  kind0_updated?: string
-  kind3?: SignedEvent
-  kind3_updated?: string
-  kind10002?: SignedEvent
-  kind10002_updated?: string
   settings?: Record<string, any>
   settings_updated_at?: number
   groups_last_synced?: number

src/engine/projections.ts

@@ -354,12 +354,6 @@ projections.addHandler(0, e => {
 })
 
 projections.addHandler(3, e => {
-  const session = getSession(e.pubkey)
-
-  if (session) {
-    updateSession(e.pubkey, $session => updateRecord($session, e.created_at, {kind3: e}))
-  }
-
   updateStore(people.key(e.pubkey), e.created_at, {
     petnames: uniqBy(nth(1), Tags.fromEvent(e).whereKey("p").unwrap()),
   })
@@ -373,14 +367,6 @@ projections.addHandler(10000, e => {
   })
 })
 
-projections.addHandler(10002, e => {
-  const session = getSession(e.pubkey)
-
-  if (session) {
-    updateSession(e.pubkey, $session => updateRecord($session, e.created_at, {kind10002: e}))
-  }
-})
-
 projections.addHandler(10004, e => {
   updateStore(people.key(e.pubkey), e.created_at, {
     communities: Tags.fromEvent(e).whereKey("a").unwrap(),

src/engine/requests/pubkeys.ts

@@ -1,12 +1,20 @@
-import {seconds} from 'hurdak'
-import {assoc, remove, now, inc} from '@welshman/lib'
-import type {Filter} from "@welshman/util"
-import {appDataKeys, personKinds} from "src/util/nostr"
-import {people, load, hints} from 'src/engine/state'
+import {seconds} from "hurdak"
+import {assoc, remove, now, inc} from "@welshman/lib"
+import {RELAYS, APP_DATA} from "@welshman/util"
+import {appDataKeys, personKinds, userKinds} from "src/util/nostr"
+import {freshness, withIndexers, load, hints} from "src/engine/state"
 
 const attempts = new Map<string, number>()
 
-export const getValidPubkeys = (pubkeys: string[], key: string, force = false) => {
+const getFreshnessKey = (key: string, pubkey: string) => `loadPubkeys:${key}:${pubkey}`
+
+const getFreshness = (key: string, pubkey: string) =>
+  freshness.get()[getFreshnessKey(key, pubkey)] || 0
+
+const setFreshness = (key: string, pubkey: string, ts: number) =>
+  freshness.update(assoc(getFreshnessKey(key, pubkey), ts))
+
+const getStalePubkeys = (pubkeys: string[], key: string, delta: number) => {
   const result = new Set<string>()
 
   for (const pubkey of pubkeys) {
@@ -14,111 +22,72 @@ export const getValidPubkeys = (pubkeys: string[], key: string, force = false) =
       continue
     }
 
-    const person = people.key(pubkey)
-    const $person = person.get()
-    const tskey = `${key}_fetched_at`
-    const ts = $person?.[tskey]
+    // If we've tried a few times, slow down the duplicate requests
+    const thisAttempts = inc(attempts.get(pubkey))
+    const thisDelta = delta * thisAttempts
 
-    if (!force) {
-      // Avoid multiple concurrent requests
-      if (ts > now() - 5) {
-        continue
-      }
-
-      // If we've tried a few times, slow down with duplicate requests
-      if (attempts.get(pubkey) > 3 && ts > now() - seconds(5, "minute")) {
-        continue
-      }
-
-      // If we have something to show the user, and we checked recently, leave it alone
-      if ($person?.[key] && ts > now() - seconds(1, "hour")) {
-        continue
-      }
+    if (getFreshness(key, pubkey) < now() - thisDelta) {
+      attempts.set(pubkey, thisAttempts)
+      result.add(pubkey)
     }
-
-    attempts.set(pubkey, inc(attempts.get(pubkey)))
-    person.merge({[tskey]: now()})
-    result.add(pubkey)
   }
 
   return Array.from(result)
 }
 
-export type LoadPubkeyOpts = {
+type LoadPubkeyOpts = {
   force?: boolean
-  kinds?: number[]
   relays?: string[]
 }
 
-export const loadPubkeyProfiles = (rawPubkeys: string[], opts: LoadPubkeyOpts = {}) => {
-  const promises = []
-  const filters = [] as Filter[]
-  const kinds = remove(10002, opts.kinds || personKinds)
-  const pubkeys = getValidPubkeys(rawPubkeys, "profile", opts.force)
+const loadPubkeyData = (
+  key: string,
+  kinds: number[],
+  rawPubkeys: string[],
+  {force = false, relays = []}: LoadPubkeyOpts = {},
+) => {
+  const delta = force ? 5 : seconds(5, "minute")
+  const pubkeys = getStalePubkeys(rawPubkeys, key, delta)
 
   if (pubkeys.length === 0) {
-    return
+    return Promise.resolve([])
   }
 
-  filters.push({kinds: remove(30078, kinds)})
-
   // Add a separate filters for app data so we're not pulling down other people's stuff,
   // or obsolete events of our own.
-  if (kinds.includes(30078)) {
-    filters.push({kinds: [30078], "#d": Object.values(appDataKeys)})
-  }
+  const filters = kinds.includes(APP_DATA)
+    ? [{kinds: [APP_DATA], "#d": Object.values(appDataKeys)}, {kinds: remove(APP_DATA, kinds)}]
+    : [{kinds}]
 
-  promises.push(
-    load({
-      skipCache: true,
-      relays: hints.Indexers(opts.relays || []).getUrls(),
-      filters: filters.map(assoc("authors", pubkeys)),
-    }),
+  return Promise.all(
+    hints
+      .FromPubkeys(pubkeys)
+      .getSelections()
+      .map(({relay, values}) =>
+        load({
+          skipCache: true,
+          relays: withIndexers([relay]),
+          filters: filters.map(assoc("authors", values)),
+          onEvent: e => setFreshness(key, e.pubkey, now()),
+        }),
+      ),
   )
-
-  for (const {relay, values} of hints.FromPubkeys(pubkeys).getSelections()) {
-    promises.push(
-      load({
-        skipCache: true,
-        relays: [relay],
-        filters: filters.map(assoc("authors", values)),
-      }),
-    )
-  }
-
-  return Promise.all(promises)
 }
 
-export const loadPubkeyRelays = (rawPubkeys: string[], opts: LoadPubkeyOpts = {}) => {
-  const promises = []
-  const pubkeys = getValidPubkeys(rawPubkeys, "relays", opts.force)
+export const loadPubkeyRelays = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
+  loadPubkeyData("relay", [RELAYS], pubkeys, opts)
 
-  if (pubkeys.length === 0) {
-    return
-  }
-
-  promises.push(
-    load({
-      skipCache: true,
-      filters: [{kinds: [10002], authors: pubkeys}],
-      relays: hints.Indexers(opts.relays || []).getUrls(),
-      onEvent: e => loadPubkeyProfiles([e.pubkey]),
-    }),
-  )
-
-  for (const {relay, values} of hints.FromPubkeys(pubkeys).getSelections()) {
-    promises.push(
-      load({
-        skipCache: true,
-        relays: [relay],
-        filters: [{kinds: [10002], authors: values}],
-        onEvent: e => loadPubkeyProfiles([e.pubkey]),
-      }),
-    )
-  }
-
-  return Promise.all(promises)
-}
+export const loadPubkeyProfiles = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
+  loadPubkeyData("profile", remove(RELAYS, personKinds), pubkeys, opts)
 
 export const loadPubkeys = async (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
-  Promise.all([loadPubkeyRelays(pubkeys, opts), loadPubkeyProfiles(pubkeys, opts)])
+  // Load relays, then load profiles so we have a better chance of finding them. But also
+  // load profiles concurrently so that if we do find them it takes as little time as possible.
+  // Requests will be deduplicated by tracking freshness and within welshman
+  Promise.all([
+    loadPubkeyRelays(pubkeys, opts).then(() => loadPubkeyProfiles(pubkeys, opts)),
+    loadPubkeyProfiles(pubkeys, opts),
+  ])
+
+export const loadPubkeyUserData = (pubkeys: string[], opts: LoadPubkeyOpts = {}) =>
+  loadPubkeyData("user", userKinds, pubkeys, {force: true, ...opts})

src/engine/state.ts

@@ -183,6 +183,7 @@ export const env = new Writable({
 
 export const pubkey = withGetter(synced<string | null>("pubkey", null))
 export const sessions = withGetter(synced<Record<string, Session>>("sessions", {}))
+export const freshness = withGetter(synced<Record<string, number>>("freshness", {}))
 
 export const relays = new CollectionStore<Relay>("url")
 export const groups = new CollectionStore<Group>("address")
@@ -1042,7 +1043,6 @@ export const hints = new Router({
   getCommunityRelays: getGroupRelayUrls,
   getPubkeyRelays: getPubkeyRelayUrls,
   getFallbackRelays: () => [...env.get().PLATFORM_RELAYS, ...env.get().DEFAULT_RELAYS],
-  getIndexerRelays: () => env.get().INDEXER_RELAYS,
   getSearchRelays: () => env.get().SEARCH_RELAYS,
   getLimit: () => parseInt(getSetting("relay_limit")),
   getRedundancy: () => parseInt(getSetting("relay_redundancy")),

src/util/nostr.ts

@@ -1,6 +1,6 @@
 import {
   fromNostrURI,
-  APPLICATION,
+  APP_DATA,
   AUDIO,
   CLASSIFIED,
   EVENT_TIME,
@@ -61,7 +61,7 @@ export const personKinds = [
   FEED,
   PROFILE,
 ] as number[]
-export const userKinds = [...personKinds, APPLICATION] as number[]
+export const userKinds = [...personKinds, APP_DATA] as number[]
 
 export const appDataKeys = {
   USER_SETTINGS: "nostr-engine/User/settings/v1",