diff --git a/.ackrc b/.ackrc index b75c27aa..6b0ca8d1 100644 --- a/.ackrc +++ b/.ackrc @@ -1,5 +1,5 @@ --ignore-dir=node_modules --ignore-dir=android --ignore-dir=dist ---ignore-file=match:package-lock.json +--ignore-file=match:yarn.lock --ignore-file=match:.svg diff --git a/src/agent/network.ts b/src/agent/network.ts index d7196b9b..07ffb1fd 100644 --- a/src/agent/network.ts +++ b/src/agent/network.ts @@ -1,39 +1,9 @@ -import { - max, - without, - mergeLeft, - fromPairs, - sortBy, - assoc, - uniq, - uniqBy, - prop, - propEq, - groupBy, - pluck, -} from "ramda" -import {personKinds, appDataKeys, findReplyId} from "src/util/nostr" +import {max, mergeLeft, fromPairs, sortBy, assoc, uniqBy, prop, propEq, groupBy, pluck} from "ramda" +import {findReplyId} from "src/util/nostr" import {chunk, ensurePlural} from "hurdak/lib/hurdak" import {batch, now, timedelta} from "src/util/misc" -import {ENABLE_ZAPS, user, routing, directory, network} from "src/app/system" - -// If we ask for a pubkey and get nothing back, don't ask again this page load -const attemptedPubkeys = new Set() - -const getStalePubkeys = pubkeys => { - // If we're not reloading, only get pubkeys we don't already know about - return uniq(pubkeys).filter(pubkey => { - if (attemptedPubkeys.has(pubkey)) { - return false - } - - attemptedPubkeys.add(pubkey) - - const profile = directory.profiles.get(pubkey) - - return !profile || profile.updated_at < now() - timedelta(1, "days") - }) -} +import {PubkeyLoader} from "src/system" +import system, {ENABLE_ZAPS, user, routing, network} from "src/app/system" class Cursor { relays: string[] @@ -137,46 +107,6 @@ class Cursor { } } -const loadPeople = async ( - pubkeys, - { - relays = null, - kinds = personKinds, - force = false, - }: {relays?: string[]; kinds?: number[]; force?: boolean} = {} -) => { - pubkeys = uniq(pubkeys) - - // If we're not reloading, only get pubkeys we don't already know about - if (!force) { - pubkeys = getStalePubkeys(pubkeys) - } - - await Promise.all( - chunk(256, pubkeys).map(async chunk => { - const chunkRelays = - relays?.length > 0 - ? relays - : routing.mergeHints( - user.getSetting("relay_limit"), - chunk.map(pubkey => routing.getPubkeyHints(3, pubkey)) - ) - - const chunkFilter = [] as Array> - - chunkFilter.push({kinds: without([30078], kinds), authors: chunk}) - - // Add a separate filter for app data so we're not pulling down other people's stuff, - // or obsolete events of our own. - if (kinds.includes(30078)) { - chunkFilter.push({kinds: [30078], authors: chunk, "#d": appDataKeys}) - } - - await network.load({relays: chunkRelays, filter: chunkFilter}) - }) - ) -} - const streamContext = ({notes, onChunk, maxDepth = 2}) => { const seen = new Set() const kinds = ENABLE_ZAPS ? [1, 7, 9735] : [1, 7] @@ -223,7 +153,7 @@ const streamContext = ({notes, onChunk, maxDepth = 2}) => { const pubkeys = pluck("pubkey", events) // Load any people we should know about - loadPeople(pubkeys) + new PubkeyLoader(system).loadPubkeys(pubkeys) // Load data prior to now for our new ids chunk(256, newIds).forEach(ids => { @@ -274,7 +204,6 @@ const applyContext = (notes, context) => { export default { Cursor, - loadPeople, streamContext, applyContext, } diff --git a/src/app/shared/PersonList.svelte b/src/app/shared/PersonList.svelte index 381a563a..02d8818f 100644 --- a/src/app/shared/PersonList.svelte +++ b/src/app/shared/PersonList.svelte @@ -6,7 +6,7 @@ import Spinner from "src/partials/Spinner.svelte" import PersonInfo from "src/app/shared/PersonInfo.svelte" import {social, routing, user, network} from "src/app/system" - import legacyNetwork from "src/agent/network" + import {pubkeyLoader} from "src/app/state" export let type export let pubkey @@ -24,7 +24,7 @@ onEvent: batch(500, events => { const newPubkeys = pluck("pubkey", events) - legacyNetwork.loadPeople(newPubkeys) + pubkeyLoader.loadPubkeys(newPubkeys) pubkeys = uniq(pubkeys.concat(newPubkeys)) }), diff --git a/src/app/state.ts b/src/app/state.ts index 84b0666d..c78f8b7b 100644 --- a/src/app/state.ts +++ b/src/app/state.ts @@ -11,7 +11,8 @@ import {hash, timedelta, now, batch, shuffle, sleep, clamp} from "src/util/misc" import {userKinds, noteKinds} from "src/util/nostr" import {findReplyId} from "src/util/nostr" import {modal, toast} from "src/partials/state" -import { +import {PubkeyLoader} from "src/system" +import system, { FORCE_RELAYS, DEFAULT_FOLLOWS, ENABLE_ZAPS, @@ -23,7 +24,6 @@ import { outbox, user, } from "src/app/system" -import legacyNetwork from "src/agent/network" // Routing @@ -86,6 +86,8 @@ export const logUsage = async name => { } } +export const pubkeyLoader = new PubkeyLoader(system) + // Synchronization from events to state export const listen = async () => { @@ -119,8 +121,8 @@ export const listen = async () => { {kinds, "#e": eventIds, since}, {kinds: [42], "#e": channelIds, since}, ], - onEvent: batch(500, events => { - legacyNetwork.loadPeople(pluck("pubkey", events)) + onEvent: batch(3000, events => { + pubkeyLoader.loadPubkeys(pluck("pubkey", events)) }), }) } @@ -157,8 +159,8 @@ export const loadAppData = async pubkey => { listen() // Make sure the user and their network is loaded - await legacyNetwork.loadPeople([pubkey], {force: true, kinds: userKinds}) - await legacyNetwork.loadPeople(user.getFollows()) + await pubkeyLoader.loadPubkeys([pubkey], {force: true, kinds: userKinds}) + await pubkeyLoader.loadPubkeys(user.getFollows()) } } @@ -175,7 +177,7 @@ export const login = async (method, key) => { await Promise.all([ sleep(1500), - legacyNetwork.loadPeople([user.getPubkey()], {force: true, kinds: userKinds}), + pubkeyLoader.loadPubkeys([user.getPubkey()], {force: true, kinds: userKinds}), ]) navigate("/notes") diff --git a/src/app/views/LoginConnect.svelte b/src/app/views/LoginConnect.svelte index 655202e0..35e270db 100644 --- a/src/app/views/LoginConnect.svelte +++ b/src/app/views/LoginConnect.svelte @@ -14,8 +14,7 @@ import RelayCard from "src/app/shared/RelayCard.svelte" import {DEFAULT_RELAYS, FORCE_RELAYS, routing, user, network} from "src/app/system" import {watch} from "src/util/loki" - import legacyNetwork from "src/agent/network" - import {loadAppData} from "src/app/state" + import {loadAppData, pubkeyLoader} from "src/app/state" let modal = null let customRelayUrl = null @@ -53,27 +52,30 @@ attemptedRelays.add(relay.url) currentRelays[i] = relay - legacyNetwork - .loadPeople([user.getPubkey()], {relays: [relay.url], force: true, kinds: userKinds}) - .then(async () => { - // Wait a bit before removing the relay to smooth out the ui - await sleep(1000) + // Wait a bit before removing the relay to smooth out the ui + await Promise.all([ + sleep(1500), + pubkeyLoader.loadPubkeys([user.getPubkey()], { + force: true, + relays: [relay.url], + kinds: userKinds, + }), + ]) - currentRelays[i] = null + currentRelays[i] = null - if (searching && user.getRelayUrls().length > 0) { - searching = false - modal = "success" + if (searching && user.getRelayUrls().length > 0) { + searching = false + modal = "success" - // Reload everything, it's possible we didn't get their petnames if we got a match - // from something like purplepag.es. This helps us avoid nuking follow lists later - await Promise.all([loadAppData(user.getPubkey()), sleep(1500)]) + // Reload everything, it's possible we didn't get their petnames if we got a match + // from something like purplepag.es. This helps us avoid nuking follow lists later + await Promise.all([loadAppData(user.getPubkey()), sleep(1500)]) - navigate("/notes") - } else { - network.pool.remove(relay.url) - } - }) + navigate("/notes") + } else { + network.pool.remove(relay.url) + } } // Wait for our relay list to load initially, then terminate when we've tried everything diff --git a/src/app/views/Onboarding.svelte b/src/app/views/Onboarding.svelte index a3540de9..8b24e2bb 100644 --- a/src/app/views/Onboarding.svelte +++ b/src/app/views/Onboarding.svelte @@ -9,9 +9,8 @@ import OnboardingRelays from "src/app/views/OnboardingRelays.svelte" import OnboardingFollows from "src/app/views/OnboardingFollows.svelte" import OnboardingNote from "src/app/views/OnboardingNote.svelte" - import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, routing, builder, user} from "src/app/system" - import network from "src/agent/network" - import {loadAppData} from "src/app/state" + import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, builder, user} from "src/app/system" + import {loadAppData, pubkeyLoader} from "src/app/state" import {modal} from "src/partials/state" export let stage @@ -45,9 +44,7 @@ onMount(() => { // Prime our database with some defaults - network.loadPeople(DEFAULT_FOLLOWS, { - relays: routing.getPubkeyHints(user.getPubkey(), "read"), - }) + pubkeyLoader.loadPubkeys(DEFAULT_FOLLOWS) }) diff --git a/src/system/index.ts b/src/system/index.ts index 59c4e76b..59192ef5 100644 --- a/src/system/index.ts +++ b/src/system/index.ts @@ -14,3 +14,4 @@ export {Builder} from "src/system/builder" export {Meta} from "src/system/meta" export {Crypt, Keys, User} from "src/system/user" export {System} from "src/system/system" +export {PubkeyLoader} from "src/system/util/PubkeyLoader" diff --git a/src/system/types.ts b/src/system/types.ts index 73ffef9b..46aed305 100644 --- a/src/system/types.ts +++ b/src/system/types.ts @@ -4,6 +4,17 @@ export type Event = NostrToolsEvent & { seen_on: string[] } +export type Filter = { + ids?: string[] + kinds?: number[] + authors?: string[] + since?: number + until?: number + limit?: number + search?: string + [key: `#${string}`]: string[] +} + export type RelayInfo = { contact?: string description?: string diff --git a/src/system/util/PubkeyLoader.ts b/src/system/util/PubkeyLoader.ts new file mode 100644 index 00000000..d8399b78 --- /dev/null +++ b/src/system/util/PubkeyLoader.ts @@ -0,0 +1,84 @@ +import {without, uniq} from "ramda" +import {chunk} from "hurdak/lib/hurdak" +import {personKinds, appDataKeys} from "src/util/nostr" +import {now, timedelta} from "src/util/misc" +import type {Filter} from "src/system/types" +import type {System} from "src/system/system" + +export type LoadPeopleOpts = { + relays?: string[] + kinds?: number[] + force?: boolean +} + +export class PubkeyLoader { + system: System + attemptedPubkeys: Set + + constructor(system) { + this.system = system + this.attemptedPubkeys = new Set() + } + + getStalePubkeys = pubkeys => { + const stale = new Set() + const since = now() - timedelta(3, "hours") + + for (const pubkey of pubkeys) { + if (stale.has(pubkey) || this.attemptedPubkeys.has(pubkey)) { + continue + } + + this.attemptedPubkeys.add(pubkey) + + const profile = this.system.directory.profiles.get(pubkey) + + if (profile?.updated_at > since) { + continue + } + + stale.add(pubkey) + } + + return stale + } + + loadPubkeys = async (rawPubkeys, {relays, force, kinds = personKinds}: LoadPeopleOpts = {}) => { + const {network, routing, user} = this.system + const pubkeys = force ? uniq(rawPubkeys) : this.getStalePubkeys(rawPubkeys) + + const getChunkRelays = chunk => { + if (relays?.length > 0) { + return relays + } + + return routing.mergeHints( + user.getSetting("relay_limit"), + chunk.map(pubkey => routing.getPubkeyHints(3, pubkey)) + ) + } + + const getChunkFilter = chunk => { + const filter = [] as Filter[] + + filter.push({kinds: without([30078], kinds), authors: chunk}) + + // Add a separate filter for app data so we're not pulling down other people's stuff, + // or obsolete events of our own. + if (kinds.includes(30078)) { + filter.push({kinds: [30078], authors: chunk, "#d": appDataKeys}) + } + + return filter + } + + await Promise.all( + chunk(256, pubkeys).map(async chunk => { + await network.load({ + relays: getChunkRelays(chunk), + filter: getChunkFilter(chunk), + }) + }) + ) + } +}