mirror of
https://github.com/coracle-social/coracle.git
synced 2024-09-30 00:41:12 +00:00
Move loadPeople to PubkeyLoader
This commit is contained in:
parent
6d0c7c796f
commit
8428c179c6
2
.ackrc
2
.ackrc
@ -1,5 +1,5 @@
|
|||||||
--ignore-dir=node_modules
|
--ignore-dir=node_modules
|
||||||
--ignore-dir=android
|
--ignore-dir=android
|
||||||
--ignore-dir=dist
|
--ignore-dir=dist
|
||||||
--ignore-file=match:package-lock.json
|
--ignore-file=match:yarn.lock
|
||||||
--ignore-file=match:.svg
|
--ignore-file=match:.svg
|
||||||
|
@ -1,39 +1,9 @@
|
|||||||
import {
|
import {max, mergeLeft, fromPairs, sortBy, assoc, uniqBy, prop, propEq, groupBy, pluck} from "ramda"
|
||||||
max,
|
import {findReplyId} from "src/util/nostr"
|
||||||
without,
|
|
||||||
mergeLeft,
|
|
||||||
fromPairs,
|
|
||||||
sortBy,
|
|
||||||
assoc,
|
|
||||||
uniq,
|
|
||||||
uniqBy,
|
|
||||||
prop,
|
|
||||||
propEq,
|
|
||||||
groupBy,
|
|
||||||
pluck,
|
|
||||||
} from "ramda"
|
|
||||||
import {personKinds, appDataKeys, findReplyId} from "src/util/nostr"
|
|
||||||
import {chunk, ensurePlural} from "hurdak/lib/hurdak"
|
import {chunk, ensurePlural} from "hurdak/lib/hurdak"
|
||||||
import {batch, now, timedelta} from "src/util/misc"
|
import {batch, now, timedelta} from "src/util/misc"
|
||||||
import {ENABLE_ZAPS, user, routing, directory, network} from "src/app/system"
|
import {PubkeyLoader} from "src/system"
|
||||||
|
import system, {ENABLE_ZAPS, user, routing, network} from "src/app/system"
|
||||||
// If we ask for a pubkey and get nothing back, don't ask again this page load
|
|
||||||
const attemptedPubkeys = new Set()
|
|
||||||
|
|
||||||
const getStalePubkeys = pubkeys => {
|
|
||||||
// If we're not reloading, only get pubkeys we don't already know about
|
|
||||||
return uniq(pubkeys).filter(pubkey => {
|
|
||||||
if (attemptedPubkeys.has(pubkey)) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
attemptedPubkeys.add(pubkey)
|
|
||||||
|
|
||||||
const profile = directory.profiles.get(pubkey)
|
|
||||||
|
|
||||||
return !profile || profile.updated_at < now() - timedelta(1, "days")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
class Cursor {
|
class Cursor {
|
||||||
relays: string[]
|
relays: string[]
|
||||||
@ -137,46 +107,6 @@ class Cursor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const loadPeople = async (
|
|
||||||
pubkeys,
|
|
||||||
{
|
|
||||||
relays = null,
|
|
||||||
kinds = personKinds,
|
|
||||||
force = false,
|
|
||||||
}: {relays?: string[]; kinds?: number[]; force?: boolean} = {}
|
|
||||||
) => {
|
|
||||||
pubkeys = uniq(pubkeys)
|
|
||||||
|
|
||||||
// If we're not reloading, only get pubkeys we don't already know about
|
|
||||||
if (!force) {
|
|
||||||
pubkeys = getStalePubkeys(pubkeys)
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(
|
|
||||||
chunk(256, pubkeys).map(async chunk => {
|
|
||||||
const chunkRelays =
|
|
||||||
relays?.length > 0
|
|
||||||
? relays
|
|
||||||
: routing.mergeHints(
|
|
||||||
user.getSetting("relay_limit"),
|
|
||||||
chunk.map(pubkey => routing.getPubkeyHints(3, pubkey))
|
|
||||||
)
|
|
||||||
|
|
||||||
const chunkFilter = [] as Array<Record<string, any>>
|
|
||||||
|
|
||||||
chunkFilter.push({kinds: without([30078], kinds), authors: chunk})
|
|
||||||
|
|
||||||
// Add a separate filter for app data so we're not pulling down other people's stuff,
|
|
||||||
// or obsolete events of our own.
|
|
||||||
if (kinds.includes(30078)) {
|
|
||||||
chunkFilter.push({kinds: [30078], authors: chunk, "#d": appDataKeys})
|
|
||||||
}
|
|
||||||
|
|
||||||
await network.load({relays: chunkRelays, filter: chunkFilter})
|
|
||||||
})
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const streamContext = ({notes, onChunk, maxDepth = 2}) => {
|
const streamContext = ({notes, onChunk, maxDepth = 2}) => {
|
||||||
const seen = new Set()
|
const seen = new Set()
|
||||||
const kinds = ENABLE_ZAPS ? [1, 7, 9735] : [1, 7]
|
const kinds = ENABLE_ZAPS ? [1, 7, 9735] : [1, 7]
|
||||||
@ -223,7 +153,7 @@ const streamContext = ({notes, onChunk, maxDepth = 2}) => {
|
|||||||
const pubkeys = pluck("pubkey", events)
|
const pubkeys = pluck("pubkey", events)
|
||||||
|
|
||||||
// Load any people we should know about
|
// Load any people we should know about
|
||||||
loadPeople(pubkeys)
|
new PubkeyLoader(system).loadPubkeys(pubkeys)
|
||||||
|
|
||||||
// Load data prior to now for our new ids
|
// Load data prior to now for our new ids
|
||||||
chunk(256, newIds).forEach(ids => {
|
chunk(256, newIds).forEach(ids => {
|
||||||
@ -274,7 +204,6 @@ const applyContext = (notes, context) => {
|
|||||||
|
|
||||||
export default {
|
export default {
|
||||||
Cursor,
|
Cursor,
|
||||||
loadPeople,
|
|
||||||
streamContext,
|
streamContext,
|
||||||
applyContext,
|
applyContext,
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,7 @@
|
|||||||
import Spinner from "src/partials/Spinner.svelte"
|
import Spinner from "src/partials/Spinner.svelte"
|
||||||
import PersonInfo from "src/app/shared/PersonInfo.svelte"
|
import PersonInfo from "src/app/shared/PersonInfo.svelte"
|
||||||
import {social, routing, user, network} from "src/app/system"
|
import {social, routing, user, network} from "src/app/system"
|
||||||
import legacyNetwork from "src/agent/network"
|
import {pubkeyLoader} from "src/app/state"
|
||||||
|
|
||||||
export let type
|
export let type
|
||||||
export let pubkey
|
export let pubkey
|
||||||
@ -24,7 +24,7 @@
|
|||||||
onEvent: batch(500, events => {
|
onEvent: batch(500, events => {
|
||||||
const newPubkeys = pluck("pubkey", events)
|
const newPubkeys = pluck("pubkey", events)
|
||||||
|
|
||||||
legacyNetwork.loadPeople(newPubkeys)
|
pubkeyLoader.loadPubkeys(newPubkeys)
|
||||||
|
|
||||||
pubkeys = uniq(pubkeys.concat(newPubkeys))
|
pubkeys = uniq(pubkeys.concat(newPubkeys))
|
||||||
}),
|
}),
|
||||||
|
@ -11,7 +11,8 @@ import {hash, timedelta, now, batch, shuffle, sleep, clamp} from "src/util/misc"
|
|||||||
import {userKinds, noteKinds} from "src/util/nostr"
|
import {userKinds, noteKinds} from "src/util/nostr"
|
||||||
import {findReplyId} from "src/util/nostr"
|
import {findReplyId} from "src/util/nostr"
|
||||||
import {modal, toast} from "src/partials/state"
|
import {modal, toast} from "src/partials/state"
|
||||||
import {
|
import {PubkeyLoader} from "src/system"
|
||||||
|
import system, {
|
||||||
FORCE_RELAYS,
|
FORCE_RELAYS,
|
||||||
DEFAULT_FOLLOWS,
|
DEFAULT_FOLLOWS,
|
||||||
ENABLE_ZAPS,
|
ENABLE_ZAPS,
|
||||||
@ -23,7 +24,6 @@ import {
|
|||||||
outbox,
|
outbox,
|
||||||
user,
|
user,
|
||||||
} from "src/app/system"
|
} from "src/app/system"
|
||||||
import legacyNetwork from "src/agent/network"
|
|
||||||
|
|
||||||
// Routing
|
// Routing
|
||||||
|
|
||||||
@ -86,6 +86,8 @@ export const logUsage = async name => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const pubkeyLoader = new PubkeyLoader(system)
|
||||||
|
|
||||||
// Synchronization from events to state
|
// Synchronization from events to state
|
||||||
|
|
||||||
export const listen = async () => {
|
export const listen = async () => {
|
||||||
@ -119,8 +121,8 @@ export const listen = async () => {
|
|||||||
{kinds, "#e": eventIds, since},
|
{kinds, "#e": eventIds, since},
|
||||||
{kinds: [42], "#e": channelIds, since},
|
{kinds: [42], "#e": channelIds, since},
|
||||||
],
|
],
|
||||||
onEvent: batch(500, events => {
|
onEvent: batch(3000, events => {
|
||||||
legacyNetwork.loadPeople(pluck("pubkey", events))
|
pubkeyLoader.loadPubkeys(pluck("pubkey", events))
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -157,8 +159,8 @@ export const loadAppData = async pubkey => {
|
|||||||
listen()
|
listen()
|
||||||
|
|
||||||
// Make sure the user and their network is loaded
|
// Make sure the user and their network is loaded
|
||||||
await legacyNetwork.loadPeople([pubkey], {force: true, kinds: userKinds})
|
await pubkeyLoader.loadPubkeys([pubkey], {force: true, kinds: userKinds})
|
||||||
await legacyNetwork.loadPeople(user.getFollows())
|
await pubkeyLoader.loadPubkeys(user.getFollows())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -175,7 +177,7 @@ export const login = async (method, key) => {
|
|||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
sleep(1500),
|
sleep(1500),
|
||||||
legacyNetwork.loadPeople([user.getPubkey()], {force: true, kinds: userKinds}),
|
pubkeyLoader.loadPubkeys([user.getPubkey()], {force: true, kinds: userKinds}),
|
||||||
])
|
])
|
||||||
|
|
||||||
navigate("/notes")
|
navigate("/notes")
|
||||||
|
@ -14,8 +14,7 @@
|
|||||||
import RelayCard from "src/app/shared/RelayCard.svelte"
|
import RelayCard from "src/app/shared/RelayCard.svelte"
|
||||||
import {DEFAULT_RELAYS, FORCE_RELAYS, routing, user, network} from "src/app/system"
|
import {DEFAULT_RELAYS, FORCE_RELAYS, routing, user, network} from "src/app/system"
|
||||||
import {watch} from "src/util/loki"
|
import {watch} from "src/util/loki"
|
||||||
import legacyNetwork from "src/agent/network"
|
import {loadAppData, pubkeyLoader} from "src/app/state"
|
||||||
import {loadAppData} from "src/app/state"
|
|
||||||
|
|
||||||
let modal = null
|
let modal = null
|
||||||
let customRelayUrl = null
|
let customRelayUrl = null
|
||||||
@ -53,11 +52,15 @@
|
|||||||
attemptedRelays.add(relay.url)
|
attemptedRelays.add(relay.url)
|
||||||
currentRelays[i] = relay
|
currentRelays[i] = relay
|
||||||
|
|
||||||
legacyNetwork
|
|
||||||
.loadPeople([user.getPubkey()], {relays: [relay.url], force: true, kinds: userKinds})
|
|
||||||
.then(async () => {
|
|
||||||
// Wait a bit before removing the relay to smooth out the ui
|
// Wait a bit before removing the relay to smooth out the ui
|
||||||
await sleep(1000)
|
await Promise.all([
|
||||||
|
sleep(1500),
|
||||||
|
pubkeyLoader.loadPubkeys([user.getPubkey()], {
|
||||||
|
force: true,
|
||||||
|
relays: [relay.url],
|
||||||
|
kinds: userKinds,
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
|
||||||
currentRelays[i] = null
|
currentRelays[i] = null
|
||||||
|
|
||||||
@ -73,7 +76,6 @@
|
|||||||
} else {
|
} else {
|
||||||
network.pool.remove(relay.url)
|
network.pool.remove(relay.url)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for our relay list to load initially, then terminate when we've tried everything
|
// Wait for our relay list to load initially, then terminate when we've tried everything
|
||||||
|
@ -9,9 +9,8 @@
|
|||||||
import OnboardingRelays from "src/app/views/OnboardingRelays.svelte"
|
import OnboardingRelays from "src/app/views/OnboardingRelays.svelte"
|
||||||
import OnboardingFollows from "src/app/views/OnboardingFollows.svelte"
|
import OnboardingFollows from "src/app/views/OnboardingFollows.svelte"
|
||||||
import OnboardingNote from "src/app/views/OnboardingNote.svelte"
|
import OnboardingNote from "src/app/views/OnboardingNote.svelte"
|
||||||
import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, routing, builder, user} from "src/app/system"
|
import {DEFAULT_FOLLOWS, DEFAULT_RELAYS, builder, user} from "src/app/system"
|
||||||
import network from "src/agent/network"
|
import {loadAppData, pubkeyLoader} from "src/app/state"
|
||||||
import {loadAppData} from "src/app/state"
|
|
||||||
import {modal} from "src/partials/state"
|
import {modal} from "src/partials/state"
|
||||||
|
|
||||||
export let stage
|
export let stage
|
||||||
@ -45,9 +44,7 @@
|
|||||||
|
|
||||||
onMount(() => {
|
onMount(() => {
|
||||||
// Prime our database with some defaults
|
// Prime our database with some defaults
|
||||||
network.loadPeople(DEFAULT_FOLLOWS, {
|
pubkeyLoader.loadPubkeys(DEFAULT_FOLLOWS)
|
||||||
relays: routing.getPubkeyHints(user.getPubkey(), "read"),
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
@ -14,3 +14,4 @@ export {Builder} from "src/system/builder"
|
|||||||
export {Meta} from "src/system/meta"
|
export {Meta} from "src/system/meta"
|
||||||
export {Crypt, Keys, User} from "src/system/user"
|
export {Crypt, Keys, User} from "src/system/user"
|
||||||
export {System} from "src/system/system"
|
export {System} from "src/system/system"
|
||||||
|
export {PubkeyLoader} from "src/system/util/PubkeyLoader"
|
||||||
|
@ -4,6 +4,17 @@ export type Event = NostrToolsEvent & {
|
|||||||
seen_on: string[]
|
seen_on: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type Filter = {
|
||||||
|
ids?: string[]
|
||||||
|
kinds?: number[]
|
||||||
|
authors?: string[]
|
||||||
|
since?: number
|
||||||
|
until?: number
|
||||||
|
limit?: number
|
||||||
|
search?: string
|
||||||
|
[key: `#${string}`]: string[]
|
||||||
|
}
|
||||||
|
|
||||||
export type RelayInfo = {
|
export type RelayInfo = {
|
||||||
contact?: string
|
contact?: string
|
||||||
description?: string
|
description?: string
|
||||||
|
84
src/system/util/PubkeyLoader.ts
Normal file
84
src/system/util/PubkeyLoader.ts
Normal file
@ -0,0 +1,84 @@
|
|||||||
|
import {without, uniq} from "ramda"
|
||||||
|
import {chunk} from "hurdak/lib/hurdak"
|
||||||
|
import {personKinds, appDataKeys} from "src/util/nostr"
|
||||||
|
import {now, timedelta} from "src/util/misc"
|
||||||
|
import type {Filter} from "src/system/types"
|
||||||
|
import type {System} from "src/system/system"
|
||||||
|
|
||||||
|
export type LoadPeopleOpts = {
|
||||||
|
relays?: string[]
|
||||||
|
kinds?: number[]
|
||||||
|
force?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export class PubkeyLoader {
|
||||||
|
system: System
|
||||||
|
attemptedPubkeys: Set<string>
|
||||||
|
|
||||||
|
constructor(system) {
|
||||||
|
this.system = system
|
||||||
|
this.attemptedPubkeys = new Set()
|
||||||
|
}
|
||||||
|
|
||||||
|
getStalePubkeys = pubkeys => {
|
||||||
|
const stale = new Set()
|
||||||
|
const since = now() - timedelta(3, "hours")
|
||||||
|
|
||||||
|
for (const pubkey of pubkeys) {
|
||||||
|
if (stale.has(pubkey) || this.attemptedPubkeys.has(pubkey)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
this.attemptedPubkeys.add(pubkey)
|
||||||
|
|
||||||
|
const profile = this.system.directory.profiles.get(pubkey)
|
||||||
|
|
||||||
|
if (profile?.updated_at > since) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
stale.add(pubkey)
|
||||||
|
}
|
||||||
|
|
||||||
|
return stale
|
||||||
|
}
|
||||||
|
|
||||||
|
loadPubkeys = async (rawPubkeys, {relays, force, kinds = personKinds}: LoadPeopleOpts = {}) => {
|
||||||
|
const {network, routing, user} = this.system
|
||||||
|
const pubkeys = force ? uniq(rawPubkeys) : this.getStalePubkeys(rawPubkeys)
|
||||||
|
|
||||||
|
const getChunkRelays = chunk => {
|
||||||
|
if (relays?.length > 0) {
|
||||||
|
return relays
|
||||||
|
}
|
||||||
|
|
||||||
|
return routing.mergeHints(
|
||||||
|
user.getSetting("relay_limit"),
|
||||||
|
chunk.map(pubkey => routing.getPubkeyHints(3, pubkey))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const getChunkFilter = chunk => {
|
||||||
|
const filter = [] as Filter[]
|
||||||
|
|
||||||
|
filter.push({kinds: without([30078], kinds), authors: chunk})
|
||||||
|
|
||||||
|
// Add a separate filter for app data so we're not pulling down other people's stuff,
|
||||||
|
// or obsolete events of our own.
|
||||||
|
if (kinds.includes(30078)) {
|
||||||
|
filter.push({kinds: [30078], authors: chunk, "#d": appDataKeys})
|
||||||
|
}
|
||||||
|
|
||||||
|
return filter
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(
|
||||||
|
chunk(256, pubkeys).map(async chunk => {
|
||||||
|
await network.load({
|
||||||
|
relays: getChunkRelays(chunk),
|
||||||
|
filter: getChunkFilter(chunk),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user