Improve relay connection stuff

This commit is contained in:
Jonathan Staab 2023-01-14 13:15:43 -08:00
parent 19ee631f19
commit b848c92669
12 changed files with 244 additions and 110 deletions

View File

@ -39,15 +39,9 @@ If you like Coracle and want to support its development, you can donate sats via
- File import/export from db, NFC transfer
- [ ] Save user notes to db
- [ ] Release to android with https://svelte-native.technology/docs
# Bugs
- [ ] Add CSP
- [ ] Reduce reflow on feeds from new stuff coming in
- [ ] Follow fiatjaf's vision of clients being smart and connecting to recommended relays to fetch content
- [ ] Add settings storage on nostr, maybe use kind 0?
- [ ] Stack views so scroll position isn't lost on navigation
- [ ] Add notification for slow relays, suggest relays based on network
- [ ] Separating events table into notes/reactions/etc would effectively give us a second index on kind.
- [ ] Suggest relays based on network
# Changelog
@ -58,24 +52,26 @@ If you like Coracle and want to support its development, you can donate sats via
- [x] Use user relays for feeds
- [x] Publish to user relays + target relays:
- [x] Add correct recommended relay to tags
- [ ] Relays
- [ ] Support some read/write config
- [ ] Get real home relays for defaults.petnames
- [ ] Add support for astral's relay hack (but don't publish to it)
- [ ] Add settings storage on nostr, maybe use kind 0?
- [ ] Warn that everything will be cleared on logout
- [x] Relays
- [x] Support some read/write config
- [x] Get real home relays for defaults.petnames
- [x] Add notification for slow relays
- [ ] Fix publishing
- [ ] Relay list isn't getting refreshed since we're using getRelay everywhere
- [x] Warn that everything will be cleared on logout
- [x] Connection management
- [x] Do I need to implement re-connecting now?
- [x] Handle failed connections
- [x] Close connections that haven't been used in a while
- [ ] Login
- [ ] Prefer extension, make private key entry "advanced"
- [ ] Improve login UX for bootstrap delay. Nostr facts?
- [ ] Connection management
- [ ] Do I need to implement re-connecting now?
- [ ] Handle failed connections
- [ ] Close connections that haven't been used in a while
- [ ] We often get the root as the reply, figure out why that is, compared to astral/damus
- [ ] Load feeds from network rather than user relays?
- Still use "my" relays for global, this could make global feed more useful
- [ ] Figure out migrations from previous version
- [ ] Add relays/mentions to note and reply composition
- [ ] Add layout component with max-w, padding, etc. Test on mobile size
## 0.2.7

View File

@ -2,6 +2,7 @@
import "@fortawesome/fontawesome-free/css/fontawesome.css"
import "@fortawesome/fontawesome-free/css/solid.css"
import {pluck} from 'ramda'
import {onMount} from "svelte"
import {writable, get} from "svelte/store"
import {fly, fade} from "svelte/transition"
@ -11,7 +12,7 @@
import {hasParent} from 'src/util/html'
import {displayPerson, isLike} from 'src/util/nostr'
import {timedelta, now} from 'src/util/misc'
import {user, getRelays} from 'src/agent'
import {user, pool, getRelays} from 'src/agent'
import {modal, toast, settings, alerts} from "src/app"
import {routes} from "src/app/ui"
import Anchor from 'src/partials/Anchor.svelte'
@ -50,6 +51,7 @@
let menuIcon
let scrollY
let suspendedSubs = []
let slowConnections = []
let {lastCheckedAlerts, mostRecentAlert} = alerts
onMount(() => {
@ -58,6 +60,27 @@
alerts.listen(getRelays(), $user.pubkey)
}
const interval = setInterval(() => {
// Only notify about relays the user is actually subscribed to
const relayUrls = pluck('url', getRelays())
// Prune connections we haven't used in a while
pool.getConnections()
.filter(conn => conn.lastRequest < Date.now() - 60_000)
.forEach(conn => conn.disconnect())
// Log stats for debugging purposes
console.log(
'Connection stats',
pool.getConnections()
.map(({url, stats: s}) => ({url, avgRequest: s.timer / s.count}))
)
// Alert the user to any heinously slow connections
slowConnections = pool.getConnections()
.filter(({url, stats: s}) => relayUrls.includes(url) && s.timer / s.count > 3000)
}, 10_000)
// Close menu on click outside
document.querySelector("html").addEventListener("click", e => {
if (e.target !== menuIcon) {
@ -82,6 +105,7 @@
})
return () => {
clearInterval(interval)
unsubModal()
}
})
@ -89,7 +113,7 @@
<Router {url}>
<div use:links class="h-full">
<div class="pt-16 text-white h-full">
<div class="pt-16 text-white h-full lg:ml-56">
<Route path="/alerts" component={Alerts} />
<Route path="/search/:type" component={Search} />
<Route path="/notes/:activeTab" component={Notes} />
@ -115,7 +139,7 @@
<ul
class="py-20 w-56 bg-dark fixed top-0 bottom-0 left-0 transition-all shadow-xl
border-r border-medium text-white overflow-hidden z-10"
border-r border-medium text-white overflow-hidden z-10 lg:ml-0"
class:-ml-56={!$menuIsOpen}
>
{#if $user}
@ -147,9 +171,12 @@
</a>
</li>
<li class="h-px mx-3 my-4 bg-medium" />
<li class="cursor-pointer">
<li class="cursor-pointer relative">
<a class="block px-4 py-2 hover:bg-accent transition-all" href="/relays">
<i class="fa-solid fa-server mr-2" /> Relays
{#if slowConnections.length > 0}
<div class="w-2 h-2 rounded bg-accent absolute top-2 left-8" />
{/if}
</a>
</li>
{#if $user}
@ -181,13 +208,15 @@
class="fixed top-0 bg-dark flex justify-between items-center text-white w-full p-4
border-b border-medium z-10"
>
<i class="fa-solid fa-bars fa-2xl cursor-pointer" bind:this={menuIcon} on:click={toggleMenu} />
<div class="lg:hidden">
<i class="fa-solid fa-bars fa-2xl cursor-pointer" bind:this={menuIcon} on:click={toggleMenu} />
</div>
<Anchor external type="unstyled" href="https://github.com/staab/coracle" class="flex items-center gap-2">
<img src="/images/favicon.png" class="w-8" />
<h1 class="staatliches text-3xl">Coracle</h1>
</Anchor>
{#if $mostRecentAlert > $lastCheckedAlerts}
<div class="w-2 h-2 rounded bg-accent absolute top-4 left-12" />
{#if $mostRecentAlert > $lastCheckedAlerts || slowConnections.length > 0}
<div class="w-2 h-2 rounded bg-accent absolute top-4 left-12 lg:hidden" />
{/if}
</div>

View File

@ -9,14 +9,20 @@ export const db = new Dexie('agent/data/db')
db.version(9).stores({
relays: '++url, name',
alerts: '++id, created_at',
people: '++pubkey, updated_at',
people: '++pubkey',
})
// A flag for hiding things that rely on people being loaded initially
export const ready = writable(false)
// Some things work better as observables than database tables
export const people = writable([])
// Bootstrap our people observable
db.people.toArray().then($p => people.set(createMap('pubkey', $p)))
db.people.toArray().then($p => {
people.set(createMap('pubkey', $p))
ready.set(true)
})
// Sync to a regular object so we have a synchronous interface
let $people = {}
@ -44,19 +50,31 @@ export const processEvents = async events => {
const updates = {}
for (const e of profileEvents) {
const person = getPerson(e.pubkey, true)
updates[e.pubkey] = {
...getPerson(e.pubkey, true),
...person,
...updates[e.pubkey],
...switcherFn(e.kind, {
0: () => JSON.parse(e.content),
2: () => ({
relays: ($people[e.pubkey]?.relays || []).concat({url: e.content}),
}),
2: () => {
if (e.created_at > person.updated_at) {
return {
relays: ($people[e.pubkey]?.relays || []).concat({url: e.content}),
relays_updated_at: e.created_at,
}
}
},
3: () => ({petnames: e.tags}),
12165: () => ({muffle: e.tags}),
10001: () => ({
relays: e.tags.map(([url, read, write]) => ({url, read, write})),
}),
10001: () => {
if (e.created_at > person.updated_at) {
return {
relays: e.tags.map(([url, read, write]) => ({url, read, write})),
relays_updated_at: e.created_at,
}
}
},
default: () => {
console.log(`Received unsupported event type ${event.kind}`)
},

View File

@ -4,11 +4,11 @@ import {getTagValues, Tags} from 'src/util/nostr'
import pool from 'src/agent/pool'
import keys from 'src/agent/keys'
import defaults from 'src/agent/defaults'
import {db, people, getPerson, processEvents} from 'src/agent/data'
import {db, people, ready, getPerson, processEvents} from 'src/agent/data'
Object.assign(window, {pool, db})
export {pool, keys, db, people, getPerson}
export {pool, keys, db, ready, people, getPerson}
export const user = derived(
[keys.pubkey, people],

View File

@ -1,10 +1,11 @@
import {relayInit} from 'nostr-tools'
import {uniqBy, prop, find, whereEq, is, filter, identity} from 'ramda'
import {uniqBy, reject, prop, find, whereEq, is, filter, identity} from 'ramda'
import {ensurePlural} from 'hurdak/lib/hurdak'
import {isRelay} from 'src/util/nostr'
import {sleep} from 'src/util/misc'
import {db} from 'src/agent/data'
const connections = []
let connections = []
class Connection {
constructor(url) {
@ -28,7 +29,7 @@ class Connection {
})
nostr.on('disconnect', () => {
delete connections[url]
connections = reject(whereEq({url}), connections)
})
return nostr
@ -58,13 +59,20 @@ class Connection {
return this
}
async disconnect() {
this.status = 'closed'
await this.nostr.close()
}
}
const getConnections = () => connections
const findConnection = url => find(whereEq({url}), connections)
const connect = async url => {
const conn = findConnection(url) || new Connection(url)
await db.relays.put({url})
await Promise.race([conn.connect(), sleep(5000)])
if (conn.status === 'ready') {
@ -74,7 +82,7 @@ const connect = async url => {
const publish = async (relays, event) => {
return Promise.all(
relays.filter(r => r.read !== '!' & isRelay(r.url)).map(async relay => {
relays.filter(r => r.write !== '!' & isRelay(r.url)).map(async relay => {
const conn = await connect(relay.url)
if (conn) {
@ -138,7 +146,10 @@ const subscribe = async (relays, filters) => {
subs,
unsub: () => {
subs.forEach(sub => {
sub.unsub()
if (sub.conn.status === 'ready') {
sub.unsub()
}
sub.conn.stats.activeCount -= 1
})
},
@ -191,9 +202,11 @@ const request = (relays, filters) => {
if (!eose.includes(sub.conn.url)) {
const conn = findConnection(sub.conn.url)
conn.stats.count += 1
conn.stats.timer += Date.now() - now
conn.stats.timeouts += 1
if (conn) {
conn.stats.count += 1
conn.stats.timer += Date.now() - now
conn.stats.timeouts += 1
}
}
})
}
@ -202,14 +215,16 @@ const request = (relays, filters) => {
agg.onEvent(e => events.push(e))
agg.onEose(async url => {
const conn = findConnection(url)
if (!eose.includes(url)) {
eose.push(url)
const conn = findConnection(url)
// Keep track of relay timing stats
conn.stats.count += 1
conn.stats.timer += Date.now() - now
if (conn) {
conn.stats.count += 1
conn.stats.timer += Date.now() - now
}
}
attemptToComplete()
@ -220,4 +235,4 @@ const request = (relays, filters) => {
})
}
export default {connect, publish, subscribe, request}
export default {getConnections, findConnection, connect, publish, subscribe, request}

View File

@ -7,8 +7,8 @@ import {keys, publish, getRelays} from 'src/agent'
const updateUser = (relays, updates) =>
publishEvent(relays, 0, {content: JSON.stringify(updates)})
const setRelays = (relays, tags) =>
publishEvent(relays, 10001, {tags})
const setRelays = (relays, newRelays) =>
publishEvent(relays, 10001, {tags: newRelays.map(r => [r.url, r.read || "", r.write || ""])})
const setPetnames = (relays, petnames) =>
publishEvent(relays, 3, {tags: petnames})

View File

@ -78,13 +78,7 @@ export const loadNote = async (relays, id) => {
return null
}
const context = await loaders.loadContext(relays, found)
const note = annotate(found, context)
// Log this for debugging purposes
console.log('loadNote', note)
return note
return annotate(found, await loaders.loadContext(relays, found))
}
export const render = (note, {showEntire = false}) => {

View File

@ -34,7 +34,7 @@
const logIn = async ({privkey, pubkey}) => {
loading = true
login({privkey, pubkey})
await login({privkey, pubkey})
navigate('/relays')
}

View File

@ -1,17 +1,31 @@
<script>
import {fly} from 'svelte/transition'
import Anchor from 'src/partials/Anchor.svelte'
import {db} from 'src/agent'
setTimeout(async () => {
// Clear localstorage and database
localStorage.clear()
db.delete()
let confirmed = false
// do a hard refresh so everything gets totally cleared
window.location = '/login'
}, 300)
const confirm = () => {
confirmed = true
// Give them a moment to see the state transition
setTimeout(async () => {
localStorage.clear()
await db.delete()
// do a hard refresh so everything gets totally cleared
window.location = '/login'
}, 300)
}
</script>
<div class="max-w-xl m-auto text-center py-20" in:fly={{y:20}}>
Clearing your local database...
{#if confirmed}
<div>Clearing your local database...</div>
{:else}
<div class="flex flex-col gap-8 items-center">
<div>Are you sure you want to log out? All data will be cleared.</div>
<Anchor type="button" on:click={confirm}>Log out</Anchor>
</div>
{/if}
</div>

View File

@ -1,27 +1,31 @@
<script>
import {liveQuery} from 'dexie'
import {whereEq, find, last, reject} from 'ramda'
import {noop} from 'hurdak/lib/hurdak'
import {onMount} from 'svelte'
import {get} from 'svelte/store'
import {fly} from 'svelte/transition'
import {fuzzy} from "src/util/misc"
import {fuzzy, poll} from "src/util/misc"
import Input from "src/partials/Input.svelte"
import Anchor from "src/partials/Anchor.svelte"
import Toggle from "src/partials/Toggle.svelte"
import {db, user} from "src/agent"
import {pool, db, getRelays, ready} from "src/agent"
import {modal, addRelay, removeRelay, setRelayWriteCondition, settings} from "src/app"
import defaults from "src/agent/defaults"
let q = ""
let search
let status = {}
let relays = []
$: relays = $user?.relays || defaults.relays
fetch(get(settings).dufflepudUrl + '/relay')
.then(async res => {
const {relays} = await res.json()
fetch(get(settings).dufflepudUrl + '/relay').then(r => r.json()).then(({relays}) => {
for (const url of relays) {
db.relays.put({url})
}
})
for (const url of relays) {
db.relays.put({url})
}
}).catch(noop)
for (const relay of defaults.relays) {
db.relays.put(relay)
@ -31,17 +35,35 @@
$: search = fuzzy($knownRelays, {keys: ["name", "description", "url"]})
const join = url => {
relays = relays.concat({url, write: "!"})
addRelay(url)
document.querySelector('input').select()
const join = async url => {
await addRelay({url, write: "!"})
}
const leave = url => {
relays = reject(whereEq({url}), relays)
removeRelay(url)
const leave = async url => {
await removeRelay(url)
}
onMount(() => {
// Attempt to connect so we can show status
relays.forEach(relay => pool.connect(relay.url))
return poll(300, () => {
if ($ready) {
relays = getRelays()
}
status = Object.fromEntries(
pool.getConnections().map(({url, status, stats}) => {
// Be more strict here than with alerts
if (status === 'ready' && stats.timer / stats.count > 1000) {
status = 'slow'
}
return [url, status]
})
)
})
})
</script>
<div class="flex flex-col gap-6 m-auto max-w-2xl py-12">
@ -59,7 +81,7 @@
interact with the network, but you can join as many as you like.
</p>
<div class="grid grid-cols-1 sm:grid-cols-2 gap-4">
{#each relays as {url, write}, i}
{#each relays as {url, write}, i (url)}
<div class="rounded border border-solid border-medium bg-dark shadow" in:fly={{y: 20, delay: i * 100}}>
<div class="flex flex-col gap-2 py-3 px-6">
<div class="flex gap-2 items-center justify-between">
@ -69,7 +91,29 @@
</strong>
<i class="fa fa-times cursor-pointer" on:click={() => leave(url)}/>
</div>
<p class="text-light">placeholder for description</p>
<p class="text-light">
{#if status[url] === 'error'}
<div class="flex gap-2 items-center">
<span class="inline-block w-2 h-2 rounded bg-danger" /> Not connected
</div>
{:else if status[url] === 'pending'}
<div class="flex gap-2 items-center">
<span class="inline-block w-2 h-2 rounded bg-warning" /> Trying to connect
</div>
{:else if status[url] === 'slow'}
<div class="flex gap-2 items-center">
<span class="inline-block w-2 h-2 rounded bg-warning" /> Slow connection
</div>
{:else if status[url] === 'ready'}
<div class="flex gap-2 items-center">
<span class="inline-block w-2 h-2 rounded bg-success" /> Connected
</div>
{:else}
<div class="flex gap-2 items-center">
<span class="inline-block w-2 h-2 rounded bg-medium" /> Waiting to reconnect
</div>
{/if}
</p>
</div>
<div class="border-b border-solid border-medium" />
<div class="flex justify-between gap-2 py-3 px-6">
@ -81,31 +125,37 @@
</div>
{/each}
</div>
{#if ($knownRelays || []).length > 0}
<div class="pt-2 mb-2 border-b border-solid border-medium" />
<div class="flex gap-2 items-center">
<i class="fa fa-globe fa-lg" />
<h2 class="staatliches text-2xl">Other relays</h2>
</div>
<Input bind:value={q} type="text" wrapperClass="flex-grow" placeholder="Type to search">
<i slot="before" class="fa-solid fa-search" />
</Input>
{/if}
{#each (search(q) || []).slice(0, 50) as {url, name, description}}
{#if !find(whereEq({url}), relays)}
<div class="flex gap-2 justify-between">
<div>
<strong>{name || url}</strong>
<p class="text-light">{description || ''}</p>
</div>
<a class="underline cursor-pointer" on:click={() => join(url)}>
Join
</a>
<div class="flex flex-col gap-6" in:fly={{y: 20, delay: 1000}}>
{#if ($knownRelays || []).length > 0}
<div class="pt-2 mb-2 border-b border-solid border-medium" />
<div class="flex gap-2 items-center">
<i class="fa fa-globe fa-lg" />
<h2 class="staatliches text-2xl">Other relays</h2>
</div>
<p>
Coracle automatically discovers relays as you browse the network. Adding more relays
will generally make things quicker to load, at the expense of higher data usage.
</p>
<Input bind:value={q} type="text" wrapperClass="flex-grow" placeholder="Type to search">
<i slot="before" class="fa-solid fa-search" />
</Input>
{/if}
{/each}
<small class="text-center">
Showing {Math.min(($knownRelays || []).length - relays.length, 50)}
of {($knownRelays || []).length - relays.length} known relays
</small>
{#each (search(q) || []).slice(0, 50) as {url, name, description} (url)}
{#if !find(whereEq({url}), relays)}
<div class="flex gap-2 justify-between">
<div>
<strong>{name || url}</strong>
<p class="text-light">{description || ''}</p>
</div>
<a class="underline cursor-pointer" on:click={() => join(url)}>
Join
</a>
</div>
{/if}
{/each}
<small class="text-center">
Showing {Math.min(($knownRelays || []).length - relays.length, 50)}
of {($knownRelays || []).length - relays.length} known relays
</small>
</div>
</div>

View File

@ -78,6 +78,22 @@ export const formatTimestampRelative = ts => {
export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
export const poll = (t, cb) => {
let active = true
;(async () => {
while (active) {
cb()
await sleep(t)
}
})()
return () => {
active = false
}
}
export const createScroller = loadMore => {
// NOTE TO FUTURE SELF
// If the scroller is saturating request channels on a slow relay, the

View File

@ -15,6 +15,8 @@ module.exports = {
medium: "#403D39",
dark: "#252422",
danger: "#ff0000",
warning: "#ebd112",
success: "#37ab51",
placeholder: "#a19989",
},
},