From 3f406ec19ed5e14ac33ab421614d2e65a9922b39 Mon Sep 17 00:00:00 2001 From: Kieran Date: Mon, 20 Feb 2023 23:14:15 +0000 Subject: [PATCH] feat: improve profile loading --- packages/app/src/Const.ts | 2 +- packages/app/src/Feed/EventPublisher.ts | 3 +- packages/app/src/Feed/LoginFeed.ts | 13 +- packages/app/src/Feed/ProfileFeed.ts | 3 +- packages/app/src/Feed/RelayState.ts | 3 +- packages/app/src/Feed/Subscription.ts | 3 +- packages/app/src/Pages/Layout.tsx | 13 +- packages/app/src/Pages/Root.tsx | 2 +- packages/app/src/Pages/SearchPage.tsx | 2 +- packages/app/src/Pages/settings/RelayInfo.tsx | 2 +- packages/app/src/State/Users/Db.ts | 78 +++-- packages/app/src/State/Users/Hooks.ts | 40 +-- packages/app/src/System.ts | 235 ++++++++++++++ packages/app/src/Util.ts | 11 +- packages/nostr/src/legacy/Connection.ts | 13 +- packages/nostr/src/legacy/Const.ts | 147 +-------- packages/nostr/src/legacy/Subscriptions.ts | 2 +- packages/nostr/src/legacy/System.ts | 290 ------------------ packages/nostr/src/legacy/index.ts | 1 - 19 files changed, 340 insertions(+), 523 deletions(-) create mode 100644 packages/app/src/System.ts delete mode 100644 packages/nostr/src/legacy/System.ts diff --git a/packages/app/src/Const.ts b/packages/app/src/Const.ts index 373e5b84..9745227f 100644 --- a/packages/app/src/Const.ts +++ b/packages/app/src/Const.ts @@ -33,7 +33,7 @@ export const DefaultConnectTimeout = 2000; /** * How long profile cache should be considered valid for */ -export const ProfileCacheExpire = 1_000 * 60 * 5; +export const ProfileCacheExpire = 1_000 * 60 * 30; /** * Default bootstrap relays diff --git a/packages/app/src/Feed/EventPublisher.ts b/packages/app/src/Feed/EventPublisher.ts index d5ad641c..b18953be 100644 --- a/packages/app/src/Feed/EventPublisher.ts +++ b/packages/app/src/Feed/EventPublisher.ts @@ -2,11 +2,12 @@ import { useSelector } from "react-redux"; import * as secp from "@noble/secp256k1"; import { TaggedRawEvent } from "@snort/nostr"; -import { EventKind, Tag, Event as NEvent, System, RelaySettings } from "@snort/nostr"; +import { EventKind, Tag, Event as NEvent, RelaySettings } from "@snort/nostr"; import { RootState } from "State/Store"; import { HexKey, RawEvent, u256, UserMetadata, Lists } from "@snort/nostr"; import { bech32ToHex, unwrap } from "Util"; import { DefaultRelays, HashtagRegex } from "Const"; +import { System } from "System"; declare global { interface Window { diff --git a/packages/app/src/Feed/LoginFeed.ts b/packages/app/src/Feed/LoginFeed.ts index 00f4bede..752791d2 100644 --- a/packages/app/src/Feed/LoginFeed.ts +++ b/packages/app/src/Feed/LoginFeed.ts @@ -19,13 +19,13 @@ import { } from "State/Login"; import { RootState } from "State/Store"; import { mapEventToProfile, MetadataCache } from "State/Users"; -import { useDb } from "State/Users/Db"; import useSubscription from "Feed/Subscription"; import { barrierNip07 } from "Feed/EventPublisher"; import { getMutedKeys } from "Feed/MuteList"; import useModeration from "Hooks/useModeration"; import { unwrap } from "Util"; import { AnyAction, ThunkDispatch } from "@reduxjs/toolkit"; +import { ReduxUDB } from "State/Users/Db"; /** * Managed loading data for the current logged in user @@ -39,7 +39,6 @@ export default function useLoginFeed() { readNotifications, } = useSelector((s: RootState) => s.login); const { isMuted } = useModeration(); - const db = useDb(); const subMetadata = useMemo(() => { if (!pubKey) return null; @@ -176,13 +175,13 @@ export default function useLoginFeed() { { created: 0, profile: null as MetadataCache | null } ); if (maxProfile.profile) { - const existing = await db.find(maxProfile.profile.pubkey); + const existing = await ReduxUDB.find(maxProfile.profile.pubkey); if ((existing?.created ?? 0) < maxProfile.created) { - await db.put(maxProfile.profile); + await ReduxUDB.put(maxProfile.profile); } } })().catch(console.warn); - }, [dispatch, metadataFeed.store, db]); + }, [dispatch, metadataFeed.store, ReduxUDB]); useEffect(() => { const replies = notificationFeed.store.notes.filter( @@ -190,13 +189,13 @@ export default function useLoginFeed() { ); replies.forEach(nx => { dispatch(setLatestNotifications(nx.created_at)); - makeNotification(db, nx).then(notification => { + makeNotification(ReduxUDB, nx).then(notification => { if (notification) { (dispatch as ThunkDispatch)(sendNotification(notification)); } }); }); - }, [dispatch, notificationFeed.store, db, readNotifications]); + }, [dispatch, notificationFeed.store, ReduxUDB, readNotifications]); useEffect(() => { const muted = getMutedKeys(mutedFeed.store.notes); diff --git a/packages/app/src/Feed/ProfileFeed.ts b/packages/app/src/Feed/ProfileFeed.ts index ea544596..a0b7d12f 100644 --- a/packages/app/src/Feed/ProfileFeed.ts +++ b/packages/app/src/Feed/ProfileFeed.ts @@ -1,7 +1,8 @@ import { useEffect } from "react"; import { MetadataCache } from "State/Users"; import { useKey, useKeys } from "State/Users/Hooks"; -import { System, HexKey } from "@snort/nostr"; +import { HexKey } from "@snort/nostr"; +import { System } from "System"; export function useUserProfile(pubKey?: HexKey): MetadataCache | undefined { const users = useKey(pubKey); diff --git a/packages/app/src/Feed/RelayState.ts b/packages/app/src/Feed/RelayState.ts index 28cb3ff3..beef8218 100644 --- a/packages/app/src/Feed/RelayState.ts +++ b/packages/app/src/Feed/RelayState.ts @@ -1,5 +1,6 @@ import { useSyncExternalStore } from "react"; -import { System, StateSnapshot } from "@snort/nostr"; +import { StateSnapshot } from "@snort/nostr"; +import { System } from "System"; const noop = () => { return () => undefined; diff --git a/packages/app/src/Feed/Subscription.ts b/packages/app/src/Feed/Subscription.ts index e73c0814..d8bd08d0 100644 --- a/packages/app/src/Feed/Subscription.ts +++ b/packages/app/src/Feed/Subscription.ts @@ -1,6 +1,7 @@ import { useEffect, useMemo, useReducer, useState } from "react"; import { TaggedRawEvent } from "@snort/nostr"; -import { System, Subscriptions } from "@snort/nostr"; +import { Subscriptions } from "@snort/nostr"; +import { System } from "System"; import { debounce, unwrap } from "Util"; import { db } from "Db"; diff --git a/packages/app/src/Pages/Layout.tsx b/packages/app/src/Pages/Layout.tsx index 67e30dcd..30c9e8d3 100644 --- a/packages/app/src/Pages/Layout.tsx +++ b/packages/app/src/Pages/Layout.tsx @@ -9,14 +9,14 @@ import Bell from "Icons/Bell"; import Search from "Icons/Search"; import { RootState } from "State/Store"; import { init, setRelays } from "State/Login"; -import { System } from "@snort/nostr"; +import { System } from "System"; import ProfileImage from "Element/ProfileImage"; import useLoginFeed from "Feed/LoginFeed"; import { totalUnread } from "Pages/MessagesPage"; import { SearchRelays, SnortPubKey } from "Const"; import useEventPublisher from "Feed/EventPublisher"; import useModeration from "Hooks/useModeration"; -import { IndexedUDB, useDb } from "State/Users/Db"; +import { IndexedUDB } from "State/Users/Db"; import { db } from "Db"; import { bech32ToHex } from "Util"; import { NoteCreator } from "Element/NoteCreator"; @@ -34,8 +34,6 @@ export default function Layout() { useSelector((s: RootState) => s.login); const { isMuted } = useModeration(); const [pageClass, setPageClass] = useState("page"); - - const usingDb = useDb(); const pub = useEventPublisher(); useLoginFeed(); @@ -73,13 +71,9 @@ export default function Layout() { ); useEffect(() => { - System.nip42Auth = pub.nip42Auth; + System.HandleAuth = pub.nip42Auth; }, [pub]); - useEffect(() => { - System.UserDb = usingDb; - }, [usingDb]); - useEffect(() => { if (relays) { for (const [k, v] of Object.entries(relays)) { @@ -125,6 +119,7 @@ export default function Layout() { // cleanup on load if (dbType === "indexdDb") { + IndexedUDB.ready = true; await db.feeds.clear(); const now = Math.floor(new Date().getTime() / 1000); diff --git a/packages/app/src/Pages/Root.tsx b/packages/app/src/Pages/Root.tsx index 6a5d85ba..92498433 100644 --- a/packages/app/src/Pages/Root.tsx +++ b/packages/app/src/Pages/Root.tsx @@ -7,7 +7,7 @@ import { useIntl, FormattedMessage } from "react-intl"; import Tabs, { Tab } from "Element/Tabs"; import { RootState } from "State/Store"; import Timeline from "Element/Timeline"; -import { System } from "@snort/nostr"; +import { System } from "System"; import { TimelineSubject } from "Feed/TimelineFeed"; import messages from "./messages"; diff --git a/packages/app/src/Pages/SearchPage.tsx b/packages/app/src/Pages/SearchPage.tsx index abd840e9..424ede5d 100644 --- a/packages/app/src/Pages/SearchPage.tsx +++ b/packages/app/src/Pages/SearchPage.tsx @@ -6,7 +6,7 @@ import { useEffect, useState } from "react"; import { debounce } from "Util"; import { router } from "index"; import { SearchRelays } from "Const"; -import { System } from "@snort/nostr"; +import { System } from "System"; import { useQuery } from "State/Users/Hooks"; import messages from "./messages"; diff --git a/packages/app/src/Pages/settings/RelayInfo.tsx b/packages/app/src/Pages/settings/RelayInfo.tsx index d17bd0c4..2746fe46 100644 --- a/packages/app/src/Pages/settings/RelayInfo.tsx +++ b/packages/app/src/Pages/settings/RelayInfo.tsx @@ -1,11 +1,11 @@ import { FormattedMessage } from "react-intl"; import ProfilePreview from "Element/ProfilePreview"; import useRelayState from "Feed/RelayState"; -import { System } from "@snort/nostr"; import { useDispatch } from "react-redux"; import { useNavigate, useParams } from "react-router-dom"; import { removeRelay } from "State/Login"; import { parseId, unwrap } from "Util"; +import { System } from "System"; import messages from "./messages"; diff --git a/packages/app/src/State/Users/Db.ts b/packages/app/src/State/Users/Db.ts index 5cefb8d5..9201e6f7 100644 --- a/packages/app/src/State/Users/Db.ts +++ b/packages/app/src/State/Users/Db.ts @@ -2,9 +2,8 @@ import { HexKey } from "@snort/nostr"; import { db as idb } from "Db"; import { UsersDb, MetadataCache, setUsers } from "State/Users"; -import store, { RootState } from "State/Store"; -import { useSelector } from "react-redux"; -import { unwrap } from "Util"; +import store from "State/Store"; +import { groupByPubkey, unixNowMs, unwrap } from "Util"; class IndexedUsersDb implements UsersDb { ready = false; @@ -63,14 +62,12 @@ class IndexedUsersDb implements UsersDb { await idb.users.bulkPut(users); } - async update(key: HexKey, fields: Record) { + async update(key: HexKey, fields: Record) { await idb.users.update(key, fields); } } -function groupByPubkey(acc: Record, user: MetadataCache) { - return { ...acc, [user.pubkey]: user }; -} +export const IndexedUDB = new IndexedUsersDb(); class ReduxUsersDb implements UsersDb { async isAvailable() { @@ -91,22 +88,49 @@ class ReduxUsersDb implements UsersDb { }); } + querySync(q: string) { + const state = store.getState(); + const { users } = state.users; + return Object.values(users).filter(user => { + const profile = user as MetadataCache; + return ( + profile.name?.includes(q) || + profile.npub?.includes(q) || + profile.display_name?.includes(q) || + profile.nip05?.includes(q) + ); + }); + } + async find(key: HexKey) { const state = store.getState(); const { users } = state.users; - return users[key]; + let ret: MetadataCache | undefined = users[key]; + if (IndexedUDB.ready && ret === undefined) { + ret = await IndexedUDB.find(key); + if (ret) { + await this.put(ret); + } + } + return ret; } async add(user: MetadataCache) { const state = store.getState(); const { users } = state.users; store.dispatch(setUsers({ ...users, [user.pubkey]: user })); + if (IndexedUDB.ready) { + await IndexedUDB.add(user); + } } async put(user: MetadataCache) { const state = store.getState(); const { users } = state.users; store.dispatch(setUsers({ ...users, [user.pubkey]: user })); + if (IndexedUDB.ready) { + await IndexedUDB.put(user); + } } async bulkAdd(newUserProfiles: MetadataCache[]) { @@ -114,23 +138,43 @@ class ReduxUsersDb implements UsersDb { const { users } = state.users; const newUsers = newUserProfiles.reduce(groupByPubkey, {}); store.dispatch(setUsers({ ...users, ...newUsers })); + if (IndexedUDB.ready) { + await IndexedUDB.bulkAdd(newUserProfiles); + } } async bulkGet(keys: HexKey[]) { const state = store.getState(); const { users } = state.users; const ids = new Set([...keys]); - return Object.values(users).filter(user => { + let ret = Object.values(users).filter(user => { return ids.has(user.pubkey); }); + if (IndexedUDB.ready && ret.length !== ids.size) { + const startLoad = unixNowMs(); + const hasKeys = new Set(Object.keys(users)); + const missing = [...ids].filter(a => !hasKeys.has(a)); + const missingFromCache = await IndexedUDB.bulkGet(missing); + store.dispatch(setUsers({ ...users, ...missingFromCache.reduce(groupByPubkey, {}) })); + console.debug( + `Loaded ${missingFromCache.length}/${missing.length} profiles from cache in ${(unixNowMs() - startLoad).toFixed( + 1 + )} ms` + ); + ret = [...ret, ...missingFromCache]; + } + return ret; } - async update(key: HexKey, fields: Record) { + async update(key: HexKey, fields: Record) { const state = store.getState(); const { users } = state.users; const current = users[key]; const updated = { ...current, ...fields }; store.dispatch(setUsers({ ...users, [key]: updated })); + if (IndexedUDB.ready) { + await IndexedUDB.update(key, fields); + } } async bulkPut(newUsers: MetadataCache[]) { @@ -138,18 +182,10 @@ class ReduxUsersDb implements UsersDb { const { users } = state.users; const newProfiles = newUsers.reduce(groupByPubkey, {}); store.dispatch(setUsers({ ...users, ...newProfiles })); + if (IndexedUDB.ready) { + await IndexedUDB.bulkPut(newUsers); + } } } -export const IndexedUDB = new IndexedUsersDb(); export const ReduxUDB = new ReduxUsersDb(); - -export function useDb(): UsersDb { - const db = useSelector((s: RootState) => s.login.useDb); - switch (db) { - case "indexdDb": - return IndexedUDB; - default: - return ReduxUDB; - } -} diff --git a/packages/app/src/State/Users/Hooks.ts b/packages/app/src/State/Users/Hooks.ts index d18da7a7..62b16541 100644 --- a/packages/app/src/State/Users/Hooks.ts +++ b/packages/app/src/State/Users/Hooks.ts @@ -1,50 +1,20 @@ import { useSelector } from "react-redux"; -import { useLiveQuery } from "dexie-react-hooks"; import { MetadataCache } from "State/Users"; import type { RootState } from "State/Store"; import { HexKey } from "@snort/nostr"; -import { useDb } from "./Db"; +import { ReduxUDB } from "./Db"; export function useQuery(query: string) { - const db = useDb(); - return useLiveQuery(async () => db.query(query), [query]); + // TODO: not observable + return ReduxUDB.querySync(query); } export function useKey(pubKey?: HexKey) { - const db = useDb(); const { users } = useSelector((state: RootState) => state.users); - const defaultUser = pubKey ? users[pubKey] : undefined; - - const user = useLiveQuery(async () => { - if (pubKey) { - try { - return await db.find(pubKey); - } catch (error) { - console.error(error); - return defaultUser; - } - } - }, [pubKey, defaultUser]); - - return user; + return pubKey ? users[pubKey] : undefined; } export function useKeys(pubKeys?: HexKey[]): Map { - const db = useDb(); const { users } = useSelector((state: RootState) => state.users); - - const dbUsers = useLiveQuery(async () => { - if (pubKeys) { - try { - const ret = await db.bulkGet(pubKeys); - return new Map(ret.map(a => [a.pubkey, a])); - } catch (error) { - console.error(error); - return new Map(pubKeys.map(a => [a, users[a]])); - } - } - return new Map(); - }, [pubKeys, users]); - - return dbUsers ?? new Map(); + return new Map((pubKeys ?? []).map(a => [a, users[a]])); } diff --git a/packages/app/src/System.ts b/packages/app/src/System.ts new file mode 100644 index 00000000..22c4ae79 --- /dev/null +++ b/packages/app/src/System.ts @@ -0,0 +1,235 @@ +import { AuthHandler, HexKey, TaggedRawEvent } from "@snort/nostr"; + +import { Event as NEvent, EventKind, RelaySettings, Connection, Subscriptions } from "@snort/nostr"; +import { ProfileCacheExpire } from "Const"; +import { mapEventToProfile } from "State/Users"; +import { ReduxUDB } from "State/Users/Db"; +import { unwrap } from "Util"; + +/** + * Manages nostr content retrieval system + */ +export class NostrSystem { + /** + * All currently connected websockets + */ + Sockets: Map; + + /** + * All active subscriptions + */ + Subscriptions: Map; + + /** + * Pending subscriptions to send when sockets become open + */ + PendingSubscriptions: Subscriptions[]; + + /** + * List of pubkeys to fetch metadata for + */ + WantsMetadata: Set; + + /** + * Handler function for NIP-42 + */ + HandleAuth?: AuthHandler; + + constructor() { + this.Sockets = new Map(); + this.Subscriptions = new Map(); + this.PendingSubscriptions = []; + this.WantsMetadata = new Set(); + this._FetchMetadata(); + } + + /** + * Connect to a NOSTR relay if not already connected + */ + async ConnectToRelay(address: string, options: RelaySettings) { + try { + if (!this.Sockets.has(address)) { + const c = new Connection(address, options, this.HandleAuth); + await c.Connect(); + this.Sockets.set(address, c); + for (const [, s] of this.Subscriptions) { + c.AddSubscription(s); + } + } else { + // update settings if already connected + unwrap(this.Sockets.get(address)).Settings = options; + } + } catch (e) { + console.error(e); + } + } + + /** + * Disconnect from a relay + */ + DisconnectRelay(address: string) { + const c = this.Sockets.get(address); + if (c) { + this.Sockets.delete(address); + c.Close(); + } + } + + AddSubscriptionToRelay(sub: Subscriptions, relay: string) { + this.Sockets.get(relay)?.AddSubscription(sub); + } + + AddSubscription(sub: Subscriptions) { + for (const [, s] of this.Sockets) { + s.AddSubscription(sub); + } + this.Subscriptions.set(sub.Id, sub); + } + + RemoveSubscription(subId: string) { + for (const [, s] of this.Sockets) { + s.RemoveSubscription(subId); + } + this.Subscriptions.delete(subId); + } + + /** + * Send events to writable relays + */ + BroadcastEvent(ev: NEvent) { + for (const [, s] of this.Sockets) { + s.SendEvent(ev); + } + } + + /** + * Write an event to a relay then disconnect + */ + async WriteOnceToRelay(address: string, ev: NEvent) { + const c = new Connection(address, { write: true, read: false }, this.HandleAuth); + await c.Connect(); + await c.SendAsync(ev); + c.Close(); + } + + /** + * Request profile metadata for a set of pubkeys + */ + TrackMetadata(pk: HexKey | Array) { + for (const p of Array.isArray(pk) ? pk : [pk]) { + if (p.length > 0) { + this.WantsMetadata.add(p); + } + } + } + + /** + * Stop tracking metadata for a set of pubkeys + */ + UntrackMetadata(pk: HexKey | Array) { + for (const p of Array.isArray(pk) ? pk : [pk]) { + if (p.length > 0) { + this.WantsMetadata.delete(p); + } + } + } + + /** + * Request/Response pattern + */ + RequestSubscription(sub: Subscriptions) { + return new Promise(resolve => { + const events: TaggedRawEvent[] = []; + + // force timeout returning current results + const timeout = setTimeout(() => { + this.RemoveSubscription(sub.Id); + resolve(events); + }, 10_000); + + const onEventPassthrough = sub.OnEvent; + sub.OnEvent = ev => { + if (typeof onEventPassthrough === "function") { + onEventPassthrough(ev); + } + if (!events.some(a => a.id === ev.id)) { + events.push(ev); + } else { + const existing = events.find(a => a.id === ev.id); + if (existing) { + for (const v of ev.relays) { + existing.relays.push(v); + } + } + } + }; + sub.OnEnd = c => { + c.RemoveSubscription(sub.Id); + if (sub.IsFinished()) { + clearInterval(timeout); + console.debug(`[${sub.Id}] Finished`); + resolve(events); + } + }; + this.AddSubscription(sub); + }); + } + + async _FetchMetadata() { + const userDb = ReduxUDB; + + const missing = new Set(); + const meta = await userDb.bulkGet(Array.from(this.WantsMetadata)); + const expire = new Date().getTime() - ProfileCacheExpire; + for (const pk of this.WantsMetadata) { + const m = meta.find(a => a.pubkey === pk); + if ((m?.loaded ?? 0) < expire) { + missing.add(pk); + // cap 100 missing profiles + if (missing.size >= 100) { + break; + } + } + } + + if (missing.size > 0) { + console.debug("Wants profiles: ", missing); + + const sub = new Subscriptions(); + sub.Id = `profiles:${sub.Id.slice(0, 8)}`; + sub.Kinds = new Set([EventKind.SetMetadata]); + sub.Authors = missing; + sub.OnEvent = async e => { + const profile = mapEventToProfile(e); + if (profile) { + const existing = await userDb.find(profile.pubkey); + if ((existing?.created ?? 0) < profile.created) { + await userDb.put(profile); + } else if (existing) { + await userDb.update(profile.pubkey, { + loaded: profile.loaded, + }); + } + } + }; + const results = await this.RequestSubscription(sub); + const couldNotFetch = Array.from(missing).filter(a => !results.some(b => b.pubkey === a)); + console.debug("No profiles: ", couldNotFetch); + if (couldNotFetch.length > 0) { + const updates = couldNotFetch + .map(a => { + return { + pubkey: a, + loaded: new Date().getTime(), + }; + }) + .map(a => userDb.update(a.pubkey, a)); + await Promise.all(updates); + } + } + + setTimeout(() => this._FetchMetadata(), 500); + } +} + +export const System = new NostrSystem(); diff --git a/packages/app/src/Util.ts b/packages/app/src/Util.ts index 1ebe29df..8d9fea15 100644 --- a/packages/app/src/Util.ts +++ b/packages/app/src/Util.ts @@ -2,6 +2,7 @@ import * as secp from "@noble/secp256k1"; import { sha256 as hash } from "@noble/hashes/sha256"; import { bech32 } from "bech32"; import { HexKey, TaggedRawEvent, u256, EventKind, encodeTLV, NostrPrefix } from "@snort/nostr"; +import { MetadataCache } from "State/Users"; export const sha256 = (str: string) => { return secp.utils.bytesToHex(hash(str)); @@ -144,7 +145,11 @@ export function extractLnAddress(lnurl: string) { } export function unixNow() { - return Math.floor(new Date().getTime() / 1000); + return Math.floor(unixNowMs() / 1000); +} + +export function unixNowMs() { + return new Date().getTime(); } /** @@ -196,3 +201,7 @@ export function tagFilterOfTextRepost(note: TaggedRawEvent, id?: u256): (tag: st return (tag, i) => tag[0] === "e" && tag[3] === "mention" && note.content === `#[${i}]` && (id ? tag[1] === id : true); } + +export function groupByPubkey(acc: Record, user: MetadataCache) { + return { ...acc, [user.pubkey]: user }; +} diff --git a/packages/nostr/src/legacy/Connection.ts b/packages/nostr/src/legacy/Connection.ts index 1e1478ea..9a0457bb 100644 --- a/packages/nostr/src/legacy/Connection.ts +++ b/packages/nostr/src/legacy/Connection.ts @@ -8,10 +8,10 @@ import { ConnectionStats } from "./ConnectionStats"; import { RawEvent, RawReqFilter, TaggedRawEvent, u256 } from "./index"; import { RelayInfo } from "./RelayInfo"; import Nips from "./Nips"; -import { System } from "./System"; import { unwrap } from "./Util"; export type CustomHook = (state: Readonly) => void; +export type AuthHandler = (challenge: string, relay: string) => Promise; /** * Relay settings @@ -36,7 +36,7 @@ export type StateSnapshot = { id: string; }; -export default class Connection { +export class Connection { Id: string; Address: string; Socket: WebSocket | null; @@ -53,10 +53,11 @@ export default class Connection { IsClosed: boolean; ReconnectTimer: ReturnType | null; EventsCallback: Map void>; + Auth?: AuthHandler; AwaitingAuth: Map; Authed: boolean; - constructor(addr: string, options: RelaySettings) { + constructor(addr: string, options: RelaySettings, auth: AuthHandler = undefined) { this.Id = uuid(); this.Address = addr; this.Socket = null; @@ -82,6 +83,7 @@ export default class Connection { this.EventsCallback = new Map(); this.AwaitingAuth = new Map(); this.Authed = false; + this.Auth = auth; } async Connect() { @@ -384,8 +386,11 @@ export default class Connection { const authCleanup = () => { this.AwaitingAuth.delete(challenge); }; + if(!this.Auth) { + throw new Error("Auth hook not registered"); + } this.AwaitingAuth.set(challenge, true); - const authEvent = await System.nip42Auth(challenge, this.Address); + const authEvent = await this.Auth(challenge, this.Address); return new Promise((resolve) => { if (!authEvent) { authCleanup(); diff --git a/packages/nostr/src/legacy/Const.ts b/packages/nostr/src/legacy/Const.ts index eb6c9b28..0a110541 100644 --- a/packages/nostr/src/legacy/Const.ts +++ b/packages/nostr/src/legacy/Const.ts @@ -1,149 +1,4 @@ -import { RelaySettings } from "./Connection"; - -/** - * Add-on api for snort features - */ -export const ApiHost = "https://api.snort.social"; - -/** - * LibreTranslate endpoint - */ -export const TranslateHost = "https://translate.snort.social"; - -/** - * Void.cat file upload service url - */ -export const VoidCatHost = "https://void.cat"; - -/** - * Kierans pubkey - */ -export const KieranPubKey = - "npub1v0lxxxxutpvrelsksy8cdhgfux9l6a42hsj2qzquu2zk7vc9qnkszrqj49"; - -/** - * Official snort account - */ -export const SnortPubKey = - "npub1sn0rtcjcf543gj4wsg7fa59s700d5ztys5ctj0g69g2x6802npjqhjjtws"; - /** * Websocket re-connect timeout */ -export const DefaultConnectTimeout = 2000; - -/** - * How long profile cache should be considered valid for - */ -export const ProfileCacheExpire = 1_000 * 60 * 5; - -/** - * Default bootstrap relays - */ -export const DefaultRelays = new Map([ - ["wss://relay.snort.social", { read: true, write: true }], - ["wss://eden.nostr.land", { read: true, write: true }], - ["wss://atlas.nostr.land", { read: true, write: true }], -]); - -/** - * Default search relays - */ -export const SearchRelays = new Map([ - ["wss://relay.nostr.band", { read: true, write: false }], -]); - -/** - * List of recommended follows for new users - */ -export const RecommendedFollows = [ - "82341f882b6eabcd2ba7f1ef90aad961cf074af15b9ef44a09f9d2a8fbfbe6a2", // jack - "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d", // fiatjaf - "020f2d21ae09bf35fcdfb65decf1478b846f5f728ab30c5eaabcd6d081a81c3e", // adam3us - "6e468422dfb74a5738702a8823b9b28168abab8655faacb6853cd0ee15deee93", // gigi - "63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed", // Kieran - "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245", // jb55 - "e33fe65f1fde44c6dc17eeb38fdad0fceaf1cae8722084332ed1e32496291d42", // wiz - "00000000827ffaa94bfea288c3dfce4422c794fbb96625b6b31e9049f729d700", // cameri - "A341F45FF9758F570A21B000C17D4E53A3A497C8397F26C0E6D61E5ACFFC7A98", // Saylor - "E88A691E98D9987C964521DFF60025F60700378A4879180DCBBB4A5027850411", // NVK - "C4EABAE1BE3CF657BC1855EE05E69DE9F059CB7A059227168B80B89761CBC4E0", // jackmallers - "85080D3BAD70CCDCD7F74C29A44F55BB85CBCD3DD0CBB957DA1D215BDB931204", // preston - "C49D52A573366792B9A6E4851587C28042FB24FA5625C6D67B8C95C8751ACA15", // holdonaut - "83E818DFBECCEA56B0F551576B3FD39A7A50E1D8159343500368FA085CCD964B", // jeffbooth - "3F770D65D3A764A9C5CB503AE123E62EC7598AD035D836E2A810F3877A745B24", // DerekRoss - "472F440F29EF996E92A186B8D320FF180C855903882E59D50DE1B8BD5669301E", // MartyBent - "1577e4599dd10c863498fe3c20bd82aafaf829a595ce83c5cf8ac3463531b09b", // yegorpetrov - "04c915daefee38317fa734444acee390a8269fe5810b2241e5e6dd343dfbecc9", // ODELL - "7fa56f5d6962ab1e3cd424e758c3002b8665f7b0d8dcee9fe9e288d7751ac194", // verbiricha - "52b4a076bcbbbdc3a1aefa3735816cf74993b1b8db202b01c883c58be7fad8bd", // semisol -]; - -/** - * Regex to match email address - */ -export const EmailRegex = - // eslint-disable-next-line no-useless-escape - /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; - -/** - * Generic URL regex - */ -export const UrlRegex = - // eslint-disable-next-line no-useless-escape - /((?:http|ftp|https):\/\/(?:[\w+?\.\w+])+(?:[a-zA-Z0-9\~\!\@\#\$\%\^\&\*\(\)_\-\=\+\\\/\?\.\:\;\'\,]*)?)/i; - -/** - * Extract file extensions regex - */ -// eslint-disable-next-line no-useless-escape -export const FileExtensionRegex = /\.([\w]+)$/i; - -/** - * Extract note reactions regex - */ -export const MentionRegex = /(#\[\d+\])/gi; - -/** - * Simple lightning invoice regex - */ -export const InvoiceRegex = /(lnbc\w+)/i; - -/** - * YouTube URL regex - */ -export const YoutubeUrlRegex = - /(?:https?:\/\/)?(?:www|m\.)?(?:youtu\.be\/|youtube\.com\/(?:shorts\/|embed\/|v\/|watch\?v=|watch\?.+&v=))((\w|-){11})/; - -/** - * Tweet Regex - */ -export const TweetUrlRegex = - /https?:\/\/twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)/; - -/** - * Hashtag regex - */ -// eslint-disable-next-line no-useless-escape -export const HashtagRegex = /(#[^\s!@#$%^&*()=+.\/,\[{\]};:'"?><]+)/; - -/** - * Tidal share link regex - */ -export const TidalRegex = /tidal\.com\/(?:browse\/)?(\w+)\/([a-z0-9-]+)/i; - -/** - * SoundCloud regex - */ -export const SoundCloudRegex = - /soundcloud\.com\/(?!live)([a-zA-Z0-9]+)\/([a-zA-Z0-9-]+)/; - -/** - * Mixcloud regex - */ - -export const MixCloudRegex = - /mixcloud\.com\/(?!live)([a-zA-Z0-9]+)\/([a-zA-Z0-9-]+)/; - -export const SpotifyRegex = - /open\.spotify\.com\/(track|album|playlist|episode)\/([a-zA-Z0-9]+)/; +export const DefaultConnectTimeout = 2000; \ No newline at end of file diff --git a/packages/nostr/src/legacy/Subscriptions.ts b/packages/nostr/src/legacy/Subscriptions.ts index 40f6cff9..81aa5bee 100644 --- a/packages/nostr/src/legacy/Subscriptions.ts +++ b/packages/nostr/src/legacy/Subscriptions.ts @@ -1,6 +1,6 @@ import { v4 as uuid } from "uuid"; import { TaggedRawEvent, RawReqFilter, u256 } from "./index"; -import Connection from "./Connection"; +import { Connection } from "./Connection"; import EventKind from "./EventKind"; export type NEventHandler = (e: TaggedRawEvent) => void; diff --git a/packages/nostr/src/legacy/System.ts b/packages/nostr/src/legacy/System.ts deleted file mode 100644 index 929a950d..00000000 --- a/packages/nostr/src/legacy/System.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { HexKey, TaggedRawEvent, UserMetadata } from "./index"; -import { ProfileCacheExpire } from "./Const"; -import Connection, { RelaySettings } from "./Connection"; -import Event from "./Event"; -import EventKind from "./EventKind"; -import { Subscriptions } from "./Subscriptions"; -import { hexToBech32, unwrap } from "./Util"; - -// TODO This interface is repeated in State/Users, revisit this. -export interface MetadataCache extends UserMetadata { - /** - * When the object was saved in cache - */ - loaded: number; - - /** - * When the source metadata event was created - */ - created: number; - - /** - * The pubkey of the owner of this metadata - */ - pubkey: HexKey; - - /** - * The bech32 encoded pubkey - */ - npub: string; -} - -// TODO This interface is repeated in State/Users, revisit this. -export interface UsersDb { - isAvailable(): Promise; - query(str: string): Promise; - find(key: HexKey): Promise; - add(user: MetadataCache): Promise; - put(user: MetadataCache): Promise; - bulkAdd(users: MetadataCache[]): Promise; - bulkGet(keys: HexKey[]): Promise; - bulkPut(users: MetadataCache[]): Promise; - update(key: HexKey, fields: Record): Promise; -} - -/** - * Manages nostr content retrieval system - */ -export class NostrSystem { - /** - * All currently connected websockets - */ - Sockets: Map; - - /** - * All active subscriptions - */ - Subscriptions: Map; - - /** - * Pending subscriptions to send when sockets become open - */ - PendingSubscriptions: Subscriptions[]; - - /** - * List of pubkeys to fetch metadata for - */ - WantsMetadata: Set; - - /** - * User db store - */ - UserDb?: UsersDb; - - constructor() { - this.Sockets = new Map(); - this.Subscriptions = new Map(); - this.PendingSubscriptions = []; - this.WantsMetadata = new Set(); - this._FetchMetadata(); - } - - /** - * Connect to a NOSTR relay if not already connected - */ - async ConnectToRelay(address: string, options: RelaySettings) { - try { - if (!this.Sockets.has(address)) { - const c = new Connection(address, options); - await c.Connect(); - this.Sockets.set(address, c); - for (const [, s] of this.Subscriptions) { - c.AddSubscription(s); - } - } else { - // update settings if already connected - unwrap(this.Sockets.get(address)).Settings = options; - } - } catch (e) { - console.error(e); - } - } - - /** - * Disconnect from a relay - */ - DisconnectRelay(address: string) { - const c = this.Sockets.get(address); - if (c) { - this.Sockets.delete(address); - c.Close(); - } - } - - AddSubscriptionToRelay(sub: Subscriptions, relay: string) { - this.Sockets.get(relay)?.AddSubscription(sub); - } - - AddSubscription(sub: Subscriptions) { - for (const [, s] of this.Sockets) { - s.AddSubscription(sub); - } - this.Subscriptions.set(sub.Id, sub); - } - - RemoveSubscription(subId: string) { - for (const [, s] of this.Sockets) { - s.RemoveSubscription(subId); - } - this.Subscriptions.delete(subId); - } - - /** - * Send events to writable relays - */ - BroadcastEvent(ev: Event) { - for (const [, s] of this.Sockets) { - s.SendEvent(ev); - } - } - - /** - * Write an event to a relay then disconnect - */ - async WriteOnceToRelay(address: string, ev: Event) { - const c = new Connection(address, { write: true, read: false }); - await c.SendAsync(ev); - c.Close(); - } - - /** - * Request profile metadata for a set of pubkeys - */ - TrackMetadata(pk: HexKey | Array) { - for (const p of Array.isArray(pk) ? pk : [pk]) { - if (p.length > 0) { - this.WantsMetadata.add(p); - } - } - } - - /** - * Stop tracking metadata for a set of pubkeys - */ - UntrackMetadata(pk: HexKey | Array) { - for (const p of Array.isArray(pk) ? pk : [pk]) { - if (p.length > 0) { - this.WantsMetadata.delete(p); - } - } - } - - /** - * Request/Response pattern - */ - RequestSubscription(sub: Subscriptions) { - return new Promise((resolve) => { - const events: TaggedRawEvent[] = []; - - // force timeout returning current results - const timeout = setTimeout(() => { - this.RemoveSubscription(sub.Id); - resolve(events); - }, 10_000); - - const onEventPassthrough = sub.OnEvent; - sub.OnEvent = (ev) => { - if (typeof onEventPassthrough === "function") { - onEventPassthrough(ev); - } - if (!events.some((a) => a.id === ev.id)) { - events.push(ev); - } else { - const existing = events.find((a) => a.id === ev.id); - if (existing) { - for (const v of ev.relays) { - existing.relays.push(v); - } - } - } - }; - sub.OnEnd = (c) => { - c.RemoveSubscription(sub.Id); - if (sub.IsFinished()) { - clearInterval(timeout); - console.debug(`[${sub.Id}] Finished`); - resolve(events); - } - }; - this.AddSubscription(sub); - }); - } - - async _FetchMetadata() { - if (this.UserDb) { - const missing = new Set(); - const meta = await this.UserDb.bulkGet(Array.from(this.WantsMetadata)); - const expire = new Date().getTime() - ProfileCacheExpire; - for (const pk of this.WantsMetadata) { - const m = meta.find((a) => a?.pubkey === pk); - if (!m || m.loaded < expire) { - missing.add(pk); - // cap 100 missing profiles - if (missing.size >= 100) { - break; - } - } - } - - if (missing.size > 0) { - console.debug("Wants profiles: ", missing); - - const sub = new Subscriptions(); - sub.Id = `profiles:${sub.Id.slice(0, 8)}`; - sub.Kinds = new Set([EventKind.SetMetadata]); - sub.Authors = missing; - sub.OnEvent = async (e) => { - const profile = mapEventToProfile(e); - const userDb = unwrap(this.UserDb); - if (profile) { - const existing = await userDb.find(profile.pubkey); - if ((existing?.created ?? 0) < profile.created) { - await userDb.put(profile); - } else if (existing) { - await userDb.update(profile.pubkey, { - loaded: profile.loaded, - }); - } - } - }; - const results = await this.RequestSubscription(sub); - const couldNotFetch = Array.from(missing).filter( - (a) => !results.some((b) => b.pubkey === a) - ); - console.debug("No profiles: ", couldNotFetch); - if (couldNotFetch.length > 0) { - const updates = couldNotFetch - .map((a) => { - return { - pubkey: a, - loaded: new Date().getTime(), - }; - }) - .map((a) => unwrap(this.UserDb).update(a.pubkey, a)); - await Promise.all(updates); - } - } - } - setTimeout(() => this._FetchMetadata(), 500); - } - - nip42Auth: (challenge: string, relay: string) => Promise = - async () => undefined; -} - -function mapEventToProfile(ev: TaggedRawEvent) { - try { - const data: UserMetadata = JSON.parse(ev.content); - return { - pubkey: ev.pubkey, - npub: hexToBech32("npub", ev.pubkey), - created: ev.created_at, - loaded: new Date().getTime(), - ...data, - } as MetadataCache; - } catch (e) { - console.error("Failed to parse JSON", ev, e); - } -} - -export const System = new NostrSystem(); diff --git a/packages/nostr/src/legacy/index.ts b/packages/nostr/src/legacy/index.ts index a3ae6c94..735dc7dd 100644 --- a/packages/nostr/src/legacy/index.ts +++ b/packages/nostr/src/legacy/index.ts @@ -1,4 +1,3 @@ -export * from "./System"; export * from "./Connection"; export { default as EventKind } from "./EventKind"; export { Subscriptions } from "./Subscriptions";