From 06b7dcad1183a021e741cf2e1b3a5564e3909c1c Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 20 Dec 2023 14:07:51 +0000 Subject: [PATCH] feat: tools pages Various other fixes: - Better handeling of limit/since/before merging - Expose timeout through request builder - Expose PickN through request builder - Fix tests --- packages/app/src/Cache/FollowsFeed.ts | 12 +- .../app/src/Element/User/FollowListBase.tsx | 6 +- packages/app/src/Feed/LoginFeed.ts | 2 +- packages/app/src/Login/Functions.ts | 14 +- packages/app/src/Pages/settings/Menu.tsx | 74 ++++++---- packages/app/src/Pages/settings/Routes.tsx | 6 + .../settings/tools/follows-relay-health.tsx | 58 ++++++++ .../app/src/Pages/settings/tools/index.tsx | 50 +++++++ .../Pages/settings/tools/prune-follows.tsx | 136 ++++++++++++++++++ packages/app/src/icons.svg | 10 +- packages/system/src/nostr-system.ts | 2 +- packages/system/src/outbox-model.ts | 73 +++++++--- packages/system/src/query-optimizer/index.ts | 1 + .../src/query-optimizer/request-expander.ts | 58 +++++--- .../src/query-optimizer/request-merger.ts | 19 +-- packages/system/src/query.ts | 10 +- packages/system/src/request-builder.ts | 22 ++- ...sip-model.test.ts => outbox-model.test.ts} | 16 ++- packages/system/tests/request-builder.test.ts | 17 ++- .../system/tests/request-expander.test.ts | 38 ++--- packages/system/tests/request-merger.test.ts | 58 ++++---- 21 files changed, 526 insertions(+), 156 deletions(-) create mode 100644 packages/app/src/Pages/settings/tools/follows-relay-health.tsx create mode 100644 packages/app/src/Pages/settings/tools/index.tsx create mode 100644 packages/app/src/Pages/settings/tools/prune-follows.tsx rename packages/system/tests/{gossip-model.test.ts => outbox-model.test.ts} (59%) diff --git a/packages/app/src/Cache/FollowsFeed.ts b/packages/app/src/Cache/FollowsFeed.ts index 89e657f9e..04735dd8e 100644 --- a/packages/app/src/Cache/FollowsFeed.ts +++ b/packages/app/src/Cache/FollowsFeed.ts @@ -27,8 +27,10 @@ export class FollowsFeedCache extends RefreshFeedCache { } buildSub(session: LoginSession, rb: RequestBuilder): void { - const authors = session.follows.item; - authors.push(session.publicKey); + const authors = [...session.follows.item]; + if (session.publicKey) { + authors.push(session.publicKey); + } const since = this.newest(); rb.withFilter() .kinds(this.#kinds) @@ -69,8 +71,10 @@ export class FollowsFeedCache extends RefreshFeedCache { async loadMore(system: SystemInterface, session: LoginSession, before: number) { if (this.#oldest && before <= this.#oldest) { const rb = new RequestBuilder(`${this.name}-loadmore`); - const authors = session.follows.item; - authors.push(session.publicKey); + const authors = [...session.follows.item]; + if (session.publicKey) { + authors.push(session.publicKey); + } rb.withFilter() .kinds(this.#kinds) .authors(authors) diff --git a/packages/app/src/Element/User/FollowListBase.tsx b/packages/app/src/Element/User/FollowListBase.tsx index f07c8371c..2517a8a43 100644 --- a/packages/app/src/Element/User/FollowListBase.tsx +++ b/packages/app/src/Element/User/FollowListBase.tsx @@ -32,13 +32,13 @@ export default function FollowListBase({ profileActions, }: FollowListBaseProps) { const { publisher, system } = useEventPublisher(); - const login = useLogin(); + const { id, follows } = useLogin(s => ({ id: s.id, follows: s.follows })); async function followAll() { if (publisher) { - const newFollows = dedupe([...pubkeys, ...login.follows.item]); + const newFollows = dedupe([...pubkeys, ...follows.item]); const ev = await publisher.contactList(newFollows.map(a => ["p", a])); - setFollows(login, newFollows, ev.created_at); + setFollows(id, newFollows, ev.created_at); await system.BroadcastEvent(ev); await FollowsFeed.backFill(system, pubkeys); } diff --git a/packages/app/src/Feed/LoginFeed.ts b/packages/app/src/Feed/LoginFeed.ts index 941ce2cc4..69d97e67e 100644 --- a/packages/app/src/Feed/LoginFeed.ts +++ b/packages/app/src/Feed/LoginFeed.ts @@ -104,7 +104,7 @@ export default function useLoginFeed() { const contactList = getNewest(loginFeed.data.filter(a => a.kind === EventKind.ContactList)); if (contactList) { const pTags = contactList.tags.filter(a => a[0] === "p").map(a => a[1]); - setFollows(login, pTags, contactList.created_at * 1000); + setFollows(login.id, pTags, contactList.created_at * 1000); FollowsFeed.backFillIfMissing(system, pTags); } diff --git a/packages/app/src/Login/Functions.ts b/packages/app/src/Login/Functions.ts index accabd1fa..ccbdae898 100644 --- a/packages/app/src/Login/Functions.ts +++ b/packages/app/src/Login/Functions.ts @@ -177,13 +177,15 @@ export function setBlocked(state: LoginSession, blocked: Array, ts: numb LoginStore.updateSession(state); } -export function setFollows(state: LoginSession, follows: Array, ts: number) { - if (state.follows.timestamp >= ts) { - return; +export function setFollows(id: string, follows: Array, ts: number) { + const session = LoginStore.get(id); + if (session) { + if (ts > session.follows.timestamp) { + session.follows.item = follows; + session.follows.timestamp = ts; + LoginStore.updateSession(session); + } } - state.follows.item = follows; - state.follows.timestamp = ts; - LoginStore.updateSession(state); } export function setPinned(state: LoginSession, pinned: Array, ts: number) { diff --git a/packages/app/src/Pages/settings/Menu.tsx b/packages/app/src/Pages/settings/Menu.tsx index acd363319..e2649f6e6 100644 --- a/packages/app/src/Pages/settings/Menu.tsx +++ b/packages/app/src/Pages/settings/Menu.tsx @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { ReactNode, useCallback } from "react"; import { FormattedMessage } from "react-intl"; import { Link, useNavigate } from "react-router-dom"; import Icon from "@/Icons/Icon"; @@ -7,6 +7,17 @@ import useLogin from "@/Hooks/useLogin"; import classNames from "classnames"; import { getCurrentSubscription } from "@/Subscription"; +export type SettingsMenuItems = Array<{ + title: ReactNode, + items: Array<{ + icon: string; + iconBg: string; + message: ReactNode, + path?: string; + action?: () => void; + }> +}>; + const SettingsIndex = () => { const login = useLogin(); const navigate = useNavigate(); @@ -53,14 +64,20 @@ const SettingsIndex = () => { }, ...(sub ? [ - { - icon: "code-circle", - iconBg: "bg-indigo-500", - message: , - path: "accounts", - }, - ] + { + icon: "code-circle", + iconBg: "bg-indigo-500", + message: , + path: "accounts", + }, + ] : []), + { + icon: "tool", + iconBg: "bg-slate-800", + message: , + path: "tools" + } ], }, { @@ -109,23 +126,23 @@ const SettingsIndex = () => { }, ...(CONFIG.features.subscriptions ? [ - { - icon: "diamond", - iconBg: "bg-violet-500", - message: , - path: "/subscribe/manage", - }, - ] + { + icon: "diamond", + iconBg: "bg-violet-500", + message: , + path: "/subscribe/manage", + }, + ] : []), ...(CONFIG.features.zapPool ? [ - { - icon: "piggy-bank", - iconBg: "bg-rose-500", - message: , - path: "/zap-pool", - }, - ] + { + icon: "piggy-bank", + iconBg: "bg-rose-500", + message: , + path: "/zap-pool", + }, + ] : []), ], }, @@ -140,11 +157,15 @@ const SettingsIndex = () => { }, ], }, - ]; + ] as SettingsMenuItems; + return +}; + +export function SettingsMenuComponent({ menu }: { menu: SettingsMenuItems }) { return (
- {settingsGroups.map((group, groupIndex) => ( + {menu.map((group, groupIndex) => (
{group.title}
{group.items.map(({ icon, iconBg, message, path, action }, index) => ( @@ -152,7 +173,6 @@ const SettingsIndex = () => { to={path || "#"} onClick={action} key={path || index} - end className={classNames("px-2.5 py-1.5 flex justify-between items-center border border-border-color", { "rounded-t-xl": index === 0, "rounded-b-xl": index === group.items.length - 1, @@ -160,7 +180,7 @@ const SettingsIndex = () => { })}>
- +
{message}
@@ -171,6 +191,6 @@ const SettingsIndex = () => { ))}
); -}; +} export default SettingsIndex; diff --git a/packages/app/src/Pages/settings/Routes.tsx b/packages/app/src/Pages/settings/Routes.tsx index 9bb75bf6e..7b81020d6 100644 --- a/packages/app/src/Pages/settings/Routes.tsx +++ b/packages/app/src/Pages/settings/Routes.tsx @@ -12,6 +12,7 @@ import ModerationSettings from "@/Pages/settings/Moderation"; import { CacheSettings } from "@/Pages/settings/Cache"; import { ReferralsPage } from "@/Pages/settings/Referrals"; import { Outlet } from "react-router-dom"; +import { ToolsPage, ToolsPages } from "./tools"; const SettingsPage = () => { return ( @@ -70,6 +71,11 @@ export default [ path: "invite", element: , }, + { + path: "tools", + element: , + children: ToolsPages, + }, ...ManageHandleRoutes, ...WalletSettingsRoutes, ], diff --git a/packages/app/src/Pages/settings/tools/follows-relay-health.tsx b/packages/app/src/Pages/settings/tools/follows-relay-health.tsx new file mode 100644 index 000000000..c4dbcbf7e --- /dev/null +++ b/packages/app/src/Pages/settings/tools/follows-relay-health.tsx @@ -0,0 +1,58 @@ +import { CollapsedSection } from "@/Element/Collapsed"; +import ProfilePreview from "@/Element/User/ProfilePreview"; +import useLogin from "@/Hooks/useLogin"; +import { getRelayName } from "@/SnortUtils"; +import { dedupe } from "@snort/shared"; +import { pickTopRelays } from "@snort/system"; +import { SnortContext } from "@snort/system-react"; +import { ReactNode, useContext, useMemo } from "react"; +import { FormattedMessage, FormattedNumber } from "react-intl"; + +export function FollowsRelayHealth({ withTitle, popularRelays, missingRelaysActions }: { withTitle?: boolean, popularRelays?: boolean, missingRelaysActions?: (k: string) => ReactNode }) { + const system = useContext(SnortContext); + const follows = useLogin(s => s.follows); + const uniqueFollows = dedupe(follows.item); + + const hasRelays = useMemo(() => { + return uniqueFollows.filter(a => (system.RelayCache.getFromCache(a)?.relays.length ?? 0) > 0); + }, [uniqueFollows]); + + const missingRelays = useMemo(() => { + return uniqueFollows.filter(a => !hasRelays.includes(a)); + }, [hasRelays]); + + const topWriteRelays = useMemo(() => { + return pickTopRelays(system.RelayCache, uniqueFollows, 1e31, "write"); + }, [uniqueFollows]); + + return
+ {(withTitle ?? true) &&
+ +
} +
+ + }} /> +
+ {missingRelays.length > 0 &&
}> +
+ {missingRelays.map(a => )} +
+ } + {(popularRelays ?? true) &&
+
Popular Relays
+ {dedupe(topWriteRelays.flatMap(a => a.relays)) + .map(a => ({ relay: a, count: topWriteRelays.filter(b => b.relays.includes(a)).length })) + .sort((a, b) => a.count > b.count ? -1 : 1) + .slice(0, 10) + .map(a =>
+
{getRelayName(a.relay)}
+
{a.count} ()
+
)} +
} +
+} \ No newline at end of file diff --git a/packages/app/src/Pages/settings/tools/index.tsx b/packages/app/src/Pages/settings/tools/index.tsx new file mode 100644 index 000000000..565edca20 --- /dev/null +++ b/packages/app/src/Pages/settings/tools/index.tsx @@ -0,0 +1,50 @@ +import { FormattedMessage } from "react-intl" +import { Outlet, RouteObject } from "react-router-dom" +import { SettingsMenuComponent, SettingsMenuItems } from "../Menu" +import { PruneFollowList } from "./prune-follows"; +import { FollowsRelayHealth } from "./follows-relay-health"; + + +const ToolMenuItems = [ + { + title: , + items: [ + { + icon: "trash", + iconBg: "bg-red-500", + message: , + path: "prune-follows" + }, + { + icon: "medical-cross", + iconBg: "bg-green-800", + message: , + path: "follows-relay-health" + } + ] + } +] as SettingsMenuItems; + +export const ToolsPages = [ + { + path: "", + element: <> +

+ +

+ + + }, + { + path: "prune-follows", + element: + }, + { + path: "follows-relay-health", + element: + } +] as Array + +export function ToolsPage() { + return +} diff --git a/packages/app/src/Pages/settings/tools/prune-follows.tsx b/packages/app/src/Pages/settings/tools/prune-follows.tsx new file mode 100644 index 000000000..d7ed4ecec --- /dev/null +++ b/packages/app/src/Pages/settings/tools/prune-follows.tsx @@ -0,0 +1,136 @@ +import { Day } from "@/Const"; +import AsyncButton from "@/Element/Button/AsyncButton"; +import useLogin from "@/Hooks/useLogin" +import { dedupe, unixNow } from "@snort/shared"; +import { RequestBuilder } from "@snort/system"; +import { useMemo, useState } from "react"; +import { FormattedMessage, FormattedNumber } from "react-intl"; +import { FollowsRelayHealth } from "./follows-relay-health"; +import ProfileImage from "@/Element/User/ProfileImage"; +import useEventPublisher from "@/Hooks/useEventPublisher"; +import { setFollows } from "@/Login"; + +const enum PruneStage { + FetchLastPostTimestamp, + Done +} + +export function PruneFollowList() { + const { id, follows } = useLogin(s => ({ id: s.id, follows: s.follows })); + const { publisher, system } = useEventPublisher(); + const uniqueFollows = dedupe(follows.item); + const [status, setStatus] = useState(); + const [progress, setProgress] = useState(0); + const [lastPost, setLastPosts] = useState>(); + const [unfollow, setUnfollow] = useState>([]); + + async function fetchLastPosts() { + setStatus(PruneStage.FetchLastPostTimestamp); + setProgress(0); + setLastPosts(undefined); + + const BatchSize = 10; + const chunks = uniqueFollows.reduce((acc, v, i) => { + const batch = Math.floor(i / BatchSize).toString(); + acc[batch] ??= []; + acc[batch].push(v); + return acc; + }, {} as Record>); + + const result = {} as Record; + const batches = Math.ceil(uniqueFollows.length / BatchSize); + for (const [batch, pubkeys] of Object.entries(chunks)) { + console.debug(batch, pubkeys); + const req = new RequestBuilder(`prune-${batch}`); + req.withOptions({ + outboxPickN: 10, + timeout: 10_000 + }); + pubkeys.forEach(p => req.withFilter().limit(1).kinds([0, 1, 3, 5, 6, 7, 10002]).authors([p])); + const results = await system.Fetch(req); + console.debug(results); + for (const rx of results) { + if ((result[rx.pubkey] ?? 0) < rx.created_at) { + result[rx.pubkey] = rx.created_at; + } + } + setProgress(Number(batch) / batches); + } + + for (const pk of uniqueFollows) { + result[pk] ??= 0; + } + setLastPosts(result); + setStatus(PruneStage.Done); + } + + const newFollowList = useMemo(() => { + return uniqueFollows.filter(a => !unfollow.includes(a) && a.length === 64); + }, [uniqueFollows, unfollow]); + + async function publishFollowList() { + const newFollows = newFollowList.map(a => ["p", a]) as Array<[string, string]>; + if (publisher) { + const ev = await publisher.contactList(newFollows); + await system.BroadcastEvent(ev); + setFollows(id, newFollowList, ev.created_at * 1000); + } + } + + + function getStatus() { + switch (status) { + case PruneStage.FetchLastPostTimestamp: return + }} /> + } + } + + function personToggle(k: string,) { + return
+ setUnfollow(v => e.target.checked ? dedupe([...v, k]) : v.filter(a => a !== k))} checked={unfollow.includes(k)} /> + +
+ } + + return
+
+ +
+

+ +

+
+ +
+ personToggle(k)} /> + + + + {getStatus()} +
+ {lastPost && Object.entries(lastPost).filter(([, v]) => v <= unixNow() - (90 * Day)).sort(([, a], [, b]) => a > b ? -1 : 1).map(([k, v]) => { + return
+ +
+ + {personToggle(k)} +
+
+ })} +
+
+

+ +

+ + + +
+
+} \ No newline at end of file diff --git a/packages/app/src/icons.svg b/packages/app/src/icons.svg index 3cba64a72..fce18f1e6 100644 --- a/packages/app/src/icons.svg +++ b/packages/app/src/icons.svg @@ -436,8 +436,8 @@ - - + + @@ -448,5 +448,11 @@ + + + + + + \ No newline at end of file diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index fabd1368e..d4d8038e3 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -300,7 +300,7 @@ export class NostrSystem extends EventEmitter implements Syst const store = new type(); const filters = req.build(this); - const q = new Query(req.id, req.instance, store, req.options?.leaveOpen); + const q = new Query(req.id, req.instance, store, req.options?.leaveOpen, req.options?.timeout); q.on("trace", r => this.#relayMetrics.onTraceReport(r)); if (filters.some(a => a.filters.some(b => b.ids))) { diff --git a/packages/system/src/outbox-model.ts b/packages/system/src/outbox-model.ts index 8320f6870..b907c2d1d 100644 --- a/packages/system/src/outbox-model.ts +++ b/packages/system/src/outbox-model.ts @@ -14,7 +14,7 @@ import { FlatReqFilter } from "./query-optimizer"; import { RelayListCacheExpire } from "./const"; import { BackgroundLoader } from "./background-loader"; -const PickNRelays = 2; +const DefaultPickNRelays = 2; export interface RelayTaggedFilter { relay: string; @@ -66,7 +66,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array { +export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter, pickN?: number): Array { const authors = filter.authors; if ((authors?.length ?? 0) === 0) { return [ @@ -77,7 +77,7 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array< ]; } - const topRelays = pickTopRelays(cache, unwrap(authors), PickNRelays, "write"); + const topRelays = pickTopRelays(cache, unwrap(authors), pickN ?? DefaultPickNRelays, "write"); const pickedRelays = dedupe(topRelays.flatMap(a => a.relays)); const picked = pickedRelays.map(a => { @@ -107,7 +107,11 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array< /** * Split filters by author */ -export function splitFlatByWriteRelays(cache: RelayCache, input: Array): Array { +export function splitFlatByWriteRelays( + cache: RelayCache, + input: Array, + pickN?: number, +): Array { const authors = input.filter(a => a.authors).map(a => unwrap(a.authors)); if (authors.length === 0) { return [ @@ -117,7 +121,7 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array a.relays)); const picked = pickedRelays.map(a => { @@ -142,7 +146,7 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array, n: number, type: "write" | "read") { +export function pickTopRelays(cache: RelayCache, authors: Array, n: number, type: "write" | "read") { // map of pubkey -> [write relays] const allRelays = authors.map(a => { return { @@ -198,10 +202,10 @@ function pickTopRelays(cache: RelayCache, authors: Array, n: number, typ /** * Pick read relays for sending reply events */ -export async function pickRelaysForReply(ev: NostrEvent, system: SystemInterface) { +export async function pickRelaysForReply(ev: NostrEvent, system: SystemInterface, pickN?: number) { const recipients = dedupe(ev.tags.filter(a => a[0] === "p").map(a => a[1])); await updateRelayLists(recipients, system); - const relays = pickTopRelays(system.RelayCache, recipients, 2, "read"); + const relays = pickTopRelays(system.RelayCache, recipients, pickN ?? DefaultPickNRelays, "read"); const ret = removeUndefined(dedupe(relays.map(a => a.relays).flat())); logger("Picked %O from authors %O", ret, recipients); return ret; @@ -221,6 +225,27 @@ export function parseRelayTags(tag: Array>) { return tag.map(parseRelayTag).filter(a => a !== null); } +export function parseRelaysFromKind(ev: NostrEvent) { + if (ev.kind === EventKind.ContactList) { + const relaysInContent = + ev.content.length > 0 ? (JSON.parse(ev.content) as Record) : undefined; + if (relaysInContent) { + return Object.entries(relaysInContent).map( + ([k, v]) => + ({ + url: sanitizeRelayUrl(k), + settings: { + read: v.read, + write: v.write, + }, + }) as FullRelaySettings, + ); + } + } else if (ev.kind === EventKind.Relays) { + return parseRelayTags(ev.tags); + } +} + export async function updateRelayLists(authors: Array, system: SystemInterface) { await system.RelayCache.buffer(authors); const expire = unixNowMs() - RelayListCacheExpire; @@ -228,15 +253,21 @@ export async function updateRelayLists(authors: Array, system: SystemInt if (expired.length > 0) { logger("Updating relays for authors: %O", expired); const rb = new RequestBuilder("system-update-relays-for-outbox"); - rb.withFilter().authors(expired).kinds([EventKind.Relays]); + rb.withFilter().authors(expired).kinds([EventKind.Relays, EventKind.ContactList]); const relayLists = await system.Fetch(rb); await system.RelayCache.bulkSet( - relayLists.map(a => ({ - relays: parseRelayTags(a.tags), - pubkey: a.pubkey, - created: a.created_at, - loaded: unixNowMs(), - })), + removeUndefined( + relayLists.map(a => { + const relays = parseRelaysFromKind(a); + if (!relays) return; + return { + relays: relays, + pubkey: a.pubkey, + created: a.created_at, + loaded: unixNowMs(), + }; + }), + ), ); } } @@ -247,8 +278,10 @@ export class RelayMetadataLoader extends BackgroundLoader { } override onEvent(e: Readonly): UsersRelays | undefined { + const relays = parseRelaysFromKind(e); + if (!relays) return; return { - relays: parseRelayTags(e.tags), + relays: relays, pubkey: e.pubkey, created: e.created_at, loaded: unixNowMs(), @@ -261,8 +294,12 @@ export class RelayMetadataLoader extends BackgroundLoader { protected override buildSub(missing: string[]): RequestBuilder { const rb = new RequestBuilder("relay-loader"); - rb.withOptions({ skipDiff: true }); - rb.withFilter().authors(missing).kinds([EventKind.Relays]); + rb.withOptions({ + skipDiff: true, + timeout: 10_000, + outboxPickN: 4, + }); + rb.withFilter().authors(missing).kinds([EventKind.Relays, EventKind.ContactList]); return rb; } diff --git a/packages/system/src/query-optimizer/index.ts b/packages/system/src/query-optimizer/index.ts index 122206651..1b1d2f865 100644 --- a/packages/system/src/query-optimizer/index.ts +++ b/packages/system/src/query-optimizer/index.ts @@ -18,6 +18,7 @@ export interface FlatReqFilter { since?: number; until?: number; limit?: number; + resultSetId: string; } export interface QueryOptimizer { diff --git a/packages/system/src/query-optimizer/request-expander.ts b/packages/system/src/query-optimizer/request-expander.ts index 3dfdb5ef7..6ceb23104 100644 --- a/packages/system/src/query-optimizer/request-expander.ts +++ b/packages/system/src/query-optimizer/request-expander.ts @@ -1,3 +1,4 @@ +import { sha256 } from "@snort/shared"; import { FlatReqFilter } from "."; import { ReqFilter } from "../nostr"; @@ -7,29 +8,52 @@ import { ReqFilter } from "../nostr"; export function expandFilter(f: ReqFilter): Array { const ret: Array = []; const src = Object.entries(f); - const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]); - const props = src.filter(([, v]) => !Array.isArray(v)); - function generateCombinations(index: number, currentCombination: FlatReqFilter) { - if (index === keys.length) { - ret.push(currentCombination); + const id = resultSetId(f); + + // Filter entries that are arrays and keep the rest as is + const arrays: [string, Array | Array][] = src.filter(([, value]) => Array.isArray(value)) as [ + string, + Array | Array, + ][]; + const constants = Object.fromEntries(src.filter(([, value]) => !Array.isArray(value))) as { + [key: string]: string | number | undefined; + }; + + // Recursive function to compute cartesian product + function cartesianProduct(arr: [string, Array | Array][], temp: [string, any][] = []) { + if (arr.length === 0) { + ret.push(createFilterObject(temp, constants, id)); return; } - - const key = keys[index]; - const values = (f as Record>)[key]; - - for (let i = 0; i < values.length; i++) { - const value = values[i]; - const updatedCombination = { ...currentCombination, [key]: value }; - generateCombinations(index + 1, updatedCombination); + for (let i = 0; i < arr[0][1].length; i++) { + cartesianProduct(arr.slice(1), temp.concat([[arr[0][0], arr[0][1][i]]])); } } - generateCombinations(0, { - keys: keys.length, - ...Object.fromEntries(props), - }); + // Create filter object from the combination + function createFilterObject( + combination: [string, any][], + constants: { [key: string]: string | number | undefined }, + resultId: string, + ) { + let filterObject = { ...Object.fromEntries(combination), ...constants } as FlatReqFilter; + filterObject.resultSetId = resultId; + return filterObject; + } + cartesianProduct(arrays); return ret; } + +function resultSetId(f: ReqFilter) { + if (f.limit !== undefined || f.since !== undefined || f.until !== undefined) { + const arrays = Object.entries(f) + .filter(([, a]) => Array.isArray(a)) + .map(a => a as [string, Array]) + .sort(); + const input = arrays.map(([, a]) => a.join(",")).join(","); + return sha256(input); + } + return ""; +} diff --git a/packages/system/src/query-optimizer/request-merger.ts b/packages/system/src/query-optimizer/request-merger.ts index 5ed3900e1..d1b26bca9 100644 --- a/packages/system/src/query-optimizer/request-merger.ts +++ b/packages/system/src/query-optimizer/request-merger.ts @@ -2,18 +2,9 @@ import { distance } from "@snort/shared"; import { ReqFilter } from ".."; import { FlatReqFilter } from "."; -/** - * Keys which can change the entire meaning of the filter outside the array types - */ -const DiscriminatorKeys = ["since", "until", "limit", "search"]; - export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean { - const aObj = a as Record; - const bObj = b as Record; - for (const key of DiscriminatorKeys) { - if (aObj[key] !== bObj[key]) { - return false; - } + if (a.resultSetId !== b.resultSetId) { + return false; } return distance(a, b) <= 1; } @@ -101,12 +92,11 @@ export function flatMerge(all: Array): Array { // to compute filters which can be merged we need to calucate the distance change between each filter // then we can merge filters which are exactly 1 change diff from each other - function mergeFiltersInSet(filters: Array) { return filters.reduce((acc, a) => { Object.entries(a).forEach(([k, v]) => { - if (k === "keys" || v === undefined) return; - if (DiscriminatorKeys.includes(k)) { + if (v === undefined) return; + if (k === "since" || k === "until" || k === "limit" || k === "search" || k === "resultSetId") { acc[k] = v; } else { acc[k] ??= []; @@ -142,5 +132,6 @@ export function flatMerge(all: Array): Array { } ret = n; } + ret.forEach(a => delete a["resultSetId"]); return ret; } diff --git a/packages/system/src/query.ts b/packages/system/src/query.ts index 3dc389cb9..656988f29 100644 --- a/packages/system/src/query.ts +++ b/packages/system/src/query.ts @@ -157,14 +157,20 @@ export class Query extends EventEmitter implements QueryBase { */ #feed: NoteStore; + /** + * Maximum waiting time for this query + */ + #timeout: number; + #log = debug("Query"); - constructor(id: string, instance: string, feed: NoteStore, leaveOpen?: boolean) { + constructor(id: string, instance: string, feed: NoteStore, leaveOpen?: boolean, timeout?: number) { super(); this.id = id; this.#feed = feed; this.fromInstance = instance; this.#leaveOpen = leaveOpen ?? false; + this.#timeout = timeout ?? 5_000; this.#checkTraces(); } @@ -292,7 +298,7 @@ export class Query extends EventEmitter implements QueryBase { this.#stopCheckTraces(); this.#checkTrace = setInterval(() => { for (const v of this.#tracing) { - if (v.runtime > 5_000 && !v.finished) { + if (v.runtime > this.#timeout && !v.finished) { v.forceEose(); } } diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts index fccdff280..bbb8db6b0 100644 --- a/packages/system/src/request-builder.ts +++ b/packages/system/src/request-builder.ts @@ -43,6 +43,16 @@ export interface RequestBuilderOptions { * Do not apply diff logic and always use full filters for query */ skipDiff?: boolean; + + /** + * Pick N relays per pubkey when using outbox strategy + */ + outboxPickN?: number; + + /** + * Max wait time for this request + */ + timeout?: number; } /** @@ -101,7 +111,7 @@ export class RequestBuilder { } build(system: SystemInterface): Array { - const expanded = this.#builders.flatMap(a => a.build(system.RelayCache, this.id)); + const expanded = this.#builders.flatMap(a => a.build(system.RelayCache, this.#options)); return this.#groupByRelay(system, expanded); } @@ -130,11 +140,9 @@ export class RequestBuilder { /** * Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests - * @param expanded - * @returns */ - #groupByRelay(system: SystemInterface, expanded: Array) { - const relayMerged = expanded.reduce((acc, v) => { + #groupByRelay(system: SystemInterface, filters: Array) { + const relayMerged = filters.reduce((acc, v) => { const existing = acc.get(v.relay); if (existing) { existing.push(v); @@ -267,7 +275,7 @@ export class RequestFilterBuilder { /** * Build/expand this filter into a set of relay specific queries */ - build(relays: RelayCache, id: string): Array { + build(relays: RelayCache, options?: RequestBuilderOptions): Array { // use the explicit relay list first if (this.#relays.size > 0) { return [...this.#relays].map(r => { @@ -281,7 +289,7 @@ export class RequestFilterBuilder { // If any authors are set use the gossip model to fetch data for each author if (this.#filter.authors) { - const split = splitByWriteRelays(relays, this.#filter); + const split = splitByWriteRelays(relays, this.#filter, options?.outboxPickN); return split.map(a => { return { filters: [a.filter], diff --git a/packages/system/tests/gossip-model.test.ts b/packages/system/tests/outbox-model.test.ts similarity index 59% rename from packages/system/tests/gossip-model.test.ts rename to packages/system/tests/outbox-model.test.ts index 8379b9cc4..b8698c7e0 100644 --- a/packages/system/tests/gossip-model.test.ts +++ b/packages/system/tests/outbox-model.test.ts @@ -1,17 +1,27 @@ -import { splitAllByWriteRelays } from "../src/gossip-model"; +import { splitAllByWriteRelays } from "../src/outbox-model"; -describe("GossipModel", () => { +describe("OutboxModel", () => { it("should not output empty", () => { const Relays = { getFromCache: (pk?: string) => { if (pk) { return { pubkey: pk, - created_at: 0, + created: 0, + loaded: 0, relays: [], }; } }, + update: () => { + return Promise.resolve<"new" | "updated" | "refresh" | "no_change">("new"); + }, + buffer: () => { + return Promise.resolve>([]); + }, + bulkSet: () => { + return Promise.resolve(); + }, }; const a = [ { diff --git a/packages/system/tests/request-builder.test.ts b/packages/system/tests/request-builder.test.ts index f28531193..5d3bc18c2 100644 --- a/packages/system/tests/request-builder.test.ts +++ b/packages/system/tests/request-builder.test.ts @@ -1,4 +1,4 @@ -import { RelayCache } from "../src/gossip-model"; +import { RelayCache } from "../src/outbox-model"; import { RequestBuilder, RequestStrategy } from "../src/request-builder"; import { describe, expect } from "@jest/globals"; import { bytesToHex } from "@noble/curves/abstract/utils"; @@ -23,7 +23,16 @@ const DummyCache = { ], }; }, -} as FeedCache; + update: () => { + return Promise.resolve<"new" | "updated" | "refresh" | "no_change">("new"); + }, + buffer: () => { + return Promise.resolve>([]); + }, + bulkSet: () => { + return Promise.resolve(); + }, +} as unknown as FeedCache; const System = new NostrSystem({ relayCache: DummyCache, @@ -112,7 +121,7 @@ describe("RequestBuilder", () => { rb.withFilter().authors(["a", "b"]).kinds([0]); const a = rb.build(System); - expect(a).toEqual([ + expect(a).toMatchObject([ { strategy: RequestStrategy.AuthorsRelays, relay: "wss://a.com/", @@ -143,7 +152,7 @@ describe("RequestBuilder", () => { rb.withFilter().authors(["a"]).limit(10).kinds([4]); const a = rb.build(System); - expect(a).toEqual([ + expect(a).toMatchObject([ { strategy: RequestStrategy.AuthorsRelays, relay: "wss://a.com/", diff --git a/packages/system/tests/request-expander.test.ts b/packages/system/tests/request-expander.test.ts index d9b96b1d5..13061a74e 100644 --- a/packages/system/tests/request-expander.test.ts +++ b/packages/system/tests/request-expander.test.ts @@ -10,25 +10,25 @@ describe("RequestExpander", () => { since: 99, limit: 10, }; - expect(expandFilter(a)).toEqual([ - { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 }, - { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 }, + expect(expandFilter(a)).toMatchObject([ + { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, ]); }); }); diff --git a/packages/system/tests/request-merger.test.ts b/packages/system/tests/request-merger.test.ts index dcc116339..81f85bf6b 100644 --- a/packages/system/tests/request-merger.test.ts +++ b/packages/system/tests/request-merger.test.ts @@ -81,25 +81,27 @@ describe("RequestMerger", () => { describe("flatMerge", () => { it("should flat merge simple", () => { const input = [ - { ids: 0, authors: "a" }, - { ids: 0, authors: "b" }, - { kinds: 1 }, - { kinds: 2 }, - { kinds: 2 }, - { ids: 0, authors: "c" }, - { authors: "c", kinds: 1 }, - { authors: "c", limit: 100 }, - { ids: 1, authors: "c" }, + { ids: "0", authors: "a", resultSetId: "" }, + { ids: "0", authors: "b", resultSetId: "" }, + { kinds: 1, resultSetId: "" }, + { kinds: 2, resultSetId: "" }, + { kinds: 2, resultSetId: "" }, + { ids: "0", authors: "c", resultSetId: "" }, + { authors: "c", kinds: 1, resultSetId: "" }, + { authors: "c", limit: 100, resultSetId: "limit-c-100" }, + { authors: "b", limit: 100, resultSetId: "limit-b-100" }, + { ids: "1", authors: "c", resultSetId: "" }, ] as Array; const output = [ - { ids: [0], authors: ["a", "b", "c"] }, + { ids: ["0"], authors: ["a", "b", "c"] }, { kinds: [1, 2] }, { authors: ["c"], kinds: [1] }, { authors: ["c"], limit: 100 }, - { ids: [1], authors: ["c"] }, + { authors: ["b"], limit: 100 }, + { ids: ["1"], authors: ["c"] }, ] as Array; - expect(flatMerge(input)).toEqual(output); + expect(flatMerge(input)).toMatchObject(output); }); it("should expand and flat merge complex same", () => { @@ -119,47 +121,47 @@ describe("canMerge", () => { it("should have 0 distance", () => { const a = { ids: "a", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; const b = { ids: "a", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; expect(canMergeFilters(a, b)).toEqual(true); }); it("should have 1 distance", () => { const a = { ids: "a", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; const b = { ids: "b", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; expect(canMergeFilters(a, b)).toEqual(true); }); it("should have 10 distance", () => { const a = { ids: "a", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; const b = { ids: "a", kinds: 1, - keys: 2, - }; + resultSetId: "", + } as FlatReqFilter; expect(canMergeFilters(a, b)).toEqual(false); }); it("should have 11 distance", () => { const a = { ids: "a", - keys: 1, - }; + resultSetId: "", + } as FlatReqFilter; const b = { ids: "b", kinds: 1, - keys: 2, - }; + resultSetId: "", + } as FlatReqFilter; expect(canMergeFilters(a, b)).toEqual(false); }); it("should have 1 distance, arrays", () => {