forked from Kieran/snort
feat: tools pages
Various other fixes: - Better handeling of limit/since/before merging - Expose timeout through request builder - Expose PickN through request builder - Fix tests
This commit is contained in:
parent
96368d4a2b
commit
06b7dcad11
GPG Key ID:
DE71CEB3925BE941
@ -27,8 +27,10 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
|
||||
}
|
||||
|
||||
buildSub(session: LoginSession, rb: RequestBuilder): void {
|
||||
const authors = session.follows.item;
|
||||
authors.push(session.publicKey);
|
||||
const authors = [...session.follows.item];
|
||||
if (session.publicKey) {
|
||||
authors.push(session.publicKey);
|
||||
}
|
||||
const since = this.newest();
|
||||
rb.withFilter()
|
||||
.kinds(this.#kinds)
|
||||
@ -69,8 +71,10 @@ export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
|
||||
async loadMore(system: SystemInterface, session: LoginSession, before: number) {
|
||||
if (this.#oldest && before <= this.#oldest) {
|
||||
const rb = new RequestBuilder(`${this.name}-loadmore`);
|
||||
const authors = session.follows.item;
|
||||
authors.push(session.publicKey);
|
||||
const authors = [...session.follows.item];
|
||||
if (session.publicKey) {
|
||||
authors.push(session.publicKey);
|
||||
}
|
||||
rb.withFilter()
|
||||
.kinds(this.#kinds)
|
||||
.authors(authors)
|
||||
|
||||
@ -32,13 +32,13 @@ export default function FollowListBase({
|
||||
profileActions,
|
||||
}: FollowListBaseProps) {
|
||||
const { publisher, system } = useEventPublisher();
|
||||
const login = useLogin();
|
||||
const { id, follows } = useLogin(s => ({ id: s.id, follows: s.follows }));
|
||||
|
||||
async function followAll() {
|
||||
if (publisher) {
|
||||
const newFollows = dedupe([...pubkeys, ...login.follows.item]);
|
||||
const newFollows = dedupe([...pubkeys, ...follows.item]);
|
||||
const ev = await publisher.contactList(newFollows.map(a => ["p", a]));
|
||||
setFollows(login, newFollows, ev.created_at);
|
||||
setFollows(id, newFollows, ev.created_at);
|
||||
await system.BroadcastEvent(ev);
|
||||
await FollowsFeed.backFill(system, pubkeys);
|
||||
}
|
||||
|
||||
@ -104,7 +104,7 @@ export default function useLoginFeed() {
|
||||
const contactList = getNewest(loginFeed.data.filter(a => a.kind === EventKind.ContactList));
|
||||
if (contactList) {
|
||||
const pTags = contactList.tags.filter(a => a[0] === "p").map(a => a[1]);
|
||||
setFollows(login, pTags, contactList.created_at * 1000);
|
||||
setFollows(login.id, pTags, contactList.created_at * 1000);
|
||||
|
||||
FollowsFeed.backFillIfMissing(system, pTags);
|
||||
}
|
||||
|
||||
@ -177,13 +177,15 @@ export function setBlocked(state: LoginSession, blocked: Array<string>, ts: numb
|
||||
LoginStore.updateSession(state);
|
||||
}
|
||||
|
||||
export function setFollows(state: LoginSession, follows: Array<string>, ts: number) {
|
||||
if (state.follows.timestamp >= ts) {
|
||||
return;
|
||||
export function setFollows(id: string, follows: Array<string>, ts: number) {
|
||||
const session = LoginStore.get(id);
|
||||
if (session) {
|
||||
if (ts > session.follows.timestamp) {
|
||||
session.follows.item = follows;
|
||||
session.follows.timestamp = ts;
|
||||
LoginStore.updateSession(session);
|
||||
}
|
||||
}
|
||||
state.follows.item = follows;
|
||||
state.follows.timestamp = ts;
|
||||
LoginStore.updateSession(state);
|
||||
}
|
||||
|
||||
export function setPinned(state: LoginSession, pinned: Array<string>, ts: number) {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { useCallback } from "react";
|
||||
import { ReactNode, useCallback } from "react";
|
||||
import { FormattedMessage } from "react-intl";
|
||||
import { Link, useNavigate } from "react-router-dom";
|
||||
import Icon from "@/Icons/Icon";
|
||||
@ -7,6 +7,17 @@ import useLogin from "@/Hooks/useLogin";
|
||||
import classNames from "classnames";
|
||||
import { getCurrentSubscription } from "@/Subscription";
|
||||
|
||||
export type SettingsMenuItems = Array<{
|
||||
title: ReactNode,
|
||||
items: Array<{
|
||||
icon: string;
|
||||
iconBg: string;
|
||||
message: ReactNode,
|
||||
path?: string;
|
||||
action?: () => void;
|
||||
}>
|
||||
}>;
|
||||
|
||||
const SettingsIndex = () => {
|
||||
const login = useLogin();
|
||||
const navigate = useNavigate();
|
||||
@ -53,14 +64,20 @@ const SettingsIndex = () => {
|
||||
},
|
||||
...(sub
|
||||
? [
|
||||
{
|
||||
icon: "code-circle",
|
||||
iconBg: "bg-indigo-500",
|
||||
message: <FormattedMessage id="FvanT6" defaultMessage="Accounts" />,
|
||||
path: "accounts",
|
||||
},
|
||||
]
|
||||
{
|
||||
icon: "code-circle",
|
||||
iconBg: "bg-indigo-500",
|
||||
message: <FormattedMessage id="FvanT6" defaultMessage="Accounts" />,
|
||||
path: "accounts",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
icon: "tool",
|
||||
iconBg: "bg-slate-800",
|
||||
message: <FormattedMessage defaultMessage="Tools" id="nUT0Lv" />,
|
||||
path: "tools"
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
@ -109,23 +126,23 @@ const SettingsIndex = () => {
|
||||
},
|
||||
...(CONFIG.features.subscriptions
|
||||
? [
|
||||
{
|
||||
icon: "diamond",
|
||||
iconBg: "bg-violet-500",
|
||||
message: <FormattedMessage id="R/6nsx" defaultMessage="Subscription" />,
|
||||
path: "/subscribe/manage",
|
||||
},
|
||||
]
|
||||
{
|
||||
icon: "diamond",
|
||||
iconBg: "bg-violet-500",
|
||||
message: <FormattedMessage id="R/6nsx" defaultMessage="Subscription" />,
|
||||
path: "/subscribe/manage",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(CONFIG.features.zapPool
|
||||
? [
|
||||
{
|
||||
icon: "piggy-bank",
|
||||
iconBg: "bg-rose-500",
|
||||
message: <FormattedMessage id="i/dBAR" defaultMessage="Zap Pool" />,
|
||||
path: "/zap-pool",
|
||||
},
|
||||
]
|
||||
{
|
||||
icon: "piggy-bank",
|
||||
iconBg: "bg-rose-500",
|
||||
message: <FormattedMessage id="i/dBAR" defaultMessage="Zap Pool" />,
|
||||
path: "/zap-pool",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
},
|
||||
@ -140,11 +157,15 @@ const SettingsIndex = () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
] as SettingsMenuItems;
|
||||
|
||||
return <SettingsMenuComponent menu={settingsGroups} />
|
||||
};
|
||||
|
||||
export function SettingsMenuComponent({ menu }: { menu: SettingsMenuItems }) {
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
{settingsGroups.map((group, groupIndex) => (
|
||||
{menu.map((group, groupIndex) => (
|
||||
<div key={groupIndex} className="mb-4">
|
||||
<div className="p-2 font-bold uppercase text-secondary text-xs tracking-wide">{group.title}</div>
|
||||
{group.items.map(({ icon, iconBg, message, path, action }, index) => (
|
||||
@ -152,7 +173,6 @@ const SettingsIndex = () => {
|
||||
to={path || "#"}
|
||||
onClick={action}
|
||||
key={path || index}
|
||||
end
|
||||
className={classNames("px-2.5 py-1.5 flex justify-between items-center border border-border-color", {
|
||||
"rounded-t-xl": index === 0,
|
||||
"rounded-b-xl": index === group.items.length - 1,
|
||||
@ -160,7 +180,7 @@ const SettingsIndex = () => {
|
||||
})}>
|
||||
<div className="flex items-center gap-3">
|
||||
<div className={`p-1 ${iconBg} rounded-lg flex justify-center items-center text-white`}>
|
||||
<Icon name={icon} size={16} className="w-4 h-4 relative" />
|
||||
<Icon name={icon} size={18} className="relative" />
|
||||
</div>
|
||||
<span className="text-base font-semibold flex-grow">{message}</span>
|
||||
</div>
|
||||
@ -171,6 +191,6 @@ const SettingsIndex = () => {
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
export default SettingsIndex;
|
||||
|
||||
@ -12,6 +12,7 @@ import ModerationSettings from "@/Pages/settings/Moderation";
|
||||
import { CacheSettings } from "@/Pages/settings/Cache";
|
||||
import { ReferralsPage } from "@/Pages/settings/Referrals";
|
||||
import { Outlet } from "react-router-dom";
|
||||
import { ToolsPage, ToolsPages } from "./tools";
|
||||
|
||||
const SettingsPage = () => {
|
||||
return (
|
||||
@ -70,6 +71,11 @@ export default [
|
||||
path: "invite",
|
||||
element: <ReferralsPage />,
|
||||
},
|
||||
{
|
||||
path: "tools",
|
||||
element: <ToolsPage />,
|
||||
children: ToolsPages,
|
||||
},
|
||||
...ManageHandleRoutes,
|
||||
...WalletSettingsRoutes,
|
||||
],
|
||||
|
||||
@ -0,0 +1,58 @@
|
||||
import { CollapsedSection } from "@/Element/Collapsed";
|
||||
import ProfilePreview from "@/Element/User/ProfilePreview";
|
||||
import useLogin from "@/Hooks/useLogin";
|
||||
import { getRelayName } from "@/SnortUtils";
|
||||
import { dedupe } from "@snort/shared";
|
||||
import { pickTopRelays } from "@snort/system";
|
||||
import { SnortContext } from "@snort/system-react";
|
||||
import { ReactNode, useContext, useMemo } from "react";
|
||||
import { FormattedMessage, FormattedNumber } from "react-intl";
|
||||
|
||||
export function FollowsRelayHealth({ withTitle, popularRelays, missingRelaysActions }: { withTitle?: boolean, popularRelays?: boolean, missingRelaysActions?: (k: string) => ReactNode }) {
|
||||
const system = useContext(SnortContext);
|
||||
const follows = useLogin(s => s.follows);
|
||||
const uniqueFollows = dedupe(follows.item);
|
||||
|
||||
const hasRelays = useMemo(() => {
|
||||
return uniqueFollows.filter(a => (system.RelayCache.getFromCache(a)?.relays.length ?? 0) > 0);
|
||||
}, [uniqueFollows]);
|
||||
|
||||
const missingRelays = useMemo(() => {
|
||||
return uniqueFollows.filter(a => !hasRelays.includes(a));
|
||||
}, [hasRelays]);
|
||||
|
||||
const topWriteRelays = useMemo(() => {
|
||||
return pickTopRelays(system.RelayCache, uniqueFollows, 1e31, "write");
|
||||
}, [uniqueFollows]);
|
||||
|
||||
return <div className="flex flex-col gap-4">
|
||||
{(withTitle ?? true) && <div className="text-2xl font-semibold">
|
||||
<FormattedMessage defaultMessage="Follows Relay Health" id="XQiFEl" />
|
||||
</div>}
|
||||
<div>
|
||||
<FormattedMessage defaultMessage="{x}/{y} have relays ({percent})" id="p9Ps2l" values={{
|
||||
x: hasRelays.length,
|
||||
y: uniqueFollows.length,
|
||||
percent: <FormattedNumber style="percent" value={hasRelays.length / uniqueFollows.length} />
|
||||
}} />
|
||||
</div>
|
||||
{missingRelays.length > 0 && <CollapsedSection className="rounded-xl border border-border-color px-3 py-4" title={<div className="text-lg"><FormattedMessage defaultMessage="Missing Relays" id="4emo2p" /></div>}>
|
||||
<div>
|
||||
{missingRelays.map(a => <ProfilePreview pubkey={a} options={{
|
||||
about: false
|
||||
}} actions={missingRelaysActions?.(a)} />)}
|
||||
</div>
|
||||
</CollapsedSection>}
|
||||
{(popularRelays ?? true) && <div>
|
||||
<div className="text-xl font-medium">Popular Relays</div>
|
||||
{dedupe(topWriteRelays.flatMap(a => a.relays))
|
||||
.map(a => ({ relay: a, count: topWriteRelays.filter(b => b.relays.includes(a)).length }))
|
||||
.sort((a, b) => a.count > b.count ? -1 : 1)
|
||||
.slice(0, 10)
|
||||
.map(a => <div className="flex justify-between">
|
||||
<div>{getRelayName(a.relay)}</div>
|
||||
<div>{a.count} (<FormattedNumber style="percent" value={a.count / uniqueFollows.length} />)</div>
|
||||
</div>)}
|
||||
</div>}
|
||||
</div>
|
||||
}
|
||||
50
packages/app/src/Pages/settings/tools/index.tsx
Normal file
50
packages/app/src/Pages/settings/tools/index.tsx
Normal file
@ -0,0 +1,50 @@
|
||||
import { FormattedMessage } from "react-intl"
|
||||
import { Outlet, RouteObject } from "react-router-dom"
|
||||
import { SettingsMenuComponent, SettingsMenuItems } from "../Menu"
|
||||
import { PruneFollowList } from "./prune-follows";
|
||||
import { FollowsRelayHealth } from "./follows-relay-health";
|
||||
|
||||
|
||||
const ToolMenuItems = [
|
||||
{
|
||||
title: <FormattedMessage defaultMessage="Follow List" id="CM+Cfj" />,
|
||||
items: [
|
||||
{
|
||||
icon: "trash",
|
||||
iconBg: "bg-red-500",
|
||||
message: <FormattedMessage defaultMessage="Prune Follow List" id="hF6IN2" />,
|
||||
path: "prune-follows"
|
||||
},
|
||||
{
|
||||
icon: "medical-cross",
|
||||
iconBg: "bg-green-800",
|
||||
message: <FormattedMessage defaultMessage="Follows Relay Health" id="XQiFEl" />,
|
||||
path: "follows-relay-health"
|
||||
}
|
||||
]
|
||||
}
|
||||
] as SettingsMenuItems;
|
||||
|
||||
export const ToolsPages = [
|
||||
{
|
||||
path: "",
|
||||
element: <>
|
||||
<h2>
|
||||
<FormattedMessage defaultMessage="Tools" id="nUT0Lv" />
|
||||
</h2>
|
||||
<SettingsMenuComponent menu={ToolMenuItems} />
|
||||
</>
|
||||
},
|
||||
{
|
||||
path: "prune-follows",
|
||||
element: <PruneFollowList />
|
||||
},
|
||||
{
|
||||
path: "follows-relay-health",
|
||||
element: <FollowsRelayHealth />
|
||||
}
|
||||
] as Array<RouteObject>
|
||||
|
||||
export function ToolsPage() {
|
||||
return <Outlet />
|
||||
}
|
||||
136
packages/app/src/Pages/settings/tools/prune-follows.tsx
Normal file
136
packages/app/src/Pages/settings/tools/prune-follows.tsx
Normal file
@ -0,0 +1,136 @@
|
||||
import { Day } from "@/Const";
|
||||
import AsyncButton from "@/Element/Button/AsyncButton";
|
||||
import useLogin from "@/Hooks/useLogin"
|
||||
import { dedupe, unixNow } from "@snort/shared";
|
||||
import { RequestBuilder } from "@snort/system";
|
||||
import { useMemo, useState } from "react";
|
||||
import { FormattedMessage, FormattedNumber } from "react-intl";
|
||||
import { FollowsRelayHealth } from "./follows-relay-health";
|
||||
import ProfileImage from "@/Element/User/ProfileImage";
|
||||
import useEventPublisher from "@/Hooks/useEventPublisher";
|
||||
import { setFollows } from "@/Login";
|
||||
|
||||
const enum PruneStage {
|
||||
FetchLastPostTimestamp,
|
||||
Done
|
||||
}
|
||||
|
||||
export function PruneFollowList() {
|
||||
const { id, follows } = useLogin(s => ({ id: s.id, follows: s.follows }));
|
||||
const { publisher, system } = useEventPublisher();
|
||||
const uniqueFollows = dedupe(follows.item);
|
||||
const [status, setStatus] = useState<PruneStage>();
|
||||
const [progress, setProgress] = useState(0);
|
||||
const [lastPost, setLastPosts] = useState<Record<string, number>>();
|
||||
const [unfollow, setUnfollow] = useState<Array<string>>([]);
|
||||
|
||||
async function fetchLastPosts() {
|
||||
setStatus(PruneStage.FetchLastPostTimestamp);
|
||||
setProgress(0);
|
||||
setLastPosts(undefined);
|
||||
|
||||
const BatchSize = 10;
|
||||
const chunks = uniqueFollows.reduce((acc, v, i) => {
|
||||
const batch = Math.floor(i / BatchSize).toString();
|
||||
acc[batch] ??= [];
|
||||
acc[batch].push(v);
|
||||
return acc;
|
||||
}, {} as Record<string, Array<string>>);
|
||||
|
||||
const result = {} as Record<string, number>;
|
||||
const batches = Math.ceil(uniqueFollows.length / BatchSize);
|
||||
for (const [batch, pubkeys] of Object.entries(chunks)) {
|
||||
console.debug(batch, pubkeys);
|
||||
const req = new RequestBuilder(`prune-${batch}`);
|
||||
req.withOptions({
|
||||
outboxPickN: 10,
|
||||
timeout: 10_000
|
||||
});
|
||||
pubkeys.forEach(p => req.withFilter().limit(1).kinds([0, 1, 3, 5, 6, 7, 10002]).authors([p]));
|
||||
const results = await system.Fetch(req);
|
||||
console.debug(results);
|
||||
for (const rx of results) {
|
||||
if ((result[rx.pubkey] ?? 0) < rx.created_at) {
|
||||
result[rx.pubkey] = rx.created_at;
|
||||
}
|
||||
}
|
||||
setProgress(Number(batch) / batches);
|
||||
}
|
||||
|
||||
for (const pk of uniqueFollows) {
|
||||
result[pk] ??= 0;
|
||||
}
|
||||
setLastPosts(result);
|
||||
setStatus(PruneStage.Done);
|
||||
}
|
||||
|
||||
const newFollowList = useMemo(() => {
|
||||
return uniqueFollows.filter(a => !unfollow.includes(a) && a.length === 64);
|
||||
}, [uniqueFollows, unfollow]);
|
||||
|
||||
async function publishFollowList() {
|
||||
const newFollows = newFollowList.map(a => ["p", a]) as Array<[string, string]>;
|
||||
if (publisher) {
|
||||
const ev = await publisher.contactList(newFollows);
|
||||
await system.BroadcastEvent(ev);
|
||||
setFollows(id, newFollowList, ev.created_at * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function getStatus() {
|
||||
switch (status) {
|
||||
case PruneStage.FetchLastPostTimestamp: return <FormattedMessage defaultMessage="Searching for account activity ({progress})" id="nIchMQ" values={{
|
||||
progress: <FormattedNumber style="percent" value={progress} />
|
||||
}} />
|
||||
}
|
||||
}
|
||||
|
||||
function personToggle(k: string,) {
|
||||
return <div className="flex gap-1">
|
||||
<input type="checkbox" onChange={e => setUnfollow(v => e.target.checked ? dedupe([...v, k]) : v.filter(a => a !== k))} checked={unfollow.includes(k)} />
|
||||
<FormattedMessage defaultMessage="Unfollow" id="izWS4J" />
|
||||
</div>
|
||||
}
|
||||
|
||||
return <div className="flex flex-col gap-4">
|
||||
<div className="text-2xl font-semibold">
|
||||
<FormattedMessage defaultMessage="Prune follow list" id="CM0k0d" />
|
||||
</div>
|
||||
<p>
|
||||
<FormattedMessage defaultMessage="This tool will search for the last event published by all of your follows and remove those who have not posted in 6 months" id="vU/Q5i" />
|
||||
</p>
|
||||
<div>
|
||||
<FormattedMessage defaultMessage="{x} follows ({y} duplicates)" id="iICVoL" values={{
|
||||
x: follows.item.length,
|
||||
y: follows.item.length - uniqueFollows.length
|
||||
}} />
|
||||
</div>
|
||||
<FollowsRelayHealth withTitle={false} popularRelays={false} missingRelaysActions={(k) => personToggle(k)} />
|
||||
<AsyncButton onClick={fetchLastPosts}>
|
||||
<FormattedMessage defaultMessage="Compute prune list" id="bJ+wrA" />
|
||||
</AsyncButton>
|
||||
{getStatus()}
|
||||
<div className="flex flex-col gap-1">
|
||||
{lastPost && Object.entries(lastPost).filter(([, v]) => v <= unixNow() - (90 * Day)).sort(([, a], [, b]) => a > b ? -1 : 1).map(([k, v]) => {
|
||||
return <div className="flex justify-between">
|
||||
<ProfileImage pubkey={k} />
|
||||
<div className="flex flex-col gap-1">
|
||||
<FormattedMessage defaultMessage="Last post {time}" id="I1AoOu" values={{
|
||||
time: new Date(v * 1000).toLocaleDateString()
|
||||
}} />
|
||||
{personToggle(k)}
|
||||
</div>
|
||||
</div>
|
||||
})}
|
||||
</div>
|
||||
<div className="px-4 pb-5 pt-2 rounded-2xl bg-bg-secondary">
|
||||
<p>
|
||||
<FormattedMessage defaultMessage="New follow list length {length}" id="6559gb" values={{ length: newFollowList.length }} />
|
||||
</p>
|
||||
<AsyncButton onClick={publishFollowList}>
|
||||
<FormattedMessage defaultMessage="Save" id="jvo0vs" />
|
||||
</AsyncButton>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
@ -436,8 +436,8 @@
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9733 0.666994C13.0465 0.667127 14.1219 0.667127 15.1931 0.666994C15.4034 0.666969 15.6082 0.666945 15.7817 0.681118C15.9722 0.696682 16.197 0.733404 16.4232 0.848652C16.7368 1.00844 16.9917 1.26341 17.1515 1.57701C17.2668 1.8032 17.3035 2.02798 17.3191 2.21847C17.3332 2.39194 17.3332 2.59677 17.3332 2.8071V6.0269C17.3332 6.23722 17.3332 6.44205 17.3191 6.61552C17.3035 6.80602 17.2668 7.03079 17.1515 7.25698C16.9917 7.57058 16.7368 7.82555 16.4232 7.98534C16.197 8.10059 15.9722 8.13731 15.7817 8.15288C15.6082 8.16705 15.4034 8.16702 15.1931 8.167H11.9733C11.763 8.16702 11.5581 8.16705 11.3847 8.15288C11.1942 8.13731 10.9694 8.10059 10.7432 7.98534C10.4296 7.82555 10.1746 7.57058 10.0148 7.25698C9.89958 7.03079 9.86286 6.80602 9.8473 6.61552C9.83312 6.44206 9.83315 6.23723 9.83317 6.02691C9.83317 6.01806 9.83317 6.0092 9.83317 6.00033V2.83366C9.83317 2.82479 9.83317 2.81593 9.83317 2.80708C9.83315 2.59676 9.83312 2.39194 9.8473 2.21847C9.86286 2.02798 9.89958 1.8032 10.0148 1.57701C10.1746 1.26341 10.4296 1.00844 10.7432 0.848652C10.9694 0.733404 11.1942 0.696682 11.3847 0.681118C11.5581 0.666945 11.7629 0.666969 11.9733 0.666994Z" fill="currentColor"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9733 9.83366C13.0465 9.83379 14.1219 9.83379 15.1931 9.83366C15.4034 9.83364 15.6082 9.83361 15.7817 9.84778C15.9722 9.86335 16.197 9.90007 16.4232 10.0153C16.7368 10.1751 16.9917 10.4301 17.1515 10.7437C17.2668 10.9699 17.3035 11.1946 17.3191 11.3851C17.3332 11.5586 17.3332 11.7634 17.3332 11.9738V15.1936C17.3332 15.4039 17.3332 15.6087 17.3191 15.7822C17.3035 15.9727 17.2668 16.1975 17.1515 16.4236C16.9917 16.7373 16.7368 16.9922 16.4232 17.152C16.197 17.2673 15.9722 17.304 15.7817 17.3195C15.6082 17.3337 15.4034 17.3337 15.1931 17.3337H11.9733C11.763 17.3337 11.5581 17.3337 11.3847 17.3195C11.1942 17.304 10.9694 17.2673 10.7432 17.152C10.4296 16.9922 10.1746 16.7373 10.0148 16.4236C9.89958 16.1975 9.86286 15.9727 9.8473 15.7822C9.83312 15.6087 9.83315 15.4039 9.83317 15.1936C9.83317 15.1847 9.83317 15.1759 9.83317 15.167V12.0003C9.83317 11.9915 9.83317 11.9826 9.83317 11.9737C9.83315 11.7634 9.83312 11.5586 9.8473 11.3851C9.86286 11.1946 9.89958 10.9699 10.0148 10.7437C10.1746 10.4301 10.4296 10.1751 10.7432 10.0153C10.9694 9.90007 11.1942 9.86335 11.3847 9.84778C11.5581 9.83361 11.7629 9.83364 11.9733 9.83366Z" fill="currentColor"/>
|
||||
</symbol>
|
||||
|
||||
<symbol id="info-solid" viewBox="0 0 22 22" fill="none" >
|
||||
|
||||
<symbol id="info-solid" viewBox="0 0 22 22" fill="none">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M11 0C4.92487 0 0 4.92487 0 11C0 17.0751 4.92487 22 11 22C17.0751 22 22 17.0751 22 11C22 4.92487 17.0751 0 11 0ZM11 6C10.4477 6 10 6.44772 10 7C10 7.55228 10.4477 8 11 8H11.01C11.5623 8 12.01 7.55228 12.01 7C12.01 6.44772 11.5623 6 11.01 6H11ZM12 11C12 10.4477 11.5523 10 11 10C10.4477 10 10 10.4477 10 11V15C10 15.5523 10.4477 16 11 16C11.5523 16 12 15.5523 12 15V11Z" fill="currentColor"/>
|
||||
</symbol>
|
||||
<symbol id="info-outline" viewBox="0 0 22 22" fill="none">
|
||||
@ -448,5 +448,11 @@
|
||||
<path d="M7.5 6.96533C7.5 6.48805 7.5 6.24941 7.59974 6.11618C7.68666 6.00007 7.81971 5.92744 7.96438 5.9171C8.13038 5.90525 8.33112 6.03429 8.73261 6.29239L13.4532 9.32706C13.8016 9.55102 13.9758 9.663 14.0359 9.80539C14.0885 9.9298 14.0885 10.0702 14.0359 10.1946C13.9758 10.337 13.8016 10.449 13.4532 10.6729L8.73261 13.7076C8.33112 13.9657 8.13038 14.0948 7.96438 14.0829C7.81971 14.0726 7.68666 13.9999 7.59974 13.8838C7.5 13.7506 7.5 13.512 7.5 13.0347V6.96533Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M1 5.8C1 4.11984 1 3.27976 1.32698 2.63803C1.6146 2.07354 2.07354 1.6146 2.63803 1.32698C3.27976 1 4.11984 1 5.8 1H14.2C15.8802 1 16.7202 1 17.362 1.32698C17.9265 1.6146 18.3854 2.07354 18.673 2.63803C19 3.27976 19 4.11984 19 5.8V14.2C19 15.8802 19 16.7202 18.673 17.362C18.3854 17.9265 17.9265 18.3854 17.362 18.673C16.7202 19 15.8802 19 14.2 19H5.8C4.11984 19 3.27976 19 2.63803 18.673C2.07354 18.3854 1.6146 17.9265 1.32698 17.362C1 16.7202 1 15.8802 1 14.2V5.8Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</symbol>
|
||||
<symbol id="tool" viewBox="0 0 22 22" fill="none">
|
||||
<path d="M14.6314 6.63137C14.2353 6.23535 14.0373 6.03735 13.9631 5.80902C13.8979 5.60817 13.8979 5.39183 13.9631 5.19098C14.0373 4.96265 14.2353 4.76465 14.6314 4.36863L17.4697 1.53026C16.7165 1.18962 15.8804 1 15 1C11.6863 1 8.99998 3.68629 8.99998 7C8.99998 7.49104 9.05897 7.9683 9.17024 8.42509C9.2894 8.91424 9.34898 9.15882 9.33841 9.31333C9.32733 9.47509 9.30321 9.56115 9.22862 9.70511C9.15736 9.84262 9.02084 9.97914 8.7478 10.2522L2.49998 16.5C1.67156 17.3284 1.67156 18.6716 2.49998 19.5C3.32841 20.3284 4.67156 20.3284 5.49998 19.5L11.7478 13.2522C12.0208 12.9791 12.1574 12.8426 12.2949 12.7714C12.4388 12.6968 12.5249 12.6727 12.6867 12.6616C12.8412 12.651 13.0857 12.7106 13.5749 12.8297C14.0317 12.941 14.5089 13 15 13C18.3137 13 21 10.3137 21 7C21 6.11959 20.8104 5.28347 20.4697 4.53026L17.6314 7.36863C17.2353 7.76465 17.0373 7.96265 16.809 8.03684C16.6082 8.1021 16.3918 8.1021 16.191 8.03684C15.9626 7.96265 15.7646 7.76465 15.3686 7.36863L14.6314 6.63137Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</symbol>
|
||||
<symbol id="medical-cross" viewBox="0 0 24 24" fill="none">
|
||||
<path d="M15 4.6C15 4.03995 15 3.75992 14.891 3.54601C14.7951 3.35785 14.6422 3.20487 14.454 3.10899C14.2401 3 13.9601 3 13.4 3H10.6C10.0399 3 9.75992 3 9.54601 3.10899C9.35785 3.20487 9.20487 3.35785 9.10899 3.54601C9 3.75992 9 4.03995 9 4.6V7.4C9 7.96005 9 8.24008 8.89101 8.45399C8.79513 8.64215 8.64215 8.79513 8.45399 8.89101C8.24008 9 7.96005 9 7.4 9H4.6C4.03995 9 3.75992 9 3.54601 9.10899C3.35785 9.20487 3.20487 9.35785 3.10899 9.54601C3 9.75992 3 10.0399 3 10.6V13.4C3 13.9601 3 14.2401 3.10899 14.454C3.20487 14.6422 3.35785 14.7951 3.54601 14.891C3.75992 15 4.03995 15 4.6 15H7.4C7.96005 15 8.24008 15 8.45399 15.109C8.64215 15.2049 8.79513 15.3578 8.89101 15.546C9 15.7599 9 16.0399 9 16.6V19.4C9 19.9601 9 20.2401 9.10899 20.454C9.20487 20.6422 9.35785 20.7951 9.54601 20.891C9.75992 21 10.0399 21 10.6 21H13.4C13.9601 21 14.2401 21 14.454 20.891C14.6422 20.7951 14.7951 20.6422 14.891 20.454C15 20.2401 15 19.9601 15 19.4V16.6C15 16.0399 15 15.7599 15.109 15.546C15.2049 15.3578 15.3578 15.2049 15.546 15.109C15.7599 15 16.0399 15 16.6 15H19.4C19.9601 15 20.2401 15 20.454 14.891C20.6422 14.7951 20.7951 14.6422 20.891 14.454C21 14.2401 21 13.9601 21 13.4V10.6C21 10.0399 21 9.75992 20.891 9.54601C20.7951 9.35785 20.6422 9.20487 20.454 9.10899C20.2401 9 19.9601 9 19.4 9L16.6 9C16.0399 9 15.7599 9 15.546 8.89101C15.3578 8.79513 15.2049 8.64215 15.109 8.45399C15 8.24008 15 7.96005 15 7.4V4.6Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</symbol>
|
||||
</defs>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 125 KiB After Width: | Height: | Size: 128 KiB |
@ -300,7 +300,7 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
|
||||
const store = new type();
|
||||
|
||||
const filters = req.build(this);
|
||||
const q = new Query(req.id, req.instance, store, req.options?.leaveOpen);
|
||||
const q = new Query(req.id, req.instance, store, req.options?.leaveOpen, req.options?.timeout);
|
||||
q.on("trace", r => this.#relayMetrics.onTraceReport(r));
|
||||
|
||||
if (filters.some(a => a.filters.some(b => b.ids))) {
|
||||
|
||||
@ -14,7 +14,7 @@ import { FlatReqFilter } from "./query-optimizer";
|
||||
import { RelayListCacheExpire } from "./const";
|
||||
import { BackgroundLoader } from "./background-loader";
|
||||
|
||||
const PickNRelays = 2;
|
||||
const DefaultPickNRelays = 2;
|
||||
|
||||
export interface RelayTaggedFilter {
|
||||
relay: string;
|
||||
@ -66,7 +66,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilte
|
||||
/**
|
||||
* Split filters by authors
|
||||
*/
|
||||
export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<RelayTaggedFilter> {
|
||||
export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter> {
|
||||
const authors = filter.authors;
|
||||
if ((authors?.length ?? 0) === 0) {
|
||||
return [
|
||||
@ -77,7 +77,7 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<
|
||||
];
|
||||
}
|
||||
|
||||
const topRelays = pickTopRelays(cache, unwrap(authors), PickNRelays, "write");
|
||||
const topRelays = pickTopRelays(cache, unwrap(authors), pickN ?? DefaultPickNRelays, "write");
|
||||
const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
|
||||
|
||||
const picked = pickedRelays.map(a => {
|
||||
@ -107,7 +107,11 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<
|
||||
/**
|
||||
* Split filters by author
|
||||
*/
|
||||
export function splitFlatByWriteRelays(cache: RelayCache, input: Array<FlatReqFilter>): Array<RelayTaggedFlatFilters> {
|
||||
export function splitFlatByWriteRelays(
|
||||
cache: RelayCache,
|
||||
input: Array<FlatReqFilter>,
|
||||
pickN?: number,
|
||||
): Array<RelayTaggedFlatFilters> {
|
||||
const authors = input.filter(a => a.authors).map(a => unwrap(a.authors));
|
||||
if (authors.length === 0) {
|
||||
return [
|
||||
@ -117,7 +121,7 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array<FlatReqFi
|
||||
},
|
||||
];
|
||||
}
|
||||
const topRelays = pickTopRelays(cache, authors, PickNRelays, "write");
|
||||
const topRelays = pickTopRelays(cache, authors, pickN ?? DefaultPickNRelays, "write");
|
||||
const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
|
||||
|
||||
const picked = pickedRelays.map(a => {
|
||||
@ -142,7 +146,7 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array<FlatReqFi
|
||||
/**
|
||||
* Pick most popular relays for each authors
|
||||
*/
|
||||
function pickTopRelays(cache: RelayCache, authors: Array<string>, n: number, type: "write" | "read") {
|
||||
export function pickTopRelays(cache: RelayCache, authors: Array<string>, n: number, type: "write" | "read") {
|
||||
// map of pubkey -> [write relays]
|
||||
const allRelays = authors.map(a => {
|
||||
return {
|
||||
@ -198,10 +202,10 @@ function pickTopRelays(cache: RelayCache, authors: Array<string>, n: number, typ
|
||||
/**
|
||||
* Pick read relays for sending reply events
|
||||
*/
|
||||
export async function pickRelaysForReply(ev: NostrEvent, system: SystemInterface) {
|
||||
export async function pickRelaysForReply(ev: NostrEvent, system: SystemInterface, pickN?: number) {
|
||||
const recipients = dedupe(ev.tags.filter(a => a[0] === "p").map(a => a[1]));
|
||||
await updateRelayLists(recipients, system);
|
||||
const relays = pickTopRelays(system.RelayCache, recipients, 2, "read");
|
||||
const relays = pickTopRelays(system.RelayCache, recipients, pickN ?? DefaultPickNRelays, "read");
|
||||
const ret = removeUndefined(dedupe(relays.map(a => a.relays).flat()));
|
||||
logger("Picked %O from authors %O", ret, recipients);
|
||||
return ret;
|
||||
@ -221,6 +225,27 @@ export function parseRelayTags(tag: Array<Array<string>>) {
|
||||
return tag.map(parseRelayTag).filter(a => a !== null);
|
||||
}
|
||||
|
||||
export function parseRelaysFromKind(ev: NostrEvent) {
|
||||
if (ev.kind === EventKind.ContactList) {
|
||||
const relaysInContent =
|
||||
ev.content.length > 0 ? (JSON.parse(ev.content) as Record<string, { read: boolean; write: boolean }>) : undefined;
|
||||
if (relaysInContent) {
|
||||
return Object.entries(relaysInContent).map(
|
||||
([k, v]) =>
|
||||
({
|
||||
url: sanitizeRelayUrl(k),
|
||||
settings: {
|
||||
read: v.read,
|
||||
write: v.write,
|
||||
},
|
||||
}) as FullRelaySettings,
|
||||
);
|
||||
}
|
||||
} else if (ev.kind === EventKind.Relays) {
|
||||
return parseRelayTags(ev.tags);
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateRelayLists(authors: Array<string>, system: SystemInterface) {
|
||||
await system.RelayCache.buffer(authors);
|
||||
const expire = unixNowMs() - RelayListCacheExpire;
|
||||
@ -228,15 +253,21 @@ export async function updateRelayLists(authors: Array<string>, system: SystemInt
|
||||
if (expired.length > 0) {
|
||||
logger("Updating relays for authors: %O", expired);
|
||||
const rb = new RequestBuilder("system-update-relays-for-outbox");
|
||||
rb.withFilter().authors(expired).kinds([EventKind.Relays]);
|
||||
rb.withFilter().authors(expired).kinds([EventKind.Relays, EventKind.ContactList]);
|
||||
const relayLists = await system.Fetch(rb);
|
||||
await system.RelayCache.bulkSet(
|
||||
relayLists.map(a => ({
|
||||
relays: parseRelayTags(a.tags),
|
||||
pubkey: a.pubkey,
|
||||
created: a.created_at,
|
||||
loaded: unixNowMs(),
|
||||
})),
|
||||
removeUndefined(
|
||||
relayLists.map(a => {
|
||||
const relays = parseRelaysFromKind(a);
|
||||
if (!relays) return;
|
||||
return {
|
||||
relays: relays,
|
||||
pubkey: a.pubkey,
|
||||
created: a.created_at,
|
||||
loaded: unixNowMs(),
|
||||
};
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -247,8 +278,10 @@ export class RelayMetadataLoader extends BackgroundLoader<UsersRelays> {
|
||||
}
|
||||
|
||||
override onEvent(e: Readonly<TaggedNostrEvent>): UsersRelays | undefined {
|
||||
const relays = parseRelaysFromKind(e);
|
||||
if (!relays) return;
|
||||
return {
|
||||
relays: parseRelayTags(e.tags),
|
||||
relays: relays,
|
||||
pubkey: e.pubkey,
|
||||
created: e.created_at,
|
||||
loaded: unixNowMs(),
|
||||
@ -261,8 +294,12 @@ export class RelayMetadataLoader extends BackgroundLoader<UsersRelays> {
|
||||
|
||||
protected override buildSub(missing: string[]): RequestBuilder {
|
||||
const rb = new RequestBuilder("relay-loader");
|
||||
rb.withOptions({ skipDiff: true });
|
||||
rb.withFilter().authors(missing).kinds([EventKind.Relays]);
|
||||
rb.withOptions({
|
||||
skipDiff: true,
|
||||
timeout: 10_000,
|
||||
outboxPickN: 4,
|
||||
});
|
||||
rb.withFilter().authors(missing).kinds([EventKind.Relays, EventKind.ContactList]);
|
||||
return rb;
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ export interface FlatReqFilter {
|
||||
since?: number;
|
||||
until?: number;
|
||||
limit?: number;
|
||||
resultSetId: string;
|
||||
}
|
||||
|
||||
export interface QueryOptimizer {
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { sha256 } from "@snort/shared";
|
||||
import { FlatReqFilter } from ".";
|
||||
import { ReqFilter } from "../nostr";
|
||||
|
||||
@ -7,29 +8,52 @@ import { ReqFilter } from "../nostr";
|
||||
export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
|
||||
const ret: Array<FlatReqFilter> = [];
|
||||
const src = Object.entries(f);
|
||||
const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]);
|
||||
const props = src.filter(([, v]) => !Array.isArray(v));
|
||||
|
||||
function generateCombinations(index: number, currentCombination: FlatReqFilter) {
|
||||
if (index === keys.length) {
|
||||
ret.push(currentCombination);
|
||||
const id = resultSetId(f);
|
||||
|
||||
// Filter entries that are arrays and keep the rest as is
|
||||
const arrays: [string, Array<string> | Array<number>][] = src.filter(([, value]) => Array.isArray(value)) as [
|
||||
string,
|
||||
Array<string> | Array<number>,
|
||||
][];
|
||||
const constants = Object.fromEntries(src.filter(([, value]) => !Array.isArray(value))) as {
|
||||
[key: string]: string | number | undefined;
|
||||
};
|
||||
|
||||
// Recursive function to compute cartesian product
|
||||
function cartesianProduct(arr: [string, Array<string> | Array<number>][], temp: [string, any][] = []) {
|
||||
if (arr.length === 0) {
|
||||
ret.push(createFilterObject(temp, constants, id));
|
||||
return;
|
||||
}
|
||||
|
||||
const key = keys[index];
|
||||
const values = (f as Record<string, Array<string | number>>)[key];
|
||||
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
const value = values[i];
|
||||
const updatedCombination = { ...currentCombination, [key]: value };
|
||||
generateCombinations(index + 1, updatedCombination);
|
||||
for (let i = 0; i < arr[0][1].length; i++) {
|
||||
cartesianProduct(arr.slice(1), temp.concat([[arr[0][0], arr[0][1][i]]]));
|
||||
}
|
||||
}
|
||||
|
||||
generateCombinations(0, {
|
||||
keys: keys.length,
|
||||
...Object.fromEntries(props),
|
||||
});
|
||||
// Create filter object from the combination
|
||||
function createFilterObject(
|
||||
combination: [string, any][],
|
||||
constants: { [key: string]: string | number | undefined },
|
||||
resultId: string,
|
||||
) {
|
||||
let filterObject = { ...Object.fromEntries(combination), ...constants } as FlatReqFilter;
|
||||
filterObject.resultSetId = resultId;
|
||||
return filterObject;
|
||||
}
|
||||
|
||||
cartesianProduct(arrays);
|
||||
return ret;
|
||||
}
|
||||
|
||||
function resultSetId(f: ReqFilter) {
|
||||
if (f.limit !== undefined || f.since !== undefined || f.until !== undefined) {
|
||||
const arrays = Object.entries(f)
|
||||
.filter(([, a]) => Array.isArray(a))
|
||||
.map(a => a as [string, Array<string | number>])
|
||||
.sort();
|
||||
const input = arrays.map(([, a]) => a.join(",")).join(",");
|
||||
return sha256(input);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
@ -2,18 +2,9 @@ import { distance } from "@snort/shared";
|
||||
import { ReqFilter } from "..";
|
||||
import { FlatReqFilter } from ".";
|
||||
|
||||
/**
|
||||
* Keys which can change the entire meaning of the filter outside the array types
|
||||
*/
|
||||
const DiscriminatorKeys = ["since", "until", "limit", "search"];
|
||||
|
||||
export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
|
||||
const aObj = a as Record<string, string | number | undefined>;
|
||||
const bObj = b as Record<string, string | number | undefined>;
|
||||
for (const key of DiscriminatorKeys) {
|
||||
if (aObj[key] !== bObj[key]) {
|
||||
return false;
|
||||
}
|
||||
if (a.resultSetId !== b.resultSetId) {
|
||||
return false;
|
||||
}
|
||||
return distance(a, b) <= 1;
|
||||
}
|
||||
@ -101,12 +92,11 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
||||
|
||||
// to compute filters which can be merged we need to calucate the distance change between each filter
|
||||
// then we can merge filters which are exactly 1 change diff from each other
|
||||
|
||||
function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
|
||||
return filters.reduce((acc, a) => {
|
||||
Object.entries(a).forEach(([k, v]) => {
|
||||
if (k === "keys" || v === undefined) return;
|
||||
if (DiscriminatorKeys.includes(k)) {
|
||||
if (v === undefined) return;
|
||||
if (k === "since" || k === "until" || k === "limit" || k === "search" || k === "resultSetId") {
|
||||
acc[k] = v;
|
||||
} else {
|
||||
acc[k] ??= [];
|
||||
@ -142,5 +132,6 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
|
||||
}
|
||||
ret = n;
|
||||
}
|
||||
ret.forEach(a => delete a["resultSetId"]);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -157,14 +157,20 @@ export class Query extends EventEmitter<QueryEvents> implements QueryBase {
|
||||
*/
|
||||
#feed: NoteStore;
|
||||
|
||||
/**
|
||||
* Maximum waiting time for this query
|
||||
*/
|
||||
#timeout: number;
|
||||
|
||||
#log = debug("Query");
|
||||
|
||||
constructor(id: string, instance: string, feed: NoteStore, leaveOpen?: boolean) {
|
||||
constructor(id: string, instance: string, feed: NoteStore, leaveOpen?: boolean, timeout?: number) {
|
||||
super();
|
||||
this.id = id;
|
||||
this.#feed = feed;
|
||||
this.fromInstance = instance;
|
||||
this.#leaveOpen = leaveOpen ?? false;
|
||||
this.#timeout = timeout ?? 5_000;
|
||||
this.#checkTraces();
|
||||
}
|
||||
|
||||
@ -292,7 +298,7 @@ export class Query extends EventEmitter<QueryEvents> implements QueryBase {
|
||||
this.#stopCheckTraces();
|
||||
this.#checkTrace = setInterval(() => {
|
||||
for (const v of this.#tracing) {
|
||||
if (v.runtime > 5_000 && !v.finished) {
|
||||
if (v.runtime > this.#timeout && !v.finished) {
|
||||
v.forceEose();
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,6 +43,16 @@ export interface RequestBuilderOptions {
|
||||
* Do not apply diff logic and always use full filters for query
|
||||
*/
|
||||
skipDiff?: boolean;
|
||||
|
||||
/**
|
||||
* Pick N relays per pubkey when using outbox strategy
|
||||
*/
|
||||
outboxPickN?: number;
|
||||
|
||||
/**
|
||||
* Max wait time for this request
|
||||
*/
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -101,7 +111,7 @@ export class RequestBuilder {
|
||||
}
|
||||
|
||||
build(system: SystemInterface): Array<BuiltRawReqFilter> {
|
||||
const expanded = this.#builders.flatMap(a => a.build(system.RelayCache, this.id));
|
||||
const expanded = this.#builders.flatMap(a => a.build(system.RelayCache, this.#options));
|
||||
return this.#groupByRelay(system, expanded);
|
||||
}
|
||||
|
||||
@ -130,11 +140,9 @@ export class RequestBuilder {
|
||||
|
||||
/**
|
||||
* Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests
|
||||
* @param expanded
|
||||
* @returns
|
||||
*/
|
||||
#groupByRelay(system: SystemInterface, expanded: Array<BuiltRawReqFilter>) {
|
||||
const relayMerged = expanded.reduce((acc, v) => {
|
||||
#groupByRelay(system: SystemInterface, filters: Array<BuiltRawReqFilter>) {
|
||||
const relayMerged = filters.reduce((acc, v) => {
|
||||
const existing = acc.get(v.relay);
|
||||
if (existing) {
|
||||
existing.push(v);
|
||||
@ -267,7 +275,7 @@ export class RequestFilterBuilder {
|
||||
/**
|
||||
* Build/expand this filter into a set of relay specific queries
|
||||
*/
|
||||
build(relays: RelayCache, id: string): Array<BuiltRawReqFilter> {
|
||||
build(relays: RelayCache, options?: RequestBuilderOptions): Array<BuiltRawReqFilter> {
|
||||
// use the explicit relay list first
|
||||
if (this.#relays.size > 0) {
|
||||
return [...this.#relays].map(r => {
|
||||
@ -281,7 +289,7 @@ export class RequestFilterBuilder {
|
||||
|
||||
// If any authors are set use the gossip model to fetch data for each author
|
||||
if (this.#filter.authors) {
|
||||
const split = splitByWriteRelays(relays, this.#filter);
|
||||
const split = splitByWriteRelays(relays, this.#filter, options?.outboxPickN);
|
||||
return split.map(a => {
|
||||
return {
|
||||
filters: [a.filter],
|
||||
|
||||
@ -1,17 +1,27 @@
|
||||
import { splitAllByWriteRelays } from "../src/gossip-model";
|
||||
import { splitAllByWriteRelays } from "../src/outbox-model";
|
||||
|
||||
describe("GossipModel", () => {
|
||||
describe("OutboxModel", () => {
|
||||
it("should not output empty", () => {
|
||||
const Relays = {
|
||||
getFromCache: (pk?: string) => {
|
||||
if (pk) {
|
||||
return {
|
||||
pubkey: pk,
|
||||
created_at: 0,
|
||||
created: 0,
|
||||
loaded: 0,
|
||||
relays: [],
|
||||
};
|
||||
}
|
||||
},
|
||||
update: () => {
|
||||
return Promise.resolve<"new" | "updated" | "refresh" | "no_change">("new");
|
||||
},
|
||||
buffer: () => {
|
||||
return Promise.resolve<Array<string>>([]);
|
||||
},
|
||||
bulkSet: () => {
|
||||
return Promise.resolve();
|
||||
},
|
||||
};
|
||||
const a = [
|
||||
{
|
||||
@ -1,4 +1,4 @@
|
||||
import { RelayCache } from "../src/gossip-model";
|
||||
import { RelayCache } from "../src/outbox-model";
|
||||
import { RequestBuilder, RequestStrategy } from "../src/request-builder";
|
||||
import { describe, expect } from "@jest/globals";
|
||||
import { bytesToHex } from "@noble/curves/abstract/utils";
|
||||
@ -23,7 +23,16 @@ const DummyCache = {
|
||||
],
|
||||
};
|
||||
},
|
||||
} as FeedCache<UsersRelays>;
|
||||
update: () => {
|
||||
return Promise.resolve<"new" | "updated" | "refresh" | "no_change">("new");
|
||||
},
|
||||
buffer: () => {
|
||||
return Promise.resolve<Array<string>>([]);
|
||||
},
|
||||
bulkSet: () => {
|
||||
return Promise.resolve();
|
||||
},
|
||||
} as unknown as FeedCache<UsersRelays>;
|
||||
|
||||
const System = new NostrSystem({
|
||||
relayCache: DummyCache,
|
||||
@ -112,7 +121,7 @@ describe("RequestBuilder", () => {
|
||||
rb.withFilter().authors(["a", "b"]).kinds([0]);
|
||||
|
||||
const a = rb.build(System);
|
||||
expect(a).toEqual([
|
||||
expect(a).toMatchObject([
|
||||
{
|
||||
strategy: RequestStrategy.AuthorsRelays,
|
||||
relay: "wss://a.com/",
|
||||
@ -143,7 +152,7 @@ describe("RequestBuilder", () => {
|
||||
rb.withFilter().authors(["a"]).limit(10).kinds([4]);
|
||||
|
||||
const a = rb.build(System);
|
||||
expect(a).toEqual([
|
||||
expect(a).toMatchObject([
|
||||
{
|
||||
strategy: RequestStrategy.AuthorsRelays,
|
||||
relay: "wss://a.com/",
|
||||
|
||||
@ -10,25 +10,25 @@ describe("RequestExpander", () => {
|
||||
since: 99,
|
||||
limit: 10,
|
||||
};
|
||||
expect(expandFilter(a)).toEqual([
|
||||
{ authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
{ authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10, keys: 4 },
|
||||
expect(expandFilter(a)).toMatchObject([
|
||||
{ authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
|
||||
{ authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@ -81,25 +81,27 @@ describe("RequestMerger", () => {
|
||||
describe("flatMerge", () => {
|
||||
it("should flat merge simple", () => {
|
||||
const input = [
|
||||
{ ids: 0, authors: "a" },
|
||||
{ ids: 0, authors: "b" },
|
||||
{ kinds: 1 },
|
||||
{ kinds: 2 },
|
||||
{ kinds: 2 },
|
||||
{ ids: 0, authors: "c" },
|
||||
{ authors: "c", kinds: 1 },
|
||||
{ authors: "c", limit: 100 },
|
||||
{ ids: 1, authors: "c" },
|
||||
{ ids: "0", authors: "a", resultSetId: "" },
|
||||
{ ids: "0", authors: "b", resultSetId: "" },
|
||||
{ kinds: 1, resultSetId: "" },
|
||||
{ kinds: 2, resultSetId: "" },
|
||||
{ kinds: 2, resultSetId: "" },
|
||||
{ ids: "0", authors: "c", resultSetId: "" },
|
||||
{ authors: "c", kinds: 1, resultSetId: "" },
|
||||
{ authors: "c", limit: 100, resultSetId: "limit-c-100" },
|
||||
{ authors: "b", limit: 100, resultSetId: "limit-b-100" },
|
||||
{ ids: "1", authors: "c", resultSetId: "" },
|
||||
] as Array<FlatReqFilter>;
|
||||
const output = [
|
||||
{ ids: [0], authors: ["a", "b", "c"] },
|
||||
{ ids: ["0"], authors: ["a", "b", "c"] },
|
||||
{ kinds: [1, 2] },
|
||||
{ authors: ["c"], kinds: [1] },
|
||||
{ authors: ["c"], limit: 100 },
|
||||
{ ids: [1], authors: ["c"] },
|
||||
{ authors: ["b"], limit: 100 },
|
||||
{ ids: ["1"], authors: ["c"] },
|
||||
] as Array<ReqFilter>;
|
||||
|
||||
expect(flatMerge(input)).toEqual(output);
|
||||
expect(flatMerge(input)).toMatchObject(output);
|
||||
});
|
||||
|
||||
it("should expand and flat merge complex same", () => {
|
||||
@ -119,47 +121,47 @@ describe("canMerge", () => {
|
||||
it("should have 0 distance", () => {
|
||||
const a = {
|
||||
ids: "a",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
const b = {
|
||||
ids: "a",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
expect(canMergeFilters(a, b)).toEqual(true);
|
||||
});
|
||||
it("should have 1 distance", () => {
|
||||
const a = {
|
||||
ids: "a",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
const b = {
|
||||
ids: "b",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
expect(canMergeFilters(a, b)).toEqual(true);
|
||||
});
|
||||
it("should have 10 distance", () => {
|
||||
const a = {
|
||||
ids: "a",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
const b = {
|
||||
ids: "a",
|
||||
kinds: 1,
|
||||
keys: 2,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
expect(canMergeFilters(a, b)).toEqual(false);
|
||||
});
|
||||
it("should have 11 distance", () => {
|
||||
const a = {
|
||||
ids: "a",
|
||||
keys: 1,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
const b = {
|
||||
ids: "b",
|
||||
kinds: 1,
|
||||
keys: 2,
|
||||
};
|
||||
resultSetId: "",
|
||||
} as FlatReqFilter;
|
||||
expect(canMergeFilters(a, b)).toEqual(false);
|
||||
});
|
||||
it("should have 1 distance, arrays", () => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user