refactor request builder to handle relay hints
This commit is contained in:
parent
ca92b365e0
commit
9a33466c7c
@ -42,6 +42,7 @@ export default function useRelaysFeedFollows(pubkeys: HexKey[]): Array<RelayList
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// instead of discarding the follow list we should also use it for follow graph
|
||||||
function mapFromContactList(notes: Array<TaggedRawEvent>): Array<RelayList> {
|
function mapFromContactList(notes: Array<TaggedRawEvent>): Array<RelayList> {
|
||||||
return notes.map(ev => {
|
return notes.map(ev => {
|
||||||
if (ev.content !== "" && ev.content !== "{}" && ev.content.startsWith("{") && ev.content.endsWith("}")) {
|
if (ev.content !== "" && ev.content !== "{}" && ev.content.startsWith("{") && ev.content.endsWith("}")) {
|
||||||
|
@ -6,10 +6,18 @@ import { FlatNoteStore, RequestBuilder } from "System";
|
|||||||
import useRequestBuilder from "Hooks/useRequestBuilder";
|
import useRequestBuilder from "Hooks/useRequestBuilder";
|
||||||
import useLogin from "Hooks/useLogin";
|
import useLogin from "Hooks/useLogin";
|
||||||
|
|
||||||
|
interface RelayTaggedEventId {
|
||||||
|
id: u256;
|
||||||
|
relay?: string;
|
||||||
|
}
|
||||||
export default function useThreadFeed(link: NostrLink) {
|
export default function useThreadFeed(link: NostrLink) {
|
||||||
const [trackingEvents, setTrackingEvent] = useState<u256[]>([link.id]);
|
const linkTagged = {
|
||||||
|
id: link.id,
|
||||||
|
relay: link.relays?.[0],
|
||||||
|
};
|
||||||
|
const [trackingEvents, setTrackingEvent] = useState<Array<RelayTaggedEventId>>([linkTagged]);
|
||||||
const [trackingATags, setTrackingATags] = useState<string[]>([]);
|
const [trackingATags, setTrackingATags] = useState<string[]>([]);
|
||||||
const [allEvents, setAllEvents] = useState<u256[]>([link.id]);
|
const [allEvents, setAllEvents] = useState<Array<RelayTaggedEventId>>([linkTagged]);
|
||||||
const pref = useLogin().preferences;
|
const pref = useLogin().preferences;
|
||||||
|
|
||||||
const sub = useMemo(() => {
|
const sub = useMemo(() => {
|
||||||
@ -17,7 +25,10 @@ export default function useThreadFeed(link: NostrLink) {
|
|||||||
sub.withOptions({
|
sub.withOptions({
|
||||||
leaveOpen: true,
|
leaveOpen: true,
|
||||||
});
|
});
|
||||||
sub.withFilter().ids(trackingEvents);
|
const fTracking = sub.withFilter();
|
||||||
|
for (const te of trackingEvents) {
|
||||||
|
fTracking.id(te.id, te.relay);
|
||||||
|
}
|
||||||
sub
|
sub
|
||||||
.withFilter()
|
.withFilter()
|
||||||
.kinds(
|
.kinds(
|
||||||
@ -25,7 +36,10 @@ export default function useThreadFeed(link: NostrLink) {
|
|||||||
? [EventKind.Reaction, EventKind.TextNote, EventKind.Repost, EventKind.ZapReceipt]
|
? [EventKind.Reaction, EventKind.TextNote, EventKind.Repost, EventKind.ZapReceipt]
|
||||||
: [EventKind.TextNote, EventKind.ZapReceipt, EventKind.Repost]
|
: [EventKind.TextNote, EventKind.ZapReceipt, EventKind.Repost]
|
||||||
)
|
)
|
||||||
.tag("e", allEvents);
|
.tag(
|
||||||
|
"e",
|
||||||
|
allEvents.map(a => a.id)
|
||||||
|
);
|
||||||
|
|
||||||
if (trackingATags.length > 0) {
|
if (trackingATags.length > 0) {
|
||||||
const parsed = trackingATags.map(a => a.split(":"));
|
const parsed = trackingATags.map(a => a.split(":"));
|
||||||
@ -45,16 +59,27 @@ export default function useThreadFeed(link: NostrLink) {
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setTrackingATags([]);
|
setTrackingATags([]);
|
||||||
setTrackingEvent([link.id]);
|
setTrackingEvent([linkTagged]);
|
||||||
setAllEvents([link.id]);
|
setAllEvents([linkTagged]);
|
||||||
}, [link.id]);
|
}, [link.id]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (store.data) {
|
if (store.data) {
|
||||||
const mainNotes = store.data?.filter(a => a.kind === EventKind.TextNote || a.kind === EventKind.Polls) ?? [];
|
const mainNotes = store.data?.filter(a => a.kind === EventKind.TextNote || a.kind === EventKind.Polls) ?? [];
|
||||||
|
|
||||||
const eTags = mainNotes.map(a => a.tags.filter(b => b[0] === "e").map(b => b[1])).flat();
|
const eTags = mainNotes
|
||||||
const eTagsMissing = eTags.filter(a => !mainNotes.some(b => b.id === a));
|
.map(a =>
|
||||||
|
a.tags
|
||||||
|
.filter(b => b[0] === "e")
|
||||||
|
.map(b => {
|
||||||
|
return {
|
||||||
|
id: b[1],
|
||||||
|
relay: b[2],
|
||||||
|
};
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.flat();
|
||||||
|
const eTagsMissing = eTags.filter(a => !mainNotes.some(b => b.id === a.id));
|
||||||
setTrackingEvent(s => appendDedupe(s, eTagsMissing));
|
setTrackingEvent(s => appendDedupe(s, eTagsMissing));
|
||||||
setAllEvents(s => appendDedupe(s, eTags));
|
setAllEvents(s => appendDedupe(s, eTags));
|
||||||
|
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
import { RawReqFilter } from "@snort/nostr";
|
import { FullRelaySettings, RawReqFilter } from "@snort/nostr";
|
||||||
import { UserRelays } from "Cache/UserRelayCache";
|
|
||||||
import { unwrap } from "SnortUtils";
|
import { unwrap } from "SnortUtils";
|
||||||
import debug from "debug";
|
import debug from "debug";
|
||||||
|
|
||||||
@ -15,18 +14,24 @@ export interface RelayTaggedFilters {
|
|||||||
filters: Array<RawReqFilter>;
|
filters: Array<RawReqFilter>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function splitAllByWriteRelays(filters: Array<RawReqFilter>) {
|
export interface RelayCache {
|
||||||
const allSplit = filters.map(splitByWriteRelays).reduce((acc, v) => {
|
get(pubkey?: string): Array<FullRelaySettings> | undefined;
|
||||||
for (const vn of v) {
|
}
|
||||||
const existing = acc.get(vn.relay);
|
|
||||||
if (existing) {
|
export function splitAllByWriteRelays(cache: RelayCache, filters: Array<RawReqFilter>) {
|
||||||
existing.push(vn.filter);
|
const allSplit = filters
|
||||||
} else {
|
.map(a => splitByWriteRelays(cache, a))
|
||||||
acc.set(vn.relay, [vn.filter]);
|
.reduce((acc, v) => {
|
||||||
|
for (const vn of v) {
|
||||||
|
const existing = acc.get(vn.relay);
|
||||||
|
if (existing) {
|
||||||
|
existing.push(vn.filter);
|
||||||
|
} else {
|
||||||
|
acc.set(vn.relay, [vn.filter]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
return acc;
|
||||||
return acc;
|
}, new Map<string, Array<RawReqFilter>>());
|
||||||
}, new Map<string, Array<RawReqFilter>>());
|
|
||||||
|
|
||||||
return [...allSplit.entries()].map(([k, v]) => {
|
return [...allSplit.entries()].map(([k, v]) => {
|
||||||
return {
|
return {
|
||||||
@ -41,7 +46,7 @@ export function splitAllByWriteRelays(filters: Array<RawReqFilter>) {
|
|||||||
* @param filter
|
* @param filter
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
export function splitByWriteRelays(filter: RawReqFilter): Array<RelayTaggedFilter> {
|
export function splitByWriteRelays(cache: RelayCache, filter: RawReqFilter): Array<RelayTaggedFilter> {
|
||||||
if ((filter.authors?.length ?? 0) === 0)
|
if ((filter.authors?.length ?? 0) === 0)
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@ -53,7 +58,7 @@ export function splitByWriteRelays(filter: RawReqFilter): Array<RelayTaggedFilte
|
|||||||
const allRelays = unwrap(filter.authors).map(a => {
|
const allRelays = unwrap(filter.authors).map(a => {
|
||||||
return {
|
return {
|
||||||
key: a,
|
key: a,
|
||||||
relays: UserRelays.getFromCache(a)?.relays,
|
relays: cache.get(a)?.filter(a => a.settings.write),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import { TaggedRawEvent, u256 } from "@snort/nostr";
|
import { TaggedRawEvent, u256 } from "@snort/nostr";
|
||||||
import { findTag } from "SnortUtils";
|
import { appendDedupe, findTag } from "SnortUtils";
|
||||||
|
|
||||||
export interface StoreSnapshot<TSnapshot> {
|
export interface StoreSnapshot<TSnapshot> {
|
||||||
data: TSnapshot | undefined;
|
data: TSnapshot | undefined;
|
||||||
@ -142,6 +142,11 @@ export class FlatNoteStore extends HookedNoteStore<Readonly<Array<TaggedRawEvent
|
|||||||
this.#events.push(a);
|
this.#events.push(a);
|
||||||
this.#ids.add(a.id);
|
this.#ids.add(a.id);
|
||||||
changes.push(a);
|
changes.push(a);
|
||||||
|
} else {
|
||||||
|
const existing = this.#events.find(b => b.id === a.id);
|
||||||
|
if (existing) {
|
||||||
|
existing.relays = appendDedupe(existing.relays, a.relays);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -13,28 +13,18 @@ class QueryTrace {
|
|||||||
readonly relay: string;
|
readonly relay: string;
|
||||||
readonly connId: string;
|
readonly connId: string;
|
||||||
readonly start: number;
|
readonly start: number;
|
||||||
readonly leaveOpen: boolean;
|
|
||||||
sent?: number;
|
sent?: number;
|
||||||
eose?: number;
|
eose?: number;
|
||||||
close?: number;
|
close?: number;
|
||||||
#wasForceClosed = false;
|
#wasForceClosed = false;
|
||||||
readonly #fnClose: (id: string) => void;
|
readonly #fnClose: (id: string) => void;
|
||||||
readonly #fnProgress: () => void;
|
readonly #fnProgress: () => void;
|
||||||
readonly #log = debug("QueryTrace");
|
|
||||||
|
|
||||||
constructor(
|
constructor(sub: string, relay: string, connId: string, fnClose: (id: string) => void, fnProgress: () => void) {
|
||||||
sub: string,
|
|
||||||
relay: string,
|
|
||||||
connId: string,
|
|
||||||
leaveOpen: boolean,
|
|
||||||
fnClose: (id: string) => void,
|
|
||||||
fnProgress: () => void
|
|
||||||
) {
|
|
||||||
this.id = uuid();
|
this.id = uuid();
|
||||||
this.subId = sub;
|
this.subId = sub;
|
||||||
this.relay = relay;
|
this.relay = relay;
|
||||||
this.connId = connId;
|
this.connId = connId;
|
||||||
this.leaveOpen = leaveOpen;
|
|
||||||
this.start = unixNowMs();
|
this.start = unixNowMs();
|
||||||
this.#fnClose = fnClose;
|
this.#fnClose = fnClose;
|
||||||
this.#fnProgress = fnProgress;
|
this.#fnProgress = fnProgress;
|
||||||
@ -48,10 +38,6 @@ class QueryTrace {
|
|||||||
gotEose() {
|
gotEose() {
|
||||||
this.eose = unixNowMs();
|
this.eose = unixNowMs();
|
||||||
this.#fnProgress();
|
this.#fnProgress();
|
||||||
if (!this.leaveOpen) {
|
|
||||||
this.sendClose();
|
|
||||||
}
|
|
||||||
//this.#log("[EOSE] %s %s", this.subId, this.relay);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
forceEose() {
|
forceEose() {
|
||||||
@ -59,7 +45,6 @@ class QueryTrace {
|
|||||||
this.#wasForceClosed = true;
|
this.#wasForceClosed = true;
|
||||||
this.#fnProgress();
|
this.#fnProgress();
|
||||||
this.sendClose();
|
this.sendClose();
|
||||||
//this.#log("[F-EOSE] %s %s", this.subId, this.relay);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sendClose() {
|
sendClose() {
|
||||||
@ -117,10 +102,16 @@ export interface QueryBase {
|
|||||||
/**
|
/**
|
||||||
* Active or queued query on the system
|
* Active or queued query on the system
|
||||||
*/
|
*/
|
||||||
export class Query implements QueryBase {
|
export class Query {
|
||||||
|
/**
|
||||||
|
* Uniquie ID of this query
|
||||||
|
*/
|
||||||
id: string;
|
id: string;
|
||||||
filters: Array<RawReqFilter>;
|
|
||||||
relays?: Array<string>;
|
/**
|
||||||
|
* A merged set of all filters send to relays for this query
|
||||||
|
*/
|
||||||
|
filters: Array<RawReqFilter> = [];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Which relays this query has already been executed on
|
* Which relays this query has already been executed on
|
||||||
@ -150,9 +141,8 @@ export class Query implements QueryBase {
|
|||||||
subQueryCounter = 0;
|
subQueryCounter = 0;
|
||||||
#log = debug("Query");
|
#log = debug("Query");
|
||||||
|
|
||||||
constructor(id: string, filters: Array<RawReqFilter>, feed: NoteStore) {
|
constructor(id: string, feed: NoteStore) {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
this.filters = filters;
|
|
||||||
this.#feed = feed;
|
this.#feed = feed;
|
||||||
this.#checkTraces();
|
this.#checkTraces();
|
||||||
}
|
}
|
||||||
@ -181,14 +171,7 @@ export class Query implements QueryBase {
|
|||||||
this.#stopCheckTraces();
|
this.#stopCheckTraces();
|
||||||
}
|
}
|
||||||
|
|
||||||
sendToRelay(c: Connection) {
|
sendToRelay(c: Connection, subq: QueryBase) {
|
||||||
if (!this.#canSendQuery(c, this)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.#sendQueryInternal(c, this);
|
|
||||||
}
|
|
||||||
|
|
||||||
sendSubQueryToRelay(c: Connection, subq: QueryBase) {
|
|
||||||
if (!this.#canSendQuery(c, subq)) {
|
if (!this.#canSendQuery(c, subq)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -200,9 +183,6 @@ export class Query implements QueryBase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
sendClose() {
|
sendClose() {
|
||||||
for (const qt of this.#tracing) {
|
|
||||||
qt.sendClose();
|
|
||||||
}
|
|
||||||
for (const qt of this.#tracing) {
|
for (const qt of this.#tracing) {
|
||||||
qt.sendClose();
|
qt.sendClose();
|
||||||
}
|
}
|
||||||
@ -212,6 +192,9 @@ export class Query implements QueryBase {
|
|||||||
eose(sub: string, conn: Readonly<Connection>) {
|
eose(sub: string, conn: Readonly<Connection>) {
|
||||||
const qt = this.#tracing.find(a => a.subId === sub && a.connId === conn.Id);
|
const qt = this.#tracing.find(a => a.subId === sub && a.connId === conn.Id);
|
||||||
qt?.gotEose();
|
qt?.gotEose();
|
||||||
|
if (!this.leaveOpen) {
|
||||||
|
qt?.sendClose();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -270,7 +253,6 @@ export class Query implements QueryBase {
|
|||||||
q.id,
|
q.id,
|
||||||
c.Address,
|
c.Address,
|
||||||
c.Id,
|
c.Id,
|
||||||
this.leaveOpen,
|
|
||||||
x => c.CloseReq(x),
|
x => c.CloseReq(x),
|
||||||
() => this.#onProgress()
|
() => this.#onProgress()
|
||||||
);
|
);
|
||||||
|
@ -1,22 +1,29 @@
|
|||||||
|
import { RelayCache } from "./GossipModel";
|
||||||
import { RequestBuilder } from "./RequestBuilder";
|
import { RequestBuilder } from "./RequestBuilder";
|
||||||
import { describe, expect } from "@jest/globals";
|
import { describe, expect } from "@jest/globals";
|
||||||
|
|
||||||
describe("RequestBuilder", () => {
|
describe("RequestBuilder", () => {
|
||||||
|
const relayCache = {
|
||||||
|
get: () => {
|
||||||
|
return undefined;
|
||||||
|
},
|
||||||
|
} as RelayCache;
|
||||||
|
|
||||||
describe("basic", () => {
|
describe("basic", () => {
|
||||||
test("empty filter", () => {
|
test("empty filter", () => {
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter();
|
b.withFilter();
|
||||||
expect(b.build()).toEqual([{}]);
|
expect(b.build(relayCache)).toEqual([{}]);
|
||||||
});
|
});
|
||||||
test("only kind", () => {
|
test("only kind", () => {
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().kinds([0]);
|
b.withFilter().kinds([0]);
|
||||||
expect(b.build()).toEqual([{ kinds: [0] }]);
|
expect(b.build(relayCache)).toEqual([{ kinds: [0] }]);
|
||||||
});
|
});
|
||||||
test("empty authors", () => {
|
test("empty authors", () => {
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().authors([]);
|
b.withFilter().authors([]);
|
||||||
expect(b.build()).toEqual([{ authors: [] }]);
|
expect(b.build(relayCache)).toEqual([{ authors: [] }]);
|
||||||
});
|
});
|
||||||
test("authors/kinds/ids", () => {
|
test("authors/kinds/ids", () => {
|
||||||
const authors = ["a1", "a2"];
|
const authors = ["a1", "a2"];
|
||||||
@ -24,7 +31,7 @@ describe("RequestBuilder", () => {
|
|||||||
const ids = ["id1", "id2", "id3"];
|
const ids = ["id1", "id2", "id3"];
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().authors(authors).kinds(kinds).ids(ids);
|
b.withFilter().authors(authors).kinds(kinds).ids(ids);
|
||||||
expect(b.build()).toEqual([{ ids, authors, kinds }]);
|
expect(b.build(relayCache)).toEqual([{ ids, authors, kinds }]);
|
||||||
});
|
});
|
||||||
test("authors and kinds, duplicates removed", () => {
|
test("authors and kinds, duplicates removed", () => {
|
||||||
const authors = ["a1", "a2"];
|
const authors = ["a1", "a2"];
|
||||||
@ -32,12 +39,12 @@ describe("RequestBuilder", () => {
|
|||||||
const ids = ["id1", "id2", "id3"];
|
const ids = ["id1", "id2", "id3"];
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().ids(ids).authors(authors).kinds(kinds).ids(ids).authors(authors).kinds(kinds);
|
b.withFilter().ids(ids).authors(authors).kinds(kinds).ids(ids).authors(authors).kinds(kinds);
|
||||||
expect(b.build()).toEqual([{ ids, authors, kinds }]);
|
expect(b.build(relayCache)).toEqual([{ ids, authors, kinds }]);
|
||||||
});
|
});
|
||||||
test("search", () => {
|
test("search", () => {
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().kinds([1]).search("test-search");
|
b.withFilter().kinds([1]).search("test-search");
|
||||||
expect(b.build()).toEqual([{ kinds: [1], search: "test-search" }]);
|
expect(b.build(relayCache)).toEqual([{ kinds: [1], search: "test-search" }]);
|
||||||
});
|
});
|
||||||
test("timeline", () => {
|
test("timeline", () => {
|
||||||
const authors = ["a1", "a2"];
|
const authors = ["a1", "a2"];
|
||||||
@ -46,7 +53,7 @@ describe("RequestBuilder", () => {
|
|||||||
const since = 5;
|
const since = 5;
|
||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
||||||
expect(b.build()).toEqual([{ kinds, authors, until, since }]);
|
expect(b.build(relayCache)).toEqual([{ kinds, authors, until, since }]);
|
||||||
});
|
});
|
||||||
test("multi-filter timeline", () => {
|
test("multi-filter timeline", () => {
|
||||||
const authors = ["a1", "a2"];
|
const authors = ["a1", "a2"];
|
||||||
@ -56,7 +63,7 @@ describe("RequestBuilder", () => {
|
|||||||
const b = new RequestBuilder("test");
|
const b = new RequestBuilder("test");
|
||||||
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
||||||
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
b.withFilter().kinds(kinds).authors(authors).since(since).until(until);
|
||||||
expect(b.build()).toEqual([
|
expect(b.build(relayCache)).toEqual([
|
||||||
{ kinds, authors, until, since },
|
{ kinds, authors, until, since },
|
||||||
{ kinds, authors, until, since },
|
{ kinds, authors, until, since },
|
||||||
]);
|
]);
|
||||||
|
@ -1,10 +1,14 @@
|
|||||||
import { RawReqFilter, u256, HexKey, EventKind } from "@snort/nostr";
|
import { RawReqFilter, u256, HexKey, EventKind } from "@snort/nostr";
|
||||||
import { appendDedupe } from "SnortUtils";
|
import { appendDedupe, dedupe } from "SnortUtils";
|
||||||
|
import { QueryBase } from "./Query";
|
||||||
|
import { diffFilters } from "./RequestSplitter";
|
||||||
|
import { RelayCache, splitByWriteRelays } from "./GossipModel";
|
||||||
|
import { mergeSimilar } from "./RequestMerger";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Which strategy is used when building REQ filters
|
* Which strategy is used when building REQ filters
|
||||||
*/
|
*/
|
||||||
export enum NostrRequestStrategy {
|
export enum RequestStrategy {
|
||||||
/**
|
/**
|
||||||
* Use the users default relays to fetch events,
|
* Use the users default relays to fetch events,
|
||||||
* this is the fallback option when there is no better way to query a given filter set
|
* this is the fallback option when there is no better way to query a given filter set
|
||||||
@ -26,10 +30,9 @@ export enum NostrRequestStrategy {
|
|||||||
* A built REQ filter ready for sending to System
|
* A built REQ filter ready for sending to System
|
||||||
*/
|
*/
|
||||||
export interface BuiltRawReqFilter {
|
export interface BuiltRawReqFilter {
|
||||||
id: string;
|
filter: RawReqFilter;
|
||||||
filter: Array<RawReqFilter>;
|
relay: string;
|
||||||
relays: Array<string>;
|
strategy: RequestStrategy;
|
||||||
strategy: NostrRequestStrategy;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestBuilderOptions {
|
export interface RequestBuilderOptions {
|
||||||
@ -76,8 +79,55 @@ export class RequestBuilder {
|
|||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
build(): Array<RawReqFilter> {
|
buildRaw(): Array<RawReqFilter> {
|
||||||
return this.#builders.map(a => a.filter);
|
return this.#builders.map(f => f.filter);
|
||||||
|
}
|
||||||
|
|
||||||
|
build(relays: RelayCache): Array<BuiltRawReqFilter> {
|
||||||
|
const expanded = this.#builders.map(a => a.build(relays)).flat();
|
||||||
|
return this.#mergeSimilar(expanded);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detects a change in request from a previous set of filters
|
||||||
|
* @param q All previous filters merged
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
buildDiff(relays: RelayCache, q: QueryBase): Array<BuiltRawReqFilter> {
|
||||||
|
const next = this.buildRaw();
|
||||||
|
const diff = diffFilters(q.filters, next);
|
||||||
|
if (diff.changed) {
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests
|
||||||
|
* @param expanded
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
#mergeSimilar(expanded: Array<BuiltRawReqFilter>) {
|
||||||
|
const relayMerged = expanded.reduce((acc, v) => {
|
||||||
|
const existing = acc.get(v.relay);
|
||||||
|
if (existing) {
|
||||||
|
existing.push(v);
|
||||||
|
} else {
|
||||||
|
acc.set(v.relay, [v]);
|
||||||
|
}
|
||||||
|
return acc;
|
||||||
|
}, new Map<string, Array<BuiltRawReqFilter>>());
|
||||||
|
|
||||||
|
const filtersSquashed = [...relayMerged.values()].flatMap(a => {
|
||||||
|
return mergeSimilar(a.map(b => b.filter)).map(b => {
|
||||||
|
return {
|
||||||
|
filter: b,
|
||||||
|
relay: a[0].relay,
|
||||||
|
strategy: a[0].strategy,
|
||||||
|
} as BuiltRawReqFilter;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return filtersSquashed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,7 +136,7 @@ export class RequestBuilder {
|
|||||||
*/
|
*/
|
||||||
export class RequestFilterBuilder {
|
export class RequestFilterBuilder {
|
||||||
#filter: RawReqFilter = {};
|
#filter: RawReqFilter = {};
|
||||||
#relayHints: Map<u256, Array<string>> = new Map();
|
#relayHints = new Map<u256, Array<string>>();
|
||||||
|
|
||||||
get filter() {
|
get filter() {
|
||||||
return { ...this.#filter };
|
return { ...this.#filter };
|
||||||
@ -149,4 +199,44 @@ export class RequestFilterBuilder {
|
|||||||
this.#filter.search = keyword;
|
this.#filter.search = keyword;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build/expand this filter into a set of relay specific queries
|
||||||
|
*/
|
||||||
|
build(relays: RelayCache): Array<BuiltRawReqFilter> {
|
||||||
|
// when querying for specific event ids with relay hints
|
||||||
|
// take the first approach which is to split the filter by relay
|
||||||
|
if (this.#filter.ids && this.#relayHints.size > 0) {
|
||||||
|
const relays = dedupe([...this.#relayHints.values()].flat());
|
||||||
|
return relays.map(r => {
|
||||||
|
return {
|
||||||
|
filter: {
|
||||||
|
...this.#filter,
|
||||||
|
ids: [...this.#relayHints.entries()].filter(([, v]) => v.includes(r)).map(([k]) => k),
|
||||||
|
},
|
||||||
|
relay: r,
|
||||||
|
strategy: RequestStrategy.RelayHintedEventIds,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// If any authors are set use the gossip model to fetch data for each author
|
||||||
|
if (this.#filter.authors) {
|
||||||
|
const split = splitByWriteRelays(relays, this.#filter);
|
||||||
|
return split.map(a => {
|
||||||
|
return {
|
||||||
|
...a,
|
||||||
|
strategy: RequestStrategy.AuthorsRelays,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
filter: this.filter,
|
||||||
|
relay: "*",
|
||||||
|
strategy: RequestStrategy.DefaultRelays,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
5
packages/app/src/System/RequestMerger.ts
Normal file
5
packages/app/src/System/RequestMerger.ts
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
import { RawReqFilter } from "@snort/nostr";
|
||||||
|
|
||||||
|
export function mergeSimilar(filters: Array<RawReqFilter>): Array<RawReqFilter> {
|
||||||
|
return filters;
|
||||||
|
}
|
@ -15,6 +15,9 @@ export function diffFilters(a: Array<RawReqFilter>, b: Array<RawReqFilter>) {
|
|||||||
for (const [k, v] of Object.entries(bN)) {
|
for (const [k, v] of Object.entries(bN)) {
|
||||||
if (Array.isArray(v)) {
|
if (Array.isArray(v)) {
|
||||||
const prevArray = prev[k] as Array<string | number>;
|
const prevArray = prev[k] as Array<string | number>;
|
||||||
|
if (!prevArray) {
|
||||||
|
throw new Error(`Tried to add new filter prop ${k} which isnt supported!`);
|
||||||
|
}
|
||||||
const thisArray = v as Array<string | number>;
|
const thisArray = v as Array<string | number>;
|
||||||
const added = thisArray.filter(a => !prevArray.includes(a));
|
const added = thisArray.filter(a => !prevArray.includes(a));
|
||||||
// support adding new values to array, removing values is ignored since we only care about getting new values
|
// support adding new values to array, removing values is ignored since we only care about getting new values
|
||||||
|
@ -15,6 +15,7 @@ import { diffFilters } from "./RequestSplitter";
|
|||||||
import { Query, QueryBase } from "./Query";
|
import { Query, QueryBase } from "./Query";
|
||||||
import { splitAllByWriteRelays } from "./GossipModel";
|
import { splitAllByWriteRelays } from "./GossipModel";
|
||||||
import ExternalStore from "ExternalStore";
|
import ExternalStore from "ExternalStore";
|
||||||
|
import { UserRelays } from "Cache/UserRelayCache";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
NoteStore,
|
NoteStore,
|
||||||
@ -59,6 +60,9 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> {
|
|||||||
HandleAuth?: AuthHandler;
|
HandleAuth?: AuthHandler;
|
||||||
|
|
||||||
#log = debug("System");
|
#log = debug("System");
|
||||||
|
#relayCache = {
|
||||||
|
get: (pk?: string) => UserRelays.getFromCache(pk)?.relays,
|
||||||
|
};
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
@ -188,17 +192,42 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> {
|
|||||||
|
|
||||||
if (!req) return new type();
|
if (!req) return new type();
|
||||||
|
|
||||||
if (this.Queries.has(req.id)) {
|
const existing = this.Queries.get(req.id);
|
||||||
const filters = req.build();
|
if (existing) {
|
||||||
const q = unwrap(this.Queries.get(req.id));
|
const filters = req.buildDiff(this.#relayCache, existing);
|
||||||
q.unCancel();
|
existing.unCancel();
|
||||||
|
|
||||||
const diff = diffFilters(q.filters, filters);
|
if (filters.length === 0 && !req.options?.skipDiff) {
|
||||||
if (!diff.changed && !req.options?.skipDiff) {
|
|
||||||
this.notifyChange();
|
this.notifyChange();
|
||||||
return unwrap(q.feed) as Readonly<T>;
|
return existing.feed as Readonly<T>;
|
||||||
} else {
|
} else {
|
||||||
const splitFilters = splitAllByWriteRelays(filters);
|
for (const subQ of filters) {
|
||||||
|
this.SendQuery(
|
||||||
|
existing,
|
||||||
|
{
|
||||||
|
id: `${existing.id}-${existing.subQueryCounter++}`,
|
||||||
|
filters: [subQ.filter],
|
||||||
|
relays: [],
|
||||||
|
},
|
||||||
|
(q, s, c) => q.sendSubQueryToRelay(c, s)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
q.filters = filters;
|
||||||
|
this.notifyChange();
|
||||||
|
return q.feed as Readonly<T>;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const store = new type();
|
||||||
|
|
||||||
|
const filters = req.build(this.#relayCache);
|
||||||
|
const q = new Query(req.id, store);
|
||||||
|
if (req.options?.leaveOpen) {
|
||||||
|
q.leaveOpen = req.options.leaveOpen;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.Queries.set(rb.id, q);
|
||||||
|
const splitFilters = splitAllByWriteRelays(filters);
|
||||||
|
if (splitFilters.length > 1) {
|
||||||
for (const sf of splitFilters) {
|
for (const sf of splitFilters) {
|
||||||
const subQ = {
|
const subQ = {
|
||||||
id: `${q.id}-${q.subQueryCounter++}`,
|
id: `${q.id}-${q.subQueryCounter++}`,
|
||||||
@ -207,45 +236,14 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> {
|
|||||||
} as QueryBase;
|
} as QueryBase;
|
||||||
this.SendQuery(q, subQ, (q, s, c) => q.sendSubQueryToRelay(c, s));
|
this.SendQuery(q, subQ, (q, s, c) => q.sendSubQueryToRelay(c, s));
|
||||||
}
|
}
|
||||||
q.filters = filters;
|
} else {
|
||||||
this.notifyChange();
|
this.SendQuery(q, q, (q, s, c) => q.sendToRelay(c));
|
||||||
return q.feed as Readonly<T>;
|
|
||||||
}
|
}
|
||||||
} else {
|
this.notifyChange();
|
||||||
return this.AddQuery<T>(type, req);
|
return store;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AddQuery<T extends NoteStore>(type: { new (): T }, rb: RequestBuilder): T {
|
|
||||||
const store = new type();
|
|
||||||
|
|
||||||
const filters = rb.build();
|
|
||||||
const q = new Query(rb.id, filters, store);
|
|
||||||
if (rb.options?.leaveOpen) {
|
|
||||||
q.leaveOpen = rb.options.leaveOpen;
|
|
||||||
}
|
|
||||||
if (rb.options?.relays && (rb.options?.relays?.length ?? 0) > 0) {
|
|
||||||
q.relays = rb.options.relays;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.Queries.set(rb.id, q);
|
|
||||||
const splitFilters = splitAllByWriteRelays(filters);
|
|
||||||
if (splitFilters.length > 1) {
|
|
||||||
for (const sf of splitFilters) {
|
|
||||||
const subQ = {
|
|
||||||
id: `${q.id}-${q.subQueryCounter++}`,
|
|
||||||
filters: sf.filters,
|
|
||||||
relays: sf.relay ? [sf.relay] : undefined,
|
|
||||||
} as QueryBase;
|
|
||||||
this.SendQuery(q, subQ, (q, s, c) => q.sendSubQueryToRelay(c, s));
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.SendQuery(q, q, (q, s, c) => q.sendToRelay(c));
|
|
||||||
}
|
|
||||||
this.notifyChange();
|
|
||||||
return store;
|
|
||||||
}
|
|
||||||
|
|
||||||
CancelQuery(sub: string) {
|
CancelQuery(sub: string) {
|
||||||
const q = this.Queries.get(sub);
|
const q = this.Queries.get(sub);
|
||||||
if (q) {
|
if (q) {
|
||||||
|
@ -211,7 +211,10 @@ export class Connection {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "EVENT": {
|
case "EVENT": {
|
||||||
this.OnEvent?.(msg[1], msg[2]);
|
this.OnEvent?.(msg[1], {
|
||||||
|
...msg[2],
|
||||||
|
relays: [this.Address]
|
||||||
|
});
|
||||||
this.Stats.EventsReceived++;
|
this.Stats.EventsReceived++;
|
||||||
this.#UpdateState();
|
this.#UpdateState();
|
||||||
break;
|
break;
|
||||||
|
Loading…
Reference in New Issue
Block a user