diff --git a/packages/app/src/Components/Embed/LinkPreview.tsx b/packages/app/src/Components/Embed/LinkPreview.tsx index bbe59321..71428c09 100644 --- a/packages/app/src/Components/Embed/LinkPreview.tsx +++ b/packages/app/src/Components/Embed/LinkPreview.tsx @@ -56,7 +56,7 @@ const LinkPreview = ({ url }: { url: string }) => { const urlTags = ["og:video:secure_url", "og:video:url", "og:video"]; const link = preview?.og_tags?.find(a => urlTags.includes(a[0].toLowerCase()))?.[1]; const videoType = preview?.og_tags?.find(a => a[0].toLowerCase() === "og:video:type")?.[1] ?? "video/mp4"; - if (link) { + if (link && videoType.startsWith("video/")) { return ; } } diff --git a/packages/app/src/Components/Event/Note/Note.tsx b/packages/app/src/Components/Event/Note/Note.tsx index e5f263ef..9a343e05 100644 --- a/packages/app/src/Components/Event/Note/Note.tsx +++ b/packages/app/src/Components/Event/Note/Note.tsx @@ -149,11 +149,12 @@ function useGoToEvent(props, options) { } function Reaction({ ev }: { ev: TaggedNostrEvent }) { - const reactedToTag = ev.tags.find((tag: string[]) => tag[0] === "e"); + const reactedToTag = ev.tags.findLast(tag => tag[0] === "e"); + const pTag = ev.tags.findLast(tag => tag[0] === "p"); if (!reactedToTag?.length) { return null; } - const link = NostrLink.fromTag(reactedToTag); + const link = NostrLink.fromTag(reactedToTag, pTag?.[1]); return (
diff --git a/packages/app/src/Feed/ThreadFeed.ts b/packages/app/src/Feed/ThreadFeed.ts index f944bafb..c239d525 100644 --- a/packages/app/src/Feed/ThreadFeed.ts +++ b/packages/app/src/Feed/ThreadFeed.ts @@ -1,10 +1,14 @@ import { EventExt, EventKind, NostrLink, RequestBuilder } from "@snort/system"; -import { useReactions, useRequestBuilder } from "@snort/system-react"; -import { useEffect, useMemo, useState } from "react"; +import { SnortContext, useRequestBuilder } from "@snort/system-react"; +import { useContext, useEffect, useMemo, useState } from "react"; + +import { randomSample } from "@/Utils"; export default function useThreadFeed(link: NostrLink) { const [root, setRoot] = useState(); + const [rootRelays, setRootRelays] = useState>(); const [allEvents, setAllEvents] = useState>([]); + const system = useContext(SnortContext); const sub = useMemo(() => { const sub = new RequestBuilder(`thread:${link.id.slice(0, 12)}`); @@ -13,7 +17,7 @@ export default function useThreadFeed(link: NostrLink) { }); sub.withFilter().link(link); if (root) { - sub.withFilter().link(root); + sub.withFilter().link(root).relay(rootRelays ?? []); } const grouped = [link, ...allEvents].reduce( (acc, v) => { @@ -24,11 +28,14 @@ export default function useThreadFeed(link: NostrLink) { {} as Record>, ); - for (const [, v] of Object.entries(grouped)) { - sub.withFilter().kinds([EventKind.TextNote]).replyToLink(v); + for (const v of Object.values(grouped)) { + sub.withFilter() + .kinds([EventKind.TextNote]) + .replyToLink(v) + .relay(rootRelays ?? []); } return sub; - }, [allEvents.length]); + }, [allEvents.length, rootRelays]); const store = useRequestBuilder(sub); @@ -57,15 +64,28 @@ export default function useThreadFeed(link: NostrLink) { ]), ); } + } else { + setRoot(link); } } } }, [store?.length]); - const reactions = useReactions(`thread:${link.id.slice(0, 12)}:reactions`, [link, ...allEvents]); + useEffect(() => { + if (root) { + const rootEvent = store?.find(a => root.matchesEvent(a)); + if (rootEvent) { + system.relayCache.buffer([rootEvent.pubkey]).then(() => { + const relays = system.relayCache.getFromCache(rootEvent.pubkey); - return { - thread: store ?? [], - reactions: reactions ?? [], - }; + if (relays) { + const readRelays = randomSample(relays.relays.filter(a => a.settings.read).map(a => a.url), 3); + setRootRelays(readRelays); + } + }) + } + } + }, [link, root, store?.length]); + + return store ?? []; } diff --git a/packages/app/src/Utils/Thread/ThreadContext.tsx b/packages/app/src/Utils/Thread/ThreadContext.tsx index 8ce0c06c..4c8d0683 100644 --- a/packages/app/src/Utils/Thread/ThreadContext.tsx +++ b/packages/app/src/Utils/Thread/ThreadContext.tsx @@ -2,13 +2,12 @@ import { TaggedNostrEvent } from "@snort/system"; import { createContext } from "react"; -interface ThreadContext { +export interface ThreadContextState { current: string; root?: TaggedNostrEvent; chains: Map>; data: Array; - reactions: Array; setCurrent: (i: string) => void; } -export const ThreadContext = createContext({} as ThreadContext); +export const ThreadContext = createContext({} as ThreadContextState); diff --git a/packages/app/src/Utils/Thread/ThreadContextWrapper.tsx b/packages/app/src/Utils/Thread/ThreadContextWrapper.tsx index 413cc58f..f43fd338 100644 --- a/packages/app/src/Utils/Thread/ThreadContextWrapper.tsx +++ b/packages/app/src/Utils/Thread/ThreadContextWrapper.tsx @@ -6,7 +6,7 @@ import { useLocation } from "react-router-dom"; import useThreadFeed from "@/Feed/ThreadFeed"; import useModeration from "@/Hooks/useModeration"; import { chainKey, replyChainKey } from "@/Utils/Thread/ChainKey"; -import { ThreadContext } from "@/Utils/Thread/ThreadContext"; +import { ThreadContext, ThreadContextState } from "@/Utils/Thread/ThreadContext"; export function ThreadContextWrapper({ link, children }: { link: NostrLink; children?: ReactNode }) { const location = useLocation(); @@ -16,8 +16,8 @@ export function ThreadContextWrapper({ link, children }: { link: NostrLink; chil const chains = useMemo(() => { const chains = new Map>(); - if (feed.thread) { - feed.thread + if (feed) { + feed ?.filter(a => !isBlocked(a.pubkey)) .forEach(v => { const replyTo = replyChainKey(v); @@ -31,30 +31,29 @@ export function ThreadContextWrapper({ link, children }: { link: NostrLink; chil }); } return chains; - }, [feed.thread]); + }, [feed]); // Root is the parent of the current note or the current note if its a root note or the root of the thread const root = useMemo(() => { const currentNote = - feed.thread?.find(a => chainKey(a) === currentId) ?? + feed?.find(a => chainKey(a) === currentId) ?? (location.state && "sig" in location.state ? (location.state as TaggedNostrEvent) : undefined); if (currentNote) { const key = replyChainKey(currentNote); if (key) { - return feed.thread?.find(a => chainKey(a) === key); + return feed?.find(a => chainKey(a) === key); } else { return currentNote; } } - }, [feed.thread.length, currentId, location]); + }, [feed.length, currentId, location]); - const ctxValue = useMemo(() => { + const ctxValue = useMemo(() => { return { current: currentId, root, chains, - reactions: feed.reactions, - data: feed.thread, + data: feed, setCurrent: v => setCurrentId(v), }; }, [root, chains]); diff --git a/packages/system-wasm/Cargo.lock b/packages/system-wasm/Cargo.lock index 227365c2..a2eead3e 100644 --- a/packages/system-wasm/Cargo.lock +++ b/packages/system-wasm/Cargo.lock @@ -23,18 +23,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" -[[package]] -name = "argon2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9" -dependencies = [ - "base64ct", - "blake2", - "cpufeatures", - "password-hash", -] - [[package]] name = "async-trait" version = "0.1.73" @@ -52,27 +40,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - [[package]] name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -283,7 +256,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -460,17 +432,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "password-hash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - [[package]] name = "plotters" version = "0.3.5" @@ -727,12 +688,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - [[package]] name = "syn" version = "2.0.31" @@ -748,7 +703,6 @@ dependencies = [ name = "system-wasm" version = "0.1.0" dependencies = [ - "argon2", "console_error_panic_hook", "criterion", "hex", diff --git a/packages/system-wasm/Cargo.toml b/packages/system-wasm/Cargo.toml index d938f066..12b17ce1 100644 --- a/packages/system-wasm/Cargo.toml +++ b/packages/system-wasm/Cargo.toml @@ -8,7 +8,6 @@ edition = "2021" crate-type = ["cdylib", "rlib"] [dependencies] -argon2 = "0.5.2" console_error_panic_hook = "0.1.7" hex = { version = "0.4.3", features = [], default-features = false } itertools = "0.11.0" diff --git a/packages/system-wasm/README.md b/packages/system-wasm/README.md index 35c32495..2512303b 100644 --- a/packages/system-wasm/README.md +++ b/packages/system-wasm/README.md @@ -1 +1,11 @@ # system-wasm + +## Building + +### Ubuntu/Debian + +```bash +sudo apt install clang +cargo install wasm-pack +yarn build +``` diff --git a/packages/system-wasm/package.json b/packages/system-wasm/package.json index 1360e5d6..3a9d7026 100644 --- a/packages/system-wasm/package.json +++ b/packages/system-wasm/package.json @@ -1,6 +1,6 @@ { "name": "@snort/system-wasm", - "version": "1.0.2", + "version": "1.0.3", "packageManager": "yarn@3.6.3", "author": "Kieran", "license": "MIT", diff --git a/packages/system-wasm/pkg/README.md b/packages/system-wasm/pkg/README.md index 35c32495..2512303b 100644 --- a/packages/system-wasm/pkg/README.md +++ b/packages/system-wasm/pkg/README.md @@ -1 +1,11 @@ # system-wasm + +## Building + +### Ubuntu/Debian + +```bash +sudo apt install clang +cargo install wasm-pack +yarn build +``` diff --git a/packages/system-wasm/pkg/system_wasm.d.ts b/packages/system-wasm/pkg/system_wasm.d.ts index 7c3dcc11..e17b552f 100644 --- a/packages/system-wasm/pkg/system_wasm.d.ts +++ b/packages/system-wasm/pkg/system_wasm.d.ts @@ -33,12 +33,6 @@ export function compress(val: any): any; * @returns {any} */ export function pow(val: any, target: any): any; -/** - * @param {any} password - * @param {any} salt - * @returns {any} - */ -export function argon2(password: any, salt: any): any; /** * @param {any} hash * @param {any} sig @@ -62,7 +56,6 @@ export interface InitOutput { readonly flat_merge: (a: number, b: number) => void; readonly compress: (a: number, b: number) => void; readonly pow: (a: number, b: number, c: number) => void; - readonly argon2: (a: number, b: number, c: number) => void; readonly schnorr_verify: (a: number, b: number, c: number, d: number) => void; readonly schnorr_verify_event: (a: number, b: number) => void; readonly rustsecp256k1_v0_9_1_context_create: (a: number) => number; diff --git a/packages/system-wasm/pkg/system_wasm.js b/packages/system-wasm/pkg/system_wasm.js index 3ebc649e..6542fa61 100644 --- a/packages/system-wasm/pkg/system_wasm.js +++ b/packages/system-wasm/pkg/system_wasm.js @@ -340,27 +340,6 @@ export function pow(val, target) { } } -/** - * @param {any} password - * @param {any} salt - * @returns {any} - */ -export function argon2(password, salt) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.argon2(retptr, addHeapObject(password), addHeapObject(salt)); - var r0 = getInt32Memory0()[retptr / 4 + 0]; - var r1 = getInt32Memory0()[retptr / 4 + 1]; - var r2 = getInt32Memory0()[retptr / 4 + 2]; - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } -} - /** * @param {any} hash * @param {any} sig @@ -484,10 +463,6 @@ function __wbg_get_imports() { const ret = new Error(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }; - imports.wbg.__wbindgen_object_clone_ref = function (arg0) { - const ret = getObject(arg0); - return addHeapObject(ret); - }; imports.wbg.__wbindgen_jsval_loose_eq = function (arg0, arg1) { const ret = getObject(arg0) == getObject(arg1); return ret; @@ -507,6 +482,10 @@ function __wbg_get_imports() { const ret = arg0; return addHeapObject(ret); }; + imports.wbg.__wbindgen_object_clone_ref = function (arg0) { + const ret = getObject(arg0); + return addHeapObject(ret); + }; imports.wbg.__wbindgen_string_new = function (arg0, arg1) { const ret = getStringFromWasm0(arg0, arg1); return addHeapObject(ret); diff --git a/packages/system-wasm/pkg/system_wasm_bg.wasm b/packages/system-wasm/pkg/system_wasm_bg.wasm index 9873f280..6fae1c8f 100644 Binary files a/packages/system-wasm/pkg/system_wasm_bg.wasm and b/packages/system-wasm/pkg/system_wasm_bg.wasm differ diff --git a/packages/system-wasm/pkg/system_wasm_bg.wasm.d.ts b/packages/system-wasm/pkg/system_wasm_bg.wasm.d.ts index d05ce246..ca0c7e8e 100644 --- a/packages/system-wasm/pkg/system_wasm_bg.wasm.d.ts +++ b/packages/system-wasm/pkg/system_wasm_bg.wasm.d.ts @@ -7,7 +7,6 @@ export function get_diff(a: number, b: number, c: number): void; export function flat_merge(a: number, b: number): void; export function compress(a: number, b: number): void; export function pow(a: number, b: number, c: number): void; -export function argon2(a: number, b: number, c: number): void; export function schnorr_verify(a: number, b: number, c: number, d: number): void; export function schnorr_verify_event(a: number, b: number): void; export function rustsecp256k1_v0_9_1_context_create(a: number): number; diff --git a/packages/system-wasm/src/diff.rs b/packages/system-wasm/src/diff.rs index 58b2af90..4b58fd0b 100644 --- a/packages/system-wasm/src/diff.rs +++ b/packages/system-wasm/src/diff.rs @@ -21,35 +21,11 @@ mod tests { fn simple_diff_same() { let prev = vec![FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }]; let next = vec![FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }]; let result = diff_filter(&prev, &next); @@ -60,52 +36,16 @@ mod tests { fn simple_diff_add() { let prev = vec![FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }]; let next = vec![ FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { id: Some("b".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ]; @@ -114,19 +54,7 @@ mod tests { result, vec![FlatReqFilter { id: Some("b".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }] ) } @@ -135,35 +63,11 @@ mod tests { fn simple_diff_replace() { let prev = vec![FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }]; let next = vec![FlatReqFilter { id: Some("b".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }]; let result = diff_filter(&prev, &next); @@ -171,19 +75,7 @@ mod tests { result, vec![FlatReqFilter { id: Some("b".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }] ) } diff --git a/packages/system-wasm/src/filter.rs b/packages/system-wasm/src/filter.rs index b53af32e..2292f193 100644 --- a/packages/system-wasm/src/filter.rs +++ b/packages/system-wasm/src/filter.rs @@ -1,9 +1,10 @@ -use serde::{Deserialize, Serialize}; use std::collections::HashSet; #[cfg(test)] use std::fmt::Debug; use std::hash::Hash; + use itertools::Itertools; +use serde::{Deserialize, Serialize}; #[derive(Clone)] enum StringOrNumberEntry<'a> { @@ -11,7 +12,7 @@ enum StringOrNumberEntry<'a> { Number((&'static str, &'a i32)), } -#[derive(PartialEq, Clone, Serialize, Deserialize)] +#[derive(PartialEq, Clone, Serialize, Deserialize, Default)] pub struct ReqFilter { #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] pub ids: Option>, @@ -33,6 +34,8 @@ pub struct ReqFilter { pub a_tag: Option>, #[serde(rename = "#g", skip_serializing_if = "Option::is_none")] pub g_tag: Option>, + #[serde(rename = "relays", skip_serializing_if = "Option::is_none")] + pub relays: Option>, #[serde(rename = "search", skip_serializing_if = "Option::is_none")] pub search: Option, #[serde(rename = "since", skip_serializing_if = "Option::is_none")] @@ -50,7 +53,7 @@ impl Debug for ReqFilter { } } -#[derive(PartialEq, PartialOrd, Clone, Serialize, Deserialize)] +#[derive(PartialEq, PartialOrd, Clone, Serialize, Deserialize, Default)] pub struct FlatReqFilter { #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] pub id: Option, @@ -74,6 +77,8 @@ pub struct FlatReqFilter { pub g_tag: Option, #[serde(rename = "search", skip_serializing_if = "Option::is_none")] pub search: Option, + #[serde(rename = "relay", skip_serializing_if = "Option::is_none")] + pub relay: Option, #[serde(rename = "since", skip_serializing_if = "Option::is_none")] pub since: Option, #[serde(rename = "until", skip_serializing_if = "Option::is_none")] @@ -117,11 +122,13 @@ impl Distance for FlatReqFilter { ret += prop_dist(&self.id, &b.id); ret += prop_dist(&self.kind, &b.kind); ret += prop_dist(&self.author, &b.author); + ret += prop_dist(&self.relay, &b.relay); ret += prop_dist(&self.e_tag, &b.e_tag); ret += prop_dist(&self.p_tag, &b.p_tag); ret += prop_dist(&self.d_tag, &b.d_tag); ret += prop_dist(&self.r_tag, &b.r_tag); ret += prop_dist(&self.t_tag, &b.t_tag); + ret += prop_dist(&self.g_tag, &b.g_tag); ret } @@ -143,26 +150,12 @@ impl CanMerge for FlatReqFilter { impl From> for ReqFilter { fn from(value: Vec<&FlatReqFilter>) -> Self { - let ret = ReqFilter { - ids: None, - authors: None, - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, - }; + let ret = Default::default(); value.iter().fold(ret, |mut acc, x| { array_prop_append(&x.id, &mut acc.ids); array_prop_append(&x.author, &mut acc.authors); array_prop_append(&x.kind, &mut acc.kinds); + array_prop_append(&x.relay, &mut acc.relays); array_prop_append(&x.e_tag, &mut acc.e_tag); array_prop_append(&x.p_tag, &mut acc.p_tag); array_prop_append(&x.t_tag, &mut acc.t_tag); @@ -182,26 +175,12 @@ impl From> for ReqFilter { impl From> for ReqFilter { fn from(value: Vec<&ReqFilter>) -> Self { - let ret = ReqFilter { - ids: None, - authors: None, - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, - }; + let ret = Default::default(); value.iter().fold(ret, |mut acc, x| { array_prop_append_vec(&x.ids, &mut acc.ids); array_prop_append_vec(&x.authors, &mut acc.authors); array_prop_append_vec(&x.kinds, &mut acc.kinds); + array_prop_append_vec(&x.relays, &mut acc.relays); array_prop_append_vec(&x.e_tag, &mut acc.e_tag); array_prop_append_vec(&x.p_tag, &mut acc.p_tag); array_prop_append_vec(&x.t_tag, &mut acc.t_tag); @@ -245,6 +224,13 @@ impl Into> for &ReqFilter { .collect(); inputs.push(t_ids); } + if let Some(relays) = &self.relays { + let t_relays = relays + .iter() + .map(|z| StringOrNumberEntry::String(("relay", z))) + .collect(); + inputs.push(t_relays); + } if let Some(e_tags) = &self.e_tag { let t_ids = e_tags .iter() @@ -313,6 +299,14 @@ impl Into> for &ReqFilter { } None }), + relay: p.iter().find_map(|q| { + if let StringOrNumberEntry::String((k, v)) = q { + if (*k).eq("relay") { + return Some((*v).to_string()); + } + } + None + }), kind: p.iter().find_map(|q| { if let StringOrNumberEntry::Number((k, v)) = q { if (*k).eq("kind") { @@ -394,6 +388,7 @@ impl Distance for ReqFilter { ret += prop_dist_vec(&self.ids, &b.ids); ret += prop_dist_vec(&self.kinds, &b.kinds); ret += prop_dist_vec(&self.authors, &b.authors); + ret += prop_dist_vec(&self.relays, &b.relays); ret += prop_dist_vec(&self.e_tag, &b.e_tag); ret += prop_dist_vec(&self.p_tag, &b.p_tag); ret += prop_dist_vec(&self.d_tag, &b.d_tag); @@ -478,9 +473,10 @@ fn array_prop_append_vec( #[cfg(test)] mod tests { - use crate::ReqFilter; use std::collections::HashSet; + use crate::filter::FlatReqFilter; + use crate::ReqFilter; #[test] fn test_expand_filter() { @@ -493,16 +489,9 @@ mod tests { kinds: Some(HashSet::from([1, 2, 3])), ids: Some(HashSet::from(["x".to_owned(), "y".to_owned()])), p_tag: Some(HashSet::from(["a".to_owned()])), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }; let output: Vec = (&input).into(); @@ -512,288 +501,162 @@ mod tests { kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("a".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, since: Some(99), - until: None, limit: Some(10), - e_tag: None, + ..Default::default() }, ]; assert_eq!(output.len(), expected.len()); diff --git a/packages/system-wasm/src/lib.rs b/packages/system-wasm/src/lib.rs index 0965af0a..de03eb07 100644 --- a/packages/system-wasm/src/lib.rs +++ b/packages/system-wasm/src/lib.rs @@ -1,6 +1,5 @@ extern crate console_error_panic_hook; -use argon2::{Argon2}; use secp256k1::{Message, XOnlyPublicKey, SECP256K1}; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -90,16 +89,6 @@ pub fn pow(val: JsValue, target: JsValue) -> Result { Ok(serde_wasm_bindgen::to_value(&val_parsed)?) } -#[wasm_bindgen] -pub fn argon2(password: JsValue, salt: JsValue) -> Result { - console_error_panic_hook::set_once(); - let password_parsed: String = serde_wasm_bindgen::from_value(password)?; - let salt_parsed: String = serde_wasm_bindgen::from_value(salt)?; - let mut key = [0u8; 32]; - Argon2::default().hash_password_into(password_parsed.as_bytes(), salt_parsed.as_bytes(), &mut key).expect("Failed to generate key"); - Ok(serde_wasm_bindgen::to_value(&hex::encode(key))?) -} - #[wasm_bindgen] pub fn schnorr_verify(hash: JsValue, sig: JsValue, pub_key: JsValue) -> Result { console_error_panic_hook::set_once(); @@ -137,15 +126,7 @@ mod tests { fn flat_merge_expanded() { let input = vec![ ReqFilter { - ids: None, kinds: Some(HashSet::from([1, 6969, 6])), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, authors: Some(HashSet::from([ "kieran".to_string(), "snort".to_string(), @@ -155,56 +136,23 @@ mod tests { ])), since: Some(1), until: Some(100), - search: None, - limit: None, + ..Default::default() }, ReqFilter { - ids: None, kinds: Some(HashSet::from([4])), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, authors: Some(HashSet::from(["kieran".to_string()])), - limit: None, + ..Default::default() }, ReqFilter { - ids: None, - authors: None, kinds: Some(HashSet::from([4])), - e_tag: None, p_tag: Some(HashSet::from(["kieran".to_string()])), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ReqFilter { - ids: None, kinds: Some(HashSet::from([1000])), authors: Some(HashSet::from(["snort".to_string()])), p_tag: Some(HashSet::from(["kieran".to_string()])), - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - e_tag: None, - limit: None, + ..Default::default() }, ]; diff --git a/packages/system-wasm/src/merge.rs b/packages/system-wasm/src/merge.rs index 56349ef1..74f4b543 100644 --- a/packages/system-wasm/src/merge.rs +++ b/packages/system-wasm/src/merge.rs @@ -59,83 +59,25 @@ mod tests { fn distance() { let a = FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }; let b = FlatReqFilter { id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }; let c = FlatReqFilter { id: Some("c".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }; let d = FlatReqFilter { id: Some("a".to_owned()), - author: None, kind: Some(1), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }; let e = FlatReqFilter { id: Some("e".to_owned()), - author: None, kind: Some(1), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }; assert_eq!(a.distance(&b), 0); assert_eq!(a.distance(&c), 1); @@ -148,51 +90,21 @@ mod tests { let a = FlatReqFilter { id: Some("0".to_owned()), author: Some("a".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(10), + ..Default::default() }; let b = FlatReqFilter { id: Some("0".to_owned()), author: Some("b".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(10), + ..Default::default() }; let output = ReqFilter { ids: Some(HashSet::from(["0".to_owned()])), authors: Some(HashSet::from(["a".to_owned(), "b".to_owned()])), - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(10), + ..Default::default() }; assert_eq!(ReqFilter::from(vec![&a, &b]), output); } @@ -202,50 +114,20 @@ mod tests { let a = FlatReqFilter { id: Some("0".to_owned()), author: Some("a".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(10), + ..Default::default() }; let b = FlatReqFilter { id: Some("0".to_owned()), author: Some("b".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(10), + ..Default::default() }; let c = FlatReqFilter { id: Some("0".to_owned()), author: Some("b".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(100), + ..Default::default() }; assert!(&a.can_merge(&b)); assert!(!&b.can_merge(&c)); @@ -257,146 +139,44 @@ mod tests { FlatReqFilter { id: Some("0".to_owned()), author: Some("a".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { id: Some("0".to_owned()), author: Some("b".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { - id: None, - author: None, kind: Some(1), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { - id: None, - author: None, kind: Some(2), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { - id: None, - author: None, kind: Some(2), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { id: Some("0".to_owned()), author: Some("c".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { - id: None, author: Some("c".to_owned()), kind: Some(1), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, FlatReqFilter { - id: None, author: Some("c".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(100), + ..Default::default() }, FlatReqFilter { id: Some("1".to_owned()), author: Some("c".to_owned()), - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ]; let output = vec![ @@ -407,82 +187,26 @@ mod tests { "b".to_owned(), "c".to_owned(), ])), - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ReqFilter { - ids: None, - authors: None, kinds: Some(HashSet::from([1, 2])), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ReqFilter { - ids: None, authors: Some(HashSet::from(["c".to_owned()])), kinds: Some(HashSet::from([1])), - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ReqFilter { - ids: None, authors: Some(HashSet::from(["c".to_owned()])), - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, limit: Some(100), + ..Default::default() }, ReqFilter { ids: Some(HashSet::from(["1".to_owned()])), authors: Some(HashSet::from(["c".to_owned()])), - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - a_tag: None, - g_tag: None, - search: None, - since: None, - until: None, - limit: None, + ..Default::default() }, ]; diff --git a/packages/system/src/connection.ts b/packages/system/src/connection.ts index 19190e52..994b9765 100644 --- a/packages/system/src/connection.ts +++ b/packages/system/src/connection.ts @@ -397,7 +397,7 @@ export class Connection extends EventEmitter { this.queueReq(["REQ", id, ...newFilters], item.cb); } }; - if (this.Info?.software?.includes("strfry")) { + if (this.Address.startsWith("wss://relay.snort.social")) { const newFilters = filters.map(a => { if (a.ids_only) { const copy = { ...a }; diff --git a/packages/system/src/const.ts b/packages/system/src/const.ts index 795345bd..5b381fcd 100644 --- a/packages/system/src/const.ts +++ b/packages/system/src/const.ts @@ -49,3 +49,12 @@ export const MentionNostrEntityRegex = /@n(pub|profile|event|ote|addr|)1[acdefgh * Regex to match markdown code content */ export const MarkdownCodeRegex = /(```.+?```)/gms; + +/** + * Public metadata relays + */ +export const MetadataRelays = [ + "wss://purplepag.es/", + "wss://relay.nostr.band/", + "wss://relay.snort.social/" +] \ No newline at end of file diff --git a/packages/system/src/filter-cache-layer.ts b/packages/system/src/filter-cache-layer.ts index 32a8e96b..fa205493 100644 --- a/packages/system/src/filter-cache-layer.ts +++ b/packages/system/src/filter-cache-layer.ts @@ -1,5 +1,5 @@ -import { BuiltRawReqFilter, RequestStrategy } from "./request-builder"; -import { NostrEvent, TaggedNostrEvent } from "./nostr"; +import { BuiltRawReqFilter } from "./request-builder"; +import { NostrEvent } from "./nostr"; import { Query } from "./query"; export interface EventCache { @@ -9,31 +9,3 @@ export interface EventCache { export interface FilterCacheLayer { processFilter(q: Query, req: BuiltRawReqFilter): Promise; } - -export class IdsFilterCacheLayer implements FilterCacheLayer { - constructor(readonly cache: EventCache) {} - - async processFilter(q: Query, req: BuiltRawReqFilter) { - for (const f of req.filters) { - if (f.ids) { - const cacheResults = await this.cache.bulkGet(f.ids); - if (cacheResults.length > 0) { - const resultIds = new Set(cacheResults.map(a => a.id)); - f.ids = f.ids.filter(a => !resultIds.has(a)); - - // this step is important for buildDiff, if a filter doesnt exist with the ids which are from cache - // we will create an infinite loop where every render we insert a new query for the ids which are missing - q.insertCompletedTrace( - { - filters: [{ ...f, ids: [...resultIds] }], - strategy: RequestStrategy.ExplicitRelays, - relay: req.relay, - }, - cacheResults as Array, - ); - } - } - } - return req; - } -} diff --git a/packages/system/src/nostr-link.ts b/packages/system/src/nostr-link.ts index b34d3ea6..14265c73 100644 --- a/packages/system/src/nostr-link.ts +++ b/packages/system/src/nostr-link.ts @@ -177,14 +177,14 @@ export class NostrLink implements ToNostrEventTag { throw new Error(`Unknown tag kind ${tag.key}`); } - static fromTag(tag: Array) { + static fromTag(tag: Array, author?: string, kind?: number) { const relays = tag.length > 2 ? [tag[2]] : undefined; switch (tag[0]) { case "e": { - return new NostrLink(NostrPrefix.Event, tag[1], undefined, undefined, relays); + return new NostrLink(NostrPrefix.Event, tag[1], kind, author, relays); } case "p": { - return new NostrLink(NostrPrefix.Profile, tag[1], undefined, undefined, relays); + return new NostrLink(NostrPrefix.Profile, tag[1], kind, author, relays); } case "a": { const [kind, author, dTag] = tag[1].split(":"); diff --git a/packages/system/src/nostr.ts b/packages/system/src/nostr.ts index 568871ff..846398a5 100644 --- a/packages/system/src/nostr.ts +++ b/packages/system/src/nostr.ts @@ -58,6 +58,7 @@ export interface ReqFilter { until?: number; limit?: number; ids_only?: boolean; + relays?: string[]; [key: string]: Array | Array | string | number | undefined | boolean; } diff --git a/packages/system/src/outbox/outbox-model.ts b/packages/system/src/outbox/outbox-model.ts index 60d1df57..75535fba 100644 --- a/packages/system/src/outbox/outbox-model.ts +++ b/packages/system/src/outbox/outbox-model.ts @@ -1,10 +1,10 @@ import { EventKind, NostrEvent, ReqFilter, RequestBuilder, SystemInterface } from ".."; -import { dedupe, removeUndefined, unixNowMs, unwrap } from "@snort/shared"; +import { appendDedupe, dedupe, removeUndefined, unixNowMs, unwrap } from "@snort/shared"; import { FlatReqFilter } from "../query-optimizer"; import { RelayListCacheExpire } from "../const"; import { AuthorsRelaysCache, EventFetcher, PickedRelays, DefaultPickNRelays, parseRelaysFromKind } from "."; import debug from "debug"; -import { BaseRequestRouter, RelayTaggedFilter, RelayTaggedFlatFilters } from "../request-router"; +import { BaseRequestRouter } from "../request-router"; /** * Simple outbox model using most popular relays @@ -89,15 +89,10 @@ export class OutboxModel extends BaseRequestRouter { * @param pickN Number of relays to pick per author * @returns */ - forRequest(filter: ReqFilter, pickN?: number): Array { + forRequest(filter: ReqFilter, pickN?: number): Array { const authors = filter.authors; if ((authors?.length ?? 0) === 0) { - return [ - { - relay: "", - filter, - }, - ]; + return [filter]; } const topRelays = this.pickTopRelays(unwrap(authors), pickN ?? DefaultPickNRelays, "write"); @@ -106,22 +101,17 @@ export class OutboxModel extends BaseRequestRouter { const picked = pickedRelays.map(a => { const keysOnPickedRelay = dedupe(topRelays.filter(b => b.relays.includes(a)).map(b => b.key)); return { - relay: a, - filter: { - ...filter, - authors: keysOnPickedRelay, - }, - } as RelayTaggedFilter; + ...filter, + authors: keysOnPickedRelay, + relays: appendDedupe(filter.relays, [a]) + } as ReqFilter; }); const noRelays = dedupe(topRelays.filter(a => a.relays.length === 0).map(a => a.key)); if (noRelays.length > 0) { picked.push({ - relay: "", - filter: { - ...filter, - authors: noRelays, - }, - }); + ...filter, + authors: noRelays, + } as ReqFilter); } this.#log("Picked %O => %O", filter, picked); return picked; @@ -133,32 +123,32 @@ export class OutboxModel extends BaseRequestRouter { * @param pickN Number of relays to pick per author * @returns */ - forFlatRequest(input: Array, pickN?: number): Array { - const authors = input.filter(a => a.authors).map(a => unwrap(a.authors)); + forFlatRequest(input: Array, pickN?: number): Array { + const authors = removeUndefined(input.flatMap(a => a.authors)); if (authors.length === 0) { - return [ - { - relay: "", - filters: input, - }, - ]; + return input; } const topRelays = this.pickTopRelays(authors, pickN ?? DefaultPickNRelays, "write"); const pickedRelays = dedupe(topRelays.flatMap(a => a.relays)); - const picked = pickedRelays.map(a => { + const picked = pickedRelays.flatMap(a => { const authorsOnRelay = new Set(topRelays.filter(v => v.relays.includes(a)).map(v => v.key)); - return { - relay: a, - filters: input.filter(v => v.authors && authorsOnRelay.has(v.authors)), - } as RelayTaggedFlatFilters; + return input + .filter(v => v.authors && authorsOnRelay.has(v.authors)) + .flatMap(b => { + // if flat filter isnt already relay tagged, set relay tag or + // create a duplicate filter with the authors picked relay + if (!b.relay) { + b.relay = a; + return [b]; + } else { + return [b, { ...b, relay: a }]; + } + }); }); const noRelays = new Set(topRelays.filter(v => v.relays.length === 0).map(v => v.key)); if (noRelays.size > 0) { - picked.push({ - relay: "", - filters: input.filter(v => !v.authors || noRelays.has(v.authors)), - } as RelayTaggedFlatFilters); + picked.push(...input.filter(v => !v.authors || noRelays.has(v.authors))); } this.#log("Picked %d relays from %d filters", picked.length, input.length); diff --git a/packages/system/src/profile-cache.ts b/packages/system/src/profile-cache.ts index 07b8e983..3e4da501 100644 --- a/packages/system/src/profile-cache.ts +++ b/packages/system/src/profile-cache.ts @@ -19,7 +19,10 @@ export class ProfileLoaderService extends BackgroundLoader { override buildSub(missing: string[]): RequestBuilder { const sub = new RequestBuilder(`profiles`); - sub.withFilter().kinds([EventKind.SetMetadata]).authors(missing); + sub.withFilter() + .kinds([EventKind.SetMetadata]) + .authors(missing) + .relay(["wss://purplepag.es/"]); return sub; } diff --git a/packages/system/src/query-manager.ts b/packages/system/src/query-manager.ts index 74e80e53..6a1d29c5 100644 --- a/packages/system/src/query-manager.ts +++ b/packages/system/src/query-manager.ts @@ -1,8 +1,8 @@ import debug from "debug"; import { EventEmitter } from "eventemitter3"; -import { BuiltRawReqFilter, RequestBuilder, RequestStrategy, SystemInterface, TaggedNostrEvent } from "."; +import { BuiltRawReqFilter, RequestBuilder, SystemInterface, TaggedNostrEvent } from "."; import { Query, TraceReport } from "./query"; -import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer"; +import { FilterCacheLayer } from "./filter-cache-layer"; import { trimFilters } from "./request-trim"; interface QueryManagerEvents { @@ -35,7 +35,6 @@ export class QueryManager extends EventEmitter { constructor(system: SystemInterface) { super(); this.#system = system; - this.#queryCacheLayers.push(new IdsFilterCacheLayer(system.eventsCache)); setInterval(() => this.#cleanup(), 1_000); } diff --git a/packages/system/src/query-optimizer/index.ts b/packages/system/src/query-optimizer/index.ts index 821ee21a..ef1905ab 100644 --- a/packages/system/src/query-optimizer/index.ts +++ b/packages/system/src/query-optimizer/index.ts @@ -19,6 +19,7 @@ export interface FlatReqFilter { since?: number; until?: number; limit?: number; + relay?: string; resultSetId: string; } diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts index 37b1a548..ba1371f2 100644 --- a/packages/system/src/request-builder.ts +++ b/packages/system/src/request-builder.ts @@ -1,41 +1,18 @@ import debug from "debug"; import { v4 as uuid } from "uuid"; -import { appendDedupe, dedupe, sanitizeRelayUrl, unixNowMs, unwrap } from "@snort/shared"; +import { appendDedupe, dedupe, removeUndefined, sanitizeRelayUrl, unixNowMs, unwrap } from "@snort/shared"; import EventKind from "./event-kind"; -import { NostrLink, NostrPrefix, SystemInterface } from "."; +import { FlatReqFilter, NostrLink, NostrPrefix, SystemInterface } from "."; import { ReqFilter, u256, HexKey, TaggedNostrEvent } from "./nostr"; import { RequestRouter } from "./request-router"; -/** - * Which strategy is used when building REQ filters - */ -export const enum RequestStrategy { - /** - * Use the users default relays to fetch events, - * this is the fallback option when there is no better way to query a given filter set - */ - DefaultRelays = "default", - - /** - * Using a cached copy of the authors relay lists NIP-65, split a given set of request filters by pubkey - */ - AuthorsRelays = "authors-relays", - - /** - * Use pre-determined relays for query - */ - ExplicitRelays = "explicit-relays", -} - /** * A built REQ filter ready for sending to System */ export interface BuiltRawReqFilter { filters: Array; relay: string; - strategy: RequestStrategy; - // Use set sync from an existing set of events syncFrom?: Array; } @@ -133,8 +110,12 @@ export class RequestBuilder { } build(system: SystemInterface): Array { - const expanded = this.#builders.flatMap(a => a.build(system.requestRouter, this.#options)); - return this.#groupByRelay(system, expanded); + let rawFilters = this.buildRaw(); + if (system.requestRouter) { + rawFilters = system.requestRouter.forAllRequest(rawFilters); + } + const expanded = rawFilters.flatMap(a => system.optimizer.expandFilter(a)); + return this.#groupFlatByRelay(system, expanded); } /** @@ -143,55 +124,41 @@ export class RequestBuilder { async buildDiff(system: SystemInterface, prev: Array): Promise> { const start = unixNowMs(); - const diff = system.optimizer.getDiff(prev, this.buildRaw()); + let rawFilters = this.buildRaw(); + if (system.requestRouter) { + rawFilters = system.requestRouter.forAllRequest(rawFilters); + } + const diff = system.optimizer.getDiff(prev, rawFilters); const ts = unixNowMs() - start; this.#log("buildDiff %s %d ms +%d", this.id, ts, diff.length); if (diff.length > 0) { - if (system.requestRouter) { - // todo: fix for explicit relays - return system.requestRouter.forFlatRequest(diff).map(a => { - return { - strategy: RequestStrategy.AuthorsRelays, - filters: system.optimizer.flatMerge(a.filters), - relay: a.relay, - }; - }); - } else { - return [ - { - strategy: RequestStrategy.DefaultRelays, - filters: system.optimizer.flatMerge(diff), - relay: "", - }, - ]; - } + return this.#groupFlatByRelay(system, diff); } return []; } - /** - * Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests - */ - #groupByRelay(system: SystemInterface, filters: Array) { + #groupFlatByRelay(system: SystemInterface, filters: Array) { const relayMerged = filters.reduce((acc, v) => { - const existing = acc.get(v.relay); + const relay = v.relay ?? ""; + delete v.relay; + const existing = acc.get(relay); if (existing) { existing.push(v); } else { - acc.set(v.relay, [v]); + acc.set(relay, [v]); } return acc; - }, new Map>()); + }, new Map>()); - const filtersSquashed = [...relayMerged.values()].map(a => { - return { - filters: system.optimizer.flatMerge(a.flatMap(b => b.filters.flatMap(c => system.optimizer.expandFilter(c)))), - relay: a[0].relay, - strategy: a[0].strategy, - } as BuiltRawReqFilter; - }); - - return filtersSquashed; + const ret = []; + for (const [k, v] of relayMerged.entries()) { + const filters = system.optimizer.flatMerge(v); + ret.push({ + relay: k, + filters, + } as BuiltRawReqFilter); + } + return ret; } } @@ -207,7 +174,10 @@ export class RequestFilterBuilder { } get filter() { - return { ...this.#filter }; + return { + ...this.#filter, + relays: this.#relays.size > 0 ? [...this.#relays] : undefined + }; } /** @@ -232,6 +202,7 @@ export class RequestFilterBuilder { authors(authors?: Array) { if (!authors) return this; this.#filter.authors = appendDedupe(this.#filter.authors, authors); + this.#filter.authors = this.#filter.authors.filter(a => a.length === 64); return this; } @@ -281,6 +252,9 @@ export class RequestFilterBuilder { .authors([unwrap(link.author)]); } else { this.ids([link.id]); + if (link.author) { + this.authors([link.author]); + } } link.relays?.forEach(v => this.relay(v)); return this; @@ -298,46 +272,18 @@ export class RequestFilterBuilder { tags[0][0], tags.map(v => v[1]), ); + this.relay(removeUndefined(links.map(a => a.relays).flat())); return this; } /** * Build/expand this filter into a set of relay specific queries */ - build(model?: RequestRouter, options?: RequestBuilderOptions): Array { - return this.#buildFromFilter(this.#filter, model, options); - } - - #buildFromFilter(f: ReqFilter, model?: RequestRouter, options?: RequestBuilderOptions) { - // use the explicit relay list first - if (this.#relays.size > 0) { - return [...this.#relays].map(r => { - return { - filters: [f], - relay: r, - strategy: RequestStrategy.ExplicitRelays, - }; - }); + build(model?: RequestRouter, options?: RequestBuilderOptions): Array { + if (model) { + return model.forRequest(this.filter, options?.outboxPickN); } - // If any authors are set use the gossip model to fetch data for each author - if (f.authors && model) { - const split = model.forRequest(f, options?.outboxPickN); - return split.map(a => { - return { - filters: [a.filter], - relay: a.relay, - strategy: RequestStrategy.AuthorsRelays, - }; - }); - } - - return [ - { - filters: [f], - relay: "", - strategy: RequestStrategy.DefaultRelays, - }, - ]; + return [this.filter]; } } diff --git a/packages/system/src/request-router.ts b/packages/system/src/request-router.ts index 008b7f58..40bd0fd6 100644 --- a/packages/system/src/request-router.ts +++ b/packages/system/src/request-router.ts @@ -1,21 +1,7 @@ +import { unwrap } from "@snort/shared"; import { NostrEvent, ReqFilter } from "./nostr"; import { FlatReqFilter } from "./query-optimizer"; -export interface RelayTaggedFilter { - relay: string; - filter: ReqFilter; -} - -export interface RelayTaggedFlatFilters { - relay: string; - filters: Array; -} - -export interface RelayTaggedFilters { - relay: string; - filters: Array; -} - /** * Request router managed splitting of requests to one or more relays, and which relay to send events to. */ @@ -35,7 +21,7 @@ export interface RequestRouter { * @param pickN Number of relays to pick * @returns */ - forRequest(filter: ReqFilter, pickN?: number): Array; + forRequest(filter: ReqFilter, pickN?: number): Array; /** * Split a request filter to one or more relays. @@ -43,34 +29,37 @@ export interface RequestRouter { * @param pickN Number of relays to pick * @returns */ - forFlatRequest(filter: Array, pickN?: number): Array; + forFlatRequest(filter: Array, pickN?: number): Array; + + /** + * Same as forRequest, but merges the results + * @param filters + */ + forAllRequest(filters: Array): Array; } export abstract class BaseRequestRouter implements RequestRouter { abstract forReply(ev: NostrEvent, pickN?: number): Promise>; - abstract forRequest(filter: ReqFilter, pickN?: number): Array; - abstract forFlatRequest(filter: FlatReqFilter[], pickN?: number): Array; + abstract forRequest(filter: ReqFilter, pickN?: number): Array; + abstract forFlatRequest(filter: FlatReqFilter[], pickN?: number): Array; forAllRequest(filters: Array) { const allSplit = filters .map(a => this.forRequest(a)) .reduce((acc, v) => { for (const vn of v) { - const existing = acc.get(vn.relay); - if (existing) { - existing.push(vn.filter); - } else { - acc.set(vn.relay, [vn.filter]); + for (const r of (vn.relays?.length ?? 0) > 0 ? unwrap(vn.relays) : [""]) { + const existing = acc.get(r); + if (existing) { + existing.push(vn); + } else { + acc.set(r, [vn]); + } } } return acc; }, new Map>()); - return [...allSplit.entries()].map(([k, v]) => { - return { - relay: k, - filters: v, - } as RelayTaggedFilters; - }); + return [...allSplit.values()].flat() } }