Kieran
/
snort
5
14
Fork 14
Code Issues 52 Pull Requests 1 Packages Projects Releases 24 Wiki Activity

refactor: thread loading improvements
continuous-integration/drone/push Build is running Details

This commit is contained in:
kieran 2024-04-04 16:30:30 +01:00
parent ad2b6dbcf7
commit ee31726961
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
29 changed files with 272 additions and 972 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/app/src/Components/Embed/LinkPreview.tsx
View File

@ -56,7 +56,7 @@ const LinkPreview = ({ url }: { url: string }) => {
const urlTags = ["og:video:secure_url", "og:video:url", "og:video"];
const link = preview?.og_tags?.find(a => urlTags.includes(a[0].toLowerCase()))?.[1];
const videoType = preview?.og_tags?.find(a => a[0].toLowerCase() === "og:video:type")?.[1] ?? "video/mp4";
if (link) {
if (link && videoType.startsWith("video/")) {
return <MediaElement url={link} mime={videoType} />;
}
}

5
packages/app/src/Components/Event/Note/Note.tsx
View File

@ -149,11 +149,12 @@ function useGoToEvent(props, options) {
}
function Reaction({ ev }: { ev: TaggedNostrEvent }) {
const reactedToTag = ev.tags.find((tag: string[]) => tag[0] === "e");
const reactedToTag = ev.tags.findLast(tag => tag[0] === "e");
const pTag = ev.tags.findLast(tag => tag[0] === "p");
if (!reactedToTag?.length) {
return null;
}
const link = NostrLink.fromTag(reactedToTag);
const link = NostrLink.fromTag(reactedToTag, pTag?.[1]);
return (
<div className="note card">
<div className="text-gray-medium font-bold">

42
packages/app/src/Feed/ThreadFeed.ts
View File

@ -1,10 +1,14 @@
import { EventExt, EventKind, NostrLink, RequestBuilder } from "@snort/system";
import { useReactions, useRequestBuilder } from "@snort/system-react";
import { useEffect, useMemo, useState } from "react";
import { SnortContext, useRequestBuilder } from "@snort/system-react";
import { useContext, useEffect, useMemo, useState } from "react";
import { randomSample } from "@/Utils";
export default function useThreadFeed(link: NostrLink) {
const [root, setRoot] = useState<NostrLink>();
const [rootRelays, setRootRelays] = useState<Array<string>>();
const [allEvents, setAllEvents] = useState<Array<NostrLink>>([]);
const system = useContext(SnortContext);
const sub = useMemo(() => {
const sub = new RequestBuilder(`thread:${link.id.slice(0, 12)}`);
@ -13,7 +17,7 @@ export default function useThreadFeed(link: NostrLink) {
});
sub.withFilter().link(link);
if (root) {
sub.withFilter().link(root);
sub.withFilter().link(root).relay(rootRelays ?? []);
}
const grouped = [link, ...allEvents].reduce(
(acc, v) => {
@ -24,11 +28,14 @@ export default function useThreadFeed(link: NostrLink) {
{} as Record<string, Array<NostrLink>>,
);
for (const [, v] of Object.entries(grouped)) {
sub.withFilter().kinds([EventKind.TextNote]).replyToLink(v);
for (const v of Object.values(grouped)) {
sub.withFilter()
.kinds([EventKind.TextNote])
.replyToLink(v)
.relay(rootRelays ?? []);
}
return sub;
}, [allEvents.length]);
}, [allEvents.length, rootRelays]);
const store = useRequestBuilder(sub);
@ -57,15 +64,28 @@ export default function useThreadFeed(link: NostrLink) {
]),
);
}
} else {
setRoot(link);
}
}
}
}, [store?.length]);
const reactions = useReactions(`thread:${link.id.slice(0, 12)}:reactions`, [link, ...allEvents]);
useEffect(() => {
if (root) {
const rootEvent = store?.find(a => root.matchesEvent(a));
if (rootEvent) {
system.relayCache.buffer([rootEvent.pubkey]).then(() => {
const relays = system.relayCache.getFromCache(rootEvent.pubkey);
return {
thread: store ?? [],
reactions: reactions ?? [],
};
if (relays) {
const readRelays = randomSample(relays.relays.filter(a => a.settings.read).map(a => a.url), 3);
setRootRelays(readRelays);
}
})
}
}
}, [link, root, store?.length]);
return store ?? [];
}

5
packages/app/src/Utils/Thread/ThreadContext.tsx
View File

@ -2,13 +2,12 @@
import { TaggedNostrEvent } from "@snort/system";
import { createContext } from "react";
interface ThreadContext {
export interface ThreadContextState {
current: string;
root?: TaggedNostrEvent;
chains: Map<string, Array<TaggedNostrEvent>>;
data: Array<TaggedNostrEvent>;
reactions: Array<TaggedNostrEvent>;
setCurrent: (i: string) => void;
}
export const ThreadContext = createContext({} as ThreadContext);
export const ThreadContext = createContext({} as ThreadContextState);

19
packages/app/src/Utils/Thread/ThreadContextWrapper.tsx
View File

@ -6,7 +6,7 @@ import { useLocation } from "react-router-dom";
import useThreadFeed from "@/Feed/ThreadFeed";
import useModeration from "@/Hooks/useModeration";
import { chainKey, replyChainKey } from "@/Utils/Thread/ChainKey";
import { ThreadContext } from "@/Utils/Thread/ThreadContext";
import { ThreadContext, ThreadContextState } from "@/Utils/Thread/ThreadContext";
export function ThreadContextWrapper({ link, children }: { link: NostrLink; children?: ReactNode }) {
const location = useLocation();
@ -16,8 +16,8 @@ export function ThreadContextWrapper({ link, children }: { link: NostrLink; chil
const chains = useMemo(() => {
const chains = new Map<u256, Array<TaggedNostrEvent>>();
if (feed.thread) {
feed.thread
if (feed) {
feed
?.filter(a => !isBlocked(a.pubkey))
.forEach(v => {
const replyTo = replyChainKey(v);
@ -31,30 +31,29 @@ export function ThreadContextWrapper({ link, children }: { link: NostrLink; chil
});
}
return chains;
}, [feed.thread]);
}, [feed]);
// Root is the parent of the current note or the current note if its a root note or the root of the thread
const root = useMemo(() => {
const currentNote =
feed.thread?.find(a => chainKey(a) === currentId) ??
feed?.find(a => chainKey(a) === currentId) ??
(location.state && "sig" in location.state ? (location.state as TaggedNostrEvent) : undefined);
if (currentNote) {
const key = replyChainKey(currentNote);
if (key) {
return feed.thread?.find(a => chainKey(a) === key);
return feed?.find(a => chainKey(a) === key);
} else {
return currentNote;
}
}
}, [feed.thread.length, currentId, location]);
}, [feed.length, currentId, location]);
const ctxValue = useMemo<ThreadContext>(() => {
const ctxValue = useMemo<ThreadContextState>(() => {
return {
current: currentId,
root,
chains,
reactions: feed.reactions,
data: feed.thread,
data: feed,
setCurrent: v => setCurrentId(v),
};
}, [root, chains]);

46
packages/system-wasm/Cargo.lock generated
View File

@ -23,18 +23,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46"
[[package]]
name = "argon2"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9"
dependencies = [
"base64ct",
"blake2",
"cpufeatures",
"password-hash",
]
[[package]]
name = "async-trait"
version = "0.1.73"
@ -52,27 +40,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64ct"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bitflags"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "blake2"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
"digest",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -283,7 +256,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
@ -460,17 +432,6 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "password-hash"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
"base64ct",
"rand_core",
"subtle",
]
[[package]]
name = "plotters"
version = "0.3.5"
@ -727,12 +688,6 @@ dependencies = [
"sha2",
]
[[package]]
name = "subtle"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
[[package]]
name = "syn"
version = "2.0.31"
@ -748,7 +703,6 @@ dependencies = [
name = "system-wasm"
version = "0.1.0"
dependencies = [
"argon2",
"console_error_panic_hook",
"criterion",
"hex",

1
packages/system-wasm/Cargo.toml
View File

@ -8,7 +8,6 @@ edition = "2021"
crate-type = ["cdylib", "rlib"]
[dependencies]
argon2 = "0.5.2"
console_error_panic_hook = "0.1.7"
hex = { version = "0.4.3", features = [], default-features = false }
itertools = "0.11.0"

10
packages/system-wasm/README.md
View File

@ -1 +1,11 @@
# system-wasm
## Building
### Ubuntu/Debian
```bash
sudo apt install clang
cargo install wasm-pack
yarn build
```

2
packages/system-wasm/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@snort/system-wasm",
"version": "1.0.2",
"version": "1.0.3",
"packageManager": "yarn@3.6.3",
"author": "Kieran",
"license": "MIT",

10
packages/system-wasm/pkg/README.md
View File

@ -1 +1,11 @@
# system-wasm
## Building
### Ubuntu/Debian
```bash
sudo apt install clang
cargo install wasm-pack
yarn build
```

7
packages/system-wasm/pkg/system_wasm.d.ts vendored
View File

@ -33,12 +33,6 @@ export function compress(val: any): any;
* @returns {any}
*/
export function pow(val: any, target: any): any;
/**
* @param {any} password
* @param {any} salt
* @returns {any}
*/
export function argon2(password: any, salt: any): any;
/**
* @param {any} hash
* @param {any} sig
@ -62,7 +56,6 @@ export interface InitOutput {
readonly flat_merge: (a: number, b: number) => void;
readonly compress: (a: number, b: number) => void;
readonly pow: (a: number, b: number, c: number) => void;
readonly argon2: (a: number, b: number, c: number) => void;
readonly schnorr_verify: (a: number, b: number, c: number, d: number) => void;
readonly schnorr_verify_event: (a: number, b: number) => void;
readonly rustsecp256k1_v0_9_1_context_create: (a: number) => number;

29
packages/system-wasm/pkg/system_wasm.js
View File

@ -340,27 +340,6 @@ export function pow(val, target) {
}
}
/**
* @param {any} password
* @param {any} salt
* @returns {any}
*/
export function argon2(password, salt) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.argon2(retptr, addHeapObject(password), addHeapObject(salt));
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
var r2 = getInt32Memory0()[retptr / 4 + 2];
if (r2) {
throw takeObject(r1);
}
return takeObject(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {any} hash
* @param {any} sig
@ -484,10 +463,6 @@ function __wbg_get_imports() {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
imports.wbg.__wbindgen_object_clone_ref = function (arg0) {
const ret = getObject(arg0);
return addHeapObject(ret);
};
imports.wbg.__wbindgen_jsval_loose_eq = function (arg0, arg1) {
const ret = getObject(arg0) == getObject(arg1);
return ret;
@ -507,6 +482,10 @@ function __wbg_get_imports() {
const ret = arg0;
return addHeapObject(ret);
};
imports.wbg.__wbindgen_object_clone_ref = function (arg0) {
const ret = getObject(arg0);
return addHeapObject(ret);
};
imports.wbg.__wbindgen_string_new = function (arg0, arg1) {
const ret = getStringFromWasm0(arg0, arg1);
return addHeapObject(ret);

BIN
packages/system-wasm/pkg/system_wasm_bg.wasm
View File

Binary file not shown.

1
packages/system-wasm/pkg/system_wasm_bg.wasm.d.ts vendored
View File

@ -7,7 +7,6 @@ export function get_diff(a: number, b: number, c: number): void;
export function flat_merge(a: number, b: number): void;
export function compress(a: number, b: number): void;
export function pow(a: number, b: number, c: number): void;
export function argon2(a: number, b: number, c: number): void;
export function schnorr_verify(a: number, b: number, c: number, d: number): void;
export function schnorr_verify_event(a: number, b: number): void;
export function rustsecp256k1_v0_9_1_context_create(a: number): number;

126
packages/system-wasm/src/diff.rs
View File

@ -21,35 +21,11 @@ mod tests {
fn simple_diff_same() {
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}];
let next = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}];
let result = diff_filter(&prev, &next);
@ -60,52 +36,16 @@ mod tests {
fn simple_diff_add() {
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}];
let next = vec![
FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
];
@ -114,19 +54,7 @@ mod tests {
result,
vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}]
)
}
@ -135,35 +63,11 @@ mod tests {
fn simple_diff_replace() {
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}];
let next = vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}];
let result = diff_filter(&prev, &next);
@ -171,19 +75,7 @@ mod tests {
result,
vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
}]
)
}

239
packages/system-wasm/src/filter.rs
View File

@ -1,9 +1,10 @@
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
#[cfg(test)]
use std::fmt::Debug;
use std::hash::Hash;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
#[derive(Clone)]
enum StringOrNumberEntry<'a> {
@ -11,7 +12,7 @@ enum StringOrNumberEntry<'a> {
Number((&'static str, &'a i32)),
}
#[derive(PartialEq, Clone, Serialize, Deserialize)]
#[derive(PartialEq, Clone, Serialize, Deserialize, Default)]
pub struct ReqFilter {
#[serde(rename = "ids", skip_serializing_if = "Option::is_none")]
pub ids: Option<HashSet<String>>,
@ -33,6 +34,8 @@ pub struct ReqFilter {
pub a_tag: Option<HashSet<String>>,
#[serde(rename = "#g", skip_serializing_if = "Option::is_none")]
pub g_tag: Option<HashSet<String>>,
#[serde(rename = "relays", skip_serializing_if = "Option::is_none")]
pub relays: Option<HashSet<String>>,
#[serde(rename = "search", skip_serializing_if = "Option::is_none")]
pub search: Option<String>,
#[serde(rename = "since", skip_serializing_if = "Option::is_none")]
@ -50,7 +53,7 @@ impl Debug for ReqFilter {
}
}
#[derive(PartialEq, PartialOrd, Clone, Serialize, Deserialize)]
#[derive(PartialEq, PartialOrd, Clone, Serialize, Deserialize, Default)]
pub struct FlatReqFilter {
#[serde(rename = "ids", skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
@ -74,6 +77,8 @@ pub struct FlatReqFilter {
pub g_tag: Option<String>,
#[serde(rename = "search", skip_serializing_if = "Option::is_none")]
pub search: Option<String>,
#[serde(rename = "relay", skip_serializing_if = "Option::is_none")]
pub relay: Option<String>,
#[serde(rename = "since", skip_serializing_if = "Option::is_none")]
pub since: Option<i32>,
#[serde(rename = "until", skip_serializing_if = "Option::is_none")]
@ -117,11 +122,13 @@ impl Distance for FlatReqFilter {
ret += prop_dist(&self.id, &b.id);
ret += prop_dist(&self.kind, &b.kind);
ret += prop_dist(&self.author, &b.author);
ret += prop_dist(&self.relay, &b.relay);
ret += prop_dist(&self.e_tag, &b.e_tag);
ret += prop_dist(&self.p_tag, &b.p_tag);
ret += prop_dist(&self.d_tag, &b.d_tag);
ret += prop_dist(&self.r_tag, &b.r_tag);
ret += prop_dist(&self.t_tag, &b.t_tag);
ret += prop_dist(&self.g_tag, &b.g_tag);
ret
}
@ -143,26 +150,12 @@ impl CanMerge for FlatReqFilter {
impl From<Vec<&FlatReqFilter>> for ReqFilter {
fn from(value: Vec<&FlatReqFilter>) -> Self {
let ret = ReqFilter {
ids: None,
authors: None,
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let ret = Default::default();
value.iter().fold(ret, |mut acc, x| {
array_prop_append(&x.id, &mut acc.ids);
array_prop_append(&x.author, &mut acc.authors);
array_prop_append(&x.kind, &mut acc.kinds);
array_prop_append(&x.relay, &mut acc.relays);
array_prop_append(&x.e_tag, &mut acc.e_tag);
array_prop_append(&x.p_tag, &mut acc.p_tag);
array_prop_append(&x.t_tag, &mut acc.t_tag);
@ -182,26 +175,12 @@ impl From<Vec<&FlatReqFilter>> for ReqFilter {
impl From<Vec<&ReqFilter>> for ReqFilter {
fn from(value: Vec<&ReqFilter>) -> Self {
let ret = ReqFilter {
ids: None,
authors: None,
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let ret = Default::default();
value.iter().fold(ret, |mut acc, x| {
array_prop_append_vec(&x.ids, &mut acc.ids);
array_prop_append_vec(&x.authors, &mut acc.authors);
array_prop_append_vec(&x.kinds, &mut acc.kinds);
array_prop_append_vec(&x.relays, &mut acc.relays);
array_prop_append_vec(&x.e_tag, &mut acc.e_tag);
array_prop_append_vec(&x.p_tag, &mut acc.p_tag);
array_prop_append_vec(&x.t_tag, &mut acc.t_tag);
@ -245,6 +224,13 @@ impl Into<Vec<FlatReqFilter>> for &ReqFilter {
.collect();
inputs.push(t_ids);
}
if let Some(relays) = &self.relays {
let t_relays = relays
.iter()
.map(|z| StringOrNumberEntry::String(("relay", z)))
.collect();
inputs.push(t_relays);
}
if let Some(e_tags) = &self.e_tag {
let t_ids = e_tags
.iter()
@ -313,6 +299,14 @@ impl Into<Vec<FlatReqFilter>> for &ReqFilter {
}
None
}),
relay: p.iter().find_map(|q| {
if let StringOrNumberEntry::String((k, v)) = q {
if (*k).eq("relay") {
return Some((*v).to_string());
}
}
None
}),
kind: p.iter().find_map(|q| {
if let StringOrNumberEntry::Number((k, v)) = q {
if (*k).eq("kind") {
@ -394,6 +388,7 @@ impl Distance for ReqFilter {
ret += prop_dist_vec(&self.ids, &b.ids);
ret += prop_dist_vec(&self.kinds, &b.kinds);
ret += prop_dist_vec(&self.authors, &b.authors);
ret += prop_dist_vec(&self.relays, &b.relays);
ret += prop_dist_vec(&self.e_tag, &b.e_tag);
ret += prop_dist_vec(&self.p_tag, &b.p_tag);
ret += prop_dist_vec(&self.d_tag, &b.d_tag);
@ -478,9 +473,10 @@ fn array_prop_append_vec<T: Clone + Eq + Hash>(
#[cfg(test)]
mod tests {
use crate::ReqFilter;
use std::collections::HashSet;
use crate::filter::FlatReqFilter;
use crate::ReqFilter;
#[test]
fn test_expand_filter() {
@ -493,16 +489,9 @@ mod tests {
kinds: Some(HashSet::from([1, 2, 3])),
ids: Some(HashSet::from(["x".to_owned(), "y".to_owned()])),
p_tag: Some(HashSet::from(["a".to_owned()])),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
};
let output: Vec<FlatReqFilter> = (&input).into();
@ -512,288 +501,162 @@ mod tests {
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
..Default::default()
},
];
assert_eq!(output.len(), expected.len());

60
packages/system-wasm/src/lib.rs
View File

@ -1,6 +1,5 @@
extern crate console_error_panic_hook;
use argon2::{Argon2};
use secp256k1::{Message, XOnlyPublicKey, SECP256K1};
use serde::{Deserialize, Serialize};
use serde_json::json;
@ -90,16 +89,6 @@ pub fn pow(val: JsValue, target: JsValue) -> Result<JsValue, JsValue> {
Ok(serde_wasm_bindgen::to_value(&val_parsed)?)
}
#[wasm_bindgen]
pub fn argon2(password: JsValue, salt: JsValue) -> Result<JsValue, JsValue> {
console_error_panic_hook::set_once();
let password_parsed: String = serde_wasm_bindgen::from_value(password)?;
let salt_parsed: String = serde_wasm_bindgen::from_value(salt)?;
let mut key = [0u8; 32];
Argon2::default().hash_password_into(password_parsed.as_bytes(), salt_parsed.as_bytes(), &mut key).expect("Failed to generate key");
Ok(serde_wasm_bindgen::to_value(&hex::encode(key))?)
}
#[wasm_bindgen]
pub fn schnorr_verify(hash: JsValue, sig: JsValue, pub_key: JsValue) -> Result<bool, JsValue> {
console_error_panic_hook::set_once();
@ -137,15 +126,7 @@ mod tests {
fn flat_merge_expanded() {
let input = vec![
ReqFilter {
ids: None,
kinds: Some(HashSet::from([1, 6969, 6])),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
authors: Some(HashSet::from([
"kieran".to_string(),
"snort".to_string(),
@ -155,56 +136,23 @@ mod tests {
])),
since: Some(1),
until: Some(100),
search: None,
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
kinds: Some(HashSet::from([4])),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
authors: Some(HashSet::from(["kieran".to_string()])),
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
authors: None,
kinds: Some(HashSet::from([4])),
e_tag: None,
p_tag: Some(HashSet::from(["kieran".to_string()])),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
kinds: Some(HashSet::from([1000])),
authors: Some(HashSet::from(["snort".to_string()])),
p_tag: Some(HashSet::from(["kieran".to_string()])),
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
e_tag: None,
limit: None,
..Default::default()
},
];

326
packages/system-wasm/src/merge.rs
View File

@ -59,83 +59,25 @@ mod tests {
fn distance() {
let a = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
};
let b = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
};
let c = FlatReqFilter {
id: Some("c".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
};
let d = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
};
let e = FlatReqFilter {
id: Some("e".to_owned()),
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
};
assert_eq!(a.distance(&b), 0);
assert_eq!(a.distance(&c), 1);
@ -148,51 +90,21 @@ mod tests {
let a = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
..Default::default()
};
let b = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
..Default::default()
};
let output = ReqFilter {
ids: Some(HashSet::from(["0".to_owned()])),
authors: Some(HashSet::from(["a".to_owned(), "b".to_owned()])),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
..Default::default()
};
assert_eq!(ReqFilter::from(vec![&a, &b]), output);
}
@ -202,50 +114,20 @@ mod tests {
let a = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
..Default::default()
};
let b = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
..Default::default()
};
let c = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
..Default::default()
};
assert!(&a.can_merge(&b));
assert!(!&b.can_merge(&c));
@ -257,146 +139,44 @@ mod tests {
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: None,
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: None,
author: None,
kind: Some(2),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: None,
author: None,
kind: Some(2),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: None,
author: Some("c".to_owned()),
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
FlatReqFilter {
id: None,
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
..Default::default()
},
FlatReqFilter {
id: Some("1".to_owned()),
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
];
let output = vec![
@ -407,82 +187,26 @@ mod tests {
"b".to_owned(),
"c".to_owned(),
])),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
authors: None,
kinds: Some(HashSet::from([1, 2])),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
authors: Some(HashSet::from(["c".to_owned()])),
kinds: Some(HashSet::from([1])),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
ReqFilter {
ids: None,
authors: Some(HashSet::from(["c".to_owned()])),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
..Default::default()
},
ReqFilter {
ids: Some(HashSet::from(["1".to_owned()])),
authors: Some(HashSet::from(["c".to_owned()])),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
a_tag: None,
g_tag: None,
search: None,
since: None,
until: None,
limit: None,
..Default::default()
},
];

2
packages/system/src/connection.ts
View File

@ -397,7 +397,7 @@ export class Connection extends EventEmitter<ConnectionEvents> {
this.queueReq(["REQ", id, ...newFilters], item.cb);
}
};
if (this.Info?.software?.includes("strfry")) {
if (this.Address.startsWith("wss://relay.snort.social")) {
const newFilters = filters.map(a => {
if (a.ids_only) {
const copy = { ...a };

9
packages/system/src/const.ts
View File

@ -49,3 +49,12 @@ export const MentionNostrEntityRegex = /@n(pub|profile|event|ote|addr|)1[acdefgh
* Regex to match markdown code content
*/
export const MarkdownCodeRegex = /(```.+?```)/gms;
/**
* Public metadata relays
*/
export const MetadataRelays = [
"wss://purplepag.es/",
"wss://relay.nostr.band/",
"wss://relay.snort.social/"
]

32
packages/system/src/filter-cache-layer.ts
View File

@ -1,5 +1,5 @@
import { BuiltRawReqFilter, RequestStrategy } from "./request-builder";
import { NostrEvent, TaggedNostrEvent } from "./nostr";
import { BuiltRawReqFilter } from "./request-builder";
import { NostrEvent } from "./nostr";
import { Query } from "./query";
export interface EventCache {
@ -9,31 +9,3 @@ export interface EventCache {
export interface FilterCacheLayer {
processFilter(q: Query, req: BuiltRawReqFilter): Promise<BuiltRawReqFilter>;
}
export class IdsFilterCacheLayer implements FilterCacheLayer {
constructor(readonly cache: EventCache) {}
async processFilter(q: Query, req: BuiltRawReqFilter) {
for (const f of req.filters) {
if (f.ids) {
const cacheResults = await this.cache.bulkGet(f.ids);
if (cacheResults.length > 0) {
const resultIds = new Set(cacheResults.map(a => a.id));
f.ids = f.ids.filter(a => !resultIds.has(a));
// this step is important for buildDiff, if a filter doesnt exist with the ids which are from cache
// we will create an infinite loop where every render we insert a new query for the ids which are missing
q.insertCompletedTrace(
{
filters: [{ ...f, ids: [...resultIds] }],
strategy: RequestStrategy.ExplicitRelays,
relay: req.relay,
},
cacheResults as Array<TaggedNostrEvent>,
);
}
}
}
return req;
}
}

6
packages/system/src/nostr-link.ts
View File

@ -177,14 +177,14 @@ export class NostrLink implements ToNostrEventTag {
throw new Error(`Unknown tag kind ${tag.key}`);
}
static fromTag(tag: Array<string>) {
static fromTag(tag: Array<string>, author?: string, kind?: number) {
const relays = tag.length > 2 ? [tag[2]] : undefined;
switch (tag[0]) {
case "e": {
return new NostrLink(NostrPrefix.Event, tag[1], undefined, undefined, relays);
return new NostrLink(NostrPrefix.Event, tag[1], kind, author, relays);
}
case "p": {
return new NostrLink(NostrPrefix.Profile, tag[1], undefined, undefined, relays);
return new NostrLink(NostrPrefix.Profile, tag[1], kind, author, relays);
}
case "a": {
const [kind, author, dTag] = tag[1].split(":");

1
packages/system/src/nostr.ts
View File

@ -58,6 +58,7 @@ export interface ReqFilter {
until?: number;
limit?: number;
ids_only?: boolean;
relays?: string[];
[key: string]: Array<string> | Array<number> | string | number | undefined | boolean;
}

66
packages/system/src/outbox/outbox-model.ts
View File

@ -1,10 +1,10 @@
import { EventKind, NostrEvent, ReqFilter, RequestBuilder, SystemInterface } from "..";
import { dedupe, removeUndefined, unixNowMs, unwrap } from "@snort/shared";
import { appendDedupe, dedupe, removeUndefined, unixNowMs, unwrap } from "@snort/shared";
import { FlatReqFilter } from "../query-optimizer";
import { RelayListCacheExpire } from "../const";
import { AuthorsRelaysCache, EventFetcher, PickedRelays, DefaultPickNRelays, parseRelaysFromKind } from ".";
import debug from "debug";
import { BaseRequestRouter, RelayTaggedFilter, RelayTaggedFlatFilters } from "../request-router";
import { BaseRequestRouter } from "../request-router";
/**
* Simple outbox model using most popular relays
@ -89,15 +89,10 @@ export class OutboxModel extends BaseRequestRouter {
* @param pickN Number of relays to pick per author
* @returns
*/
forRequest(filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter> {
forRequest(filter: ReqFilter, pickN?: number): Array<ReqFilter> {
const authors = filter.authors;
if ((authors?.length ?? 0) === 0) {
return [
{
relay: "",
filter,
},
];
return [filter];
}
const topRelays = this.pickTopRelays(unwrap(authors), pickN ?? DefaultPickNRelays, "write");
@ -106,22 +101,17 @@ export class OutboxModel extends BaseRequestRouter {
const picked = pickedRelays.map(a => {
const keysOnPickedRelay = dedupe(topRelays.filter(b => b.relays.includes(a)).map(b => b.key));
return {
relay: a,
filter: {
...filter,
authors: keysOnPickedRelay,
},
} as RelayTaggedFilter;
...filter,
authors: keysOnPickedRelay,
relays: appendDedupe(filter.relays, [a])
} as ReqFilter;
});
const noRelays = dedupe(topRelays.filter(a => a.relays.length === 0).map(a => a.key));
if (noRelays.length > 0) {
picked.push({
relay: "",
filter: {
...filter,
authors: noRelays,
},
});
...filter,
authors: noRelays,
} as ReqFilter);
}
this.#log("Picked %O => %O", filter, picked);
return picked;
@ -133,32 +123,32 @@ export class OutboxModel extends BaseRequestRouter {
* @param pickN Number of relays to pick per author
* @returns
*/
forFlatRequest(input: Array<FlatReqFilter>, pickN?: number): Array<RelayTaggedFlatFilters> {
const authors = input.filter(a => a.authors).map(a => unwrap(a.authors));
forFlatRequest(input: Array<FlatReqFilter>, pickN?: number): Array<FlatReqFilter> {
const authors = removeUndefined(input.flatMap(a => a.authors));
if (authors.length === 0) {
return [
{
relay: "",
filters: input,
},
];
return input;
}
const topRelays = this.pickTopRelays(authors, pickN ?? DefaultPickNRelays, "write");
const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
const picked = pickedRelays.map(a => {
const picked = pickedRelays.flatMap(a => {
const authorsOnRelay = new Set(topRelays.filter(v => v.relays.includes(a)).map(v => v.key));
return {
relay: a,
filters: input.filter(v => v.authors && authorsOnRelay.has(v.authors)),
} as RelayTaggedFlatFilters;
return input
.filter(v => v.authors && authorsOnRelay.has(v.authors))
.flatMap(b => {
// if flat filter isnt already relay tagged, set relay tag or
// create a duplicate filter with the authors picked relay
if (!b.relay) {
b.relay = a;
return [b];
} else {
return [b, { ...b, relay: a }];
}
});
});
const noRelays = new Set(topRelays.filter(v => v.relays.length === 0).map(v => v.key));
if (noRelays.size > 0) {
picked.push({
relay: "",
filters: input.filter(v => !v.authors || noRelays.has(v.authors)),
} as RelayTaggedFlatFilters);
picked.push(...input.filter(v => !v.authors || noRelays.has(v.authors)));
}
this.#log("Picked %d relays from %d filters", picked.length, input.length);

5
packages/system/src/profile-cache.ts
View File

@ -19,7 +19,10 @@ export class ProfileLoaderService extends BackgroundLoader<CachedMetadata> {
override buildSub(missing: string[]): RequestBuilder {
const sub = new RequestBuilder(`profiles`);
sub.withFilter().kinds([EventKind.SetMetadata]).authors(missing);
sub.withFilter()
.kinds([EventKind.SetMetadata])
.authors(missing)
.relay(["wss://purplepag.es/"]);
return sub;
}

5
packages/system/src/query-manager.ts
View File

@ -1,8 +1,8 @@
import debug from "debug";
import { EventEmitter } from "eventemitter3";
import { BuiltRawReqFilter, RequestBuilder, RequestStrategy, SystemInterface, TaggedNostrEvent } from ".";
import { BuiltRawReqFilter, RequestBuilder, SystemInterface, TaggedNostrEvent } from ".";
import { Query, TraceReport } from "./query";
import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer";
import { FilterCacheLayer } from "./filter-cache-layer";
import { trimFilters } from "./request-trim";
interface QueryManagerEvents {
@ -35,7 +35,6 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
constructor(system: SystemInterface) {
super();
this.#system = system;
this.#queryCacheLayers.push(new IdsFilterCacheLayer(system.eventsCache));
setInterval(() => this.#cleanup(), 1_000);
}

1
packages/system/src/query-optimizer/index.ts
View File

@ -19,6 +19,7 @@ export interface FlatReqFilter {
since?: number;
until?: number;
limit?: number;
relay?: string;
resultSetId: string;
}

138
packages/system/src/request-builder.ts
View File

@ -1,41 +1,18 @@
import debug from "debug";
import { v4 as uuid } from "uuid";
import { appendDedupe, dedupe, sanitizeRelayUrl, unixNowMs, unwrap } from "@snort/shared";
import { appendDedupe, dedupe, removeUndefined, sanitizeRelayUrl, unixNowMs, unwrap } from "@snort/shared";
import EventKind from "./event-kind";
import { NostrLink, NostrPrefix, SystemInterface } from ".";
import { FlatReqFilter, NostrLink, NostrPrefix, SystemInterface } from ".";
import { ReqFilter, u256, HexKey, TaggedNostrEvent } from "./nostr";
import { RequestRouter } from "./request-router";
/**
* Which strategy is used when building REQ filters
*/
export const enum RequestStrategy {
/**
* Use the users default relays to fetch events,
* this is the fallback option when there is no better way to query a given filter set
*/
DefaultRelays = "default",
/**
* Using a cached copy of the authors relay lists NIP-65, split a given set of request filters by pubkey
*/
AuthorsRelays = "authors-relays",
/**
* Use pre-determined relays for query
*/
ExplicitRelays = "explicit-relays",
}
/**
* A built REQ filter ready for sending to System
*/
export interface BuiltRawReqFilter {
filters: Array<ReqFilter>;
relay: string;
strategy: RequestStrategy;
// Use set sync from an existing set of events
syncFrom?: Array<TaggedNostrEvent>;
}
@ -133,8 +110,12 @@ export class RequestBuilder {
}
build(system: SystemInterface): Array<BuiltRawReqFilter> {
const expanded = this.#builders.flatMap(a => a.build(system.requestRouter, this.#options));
return this.#groupByRelay(system, expanded);
let rawFilters = this.buildRaw();
if (system.requestRouter) {
rawFilters = system.requestRouter.forAllRequest(rawFilters);
}
const expanded = rawFilters.flatMap(a => system.optimizer.expandFilter(a));
return this.#groupFlatByRelay(system, expanded);
}
/**
@ -143,55 +124,41 @@ export class RequestBuilder {
async buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Promise<Array<BuiltRawReqFilter>> {
const start = unixNowMs();
const diff = system.optimizer.getDiff(prev, this.buildRaw());
let rawFilters = this.buildRaw();
if (system.requestRouter) {
rawFilters = system.requestRouter.forAllRequest(rawFilters);
}
const diff = system.optimizer.getDiff(prev, rawFilters);
const ts = unixNowMs() - start;
this.#log("buildDiff %s %d ms +%d", this.id, ts, diff.length);
if (diff.length > 0) {
if (system.requestRouter) {
// todo: fix for explicit relays
return system.requestRouter.forFlatRequest(diff).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
filters: system.optimizer.flatMerge(a.filters),
relay: a.relay,
};
});
} else {
return [
{
strategy: RequestStrategy.DefaultRelays,
filters: system.optimizer.flatMerge(diff),
relay: "",
},
];
}
return this.#groupFlatByRelay(system, diff);
}
return [];
}
/**
* Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests
*/
#groupByRelay(system: SystemInterface, filters: Array<BuiltRawReqFilter>) {
#groupFlatByRelay(system: SystemInterface, filters: Array<FlatReqFilter>) {
const relayMerged = filters.reduce((acc, v) => {
const existing = acc.get(v.relay);
const relay = v.relay ?? "";
delete v.relay;
const existing = acc.get(relay);
if (existing) {
existing.push(v);
} else {
acc.set(v.relay, [v]);
acc.set(relay, [v]);
}
return acc;
}, new Map<string, Array<BuiltRawReqFilter>>());
}, new Map<string, Array<FlatReqFilter>>());
const filtersSquashed = [...relayMerged.values()].map(a => {
return {
filters: system.optimizer.flatMerge(a.flatMap(b => b.filters.flatMap(c => system.optimizer.expandFilter(c)))),
relay: a[0].relay,
strategy: a[0].strategy,
} as BuiltRawReqFilter;
});
return filtersSquashed;
const ret = [];
for (const [k, v] of relayMerged.entries()) {
const filters = system.optimizer.flatMerge(v);
ret.push({
relay: k,
filters,
} as BuiltRawReqFilter);
}
return ret;
}
}
@ -207,7 +174,10 @@ export class RequestFilterBuilder {
}
get filter() {
return { ...this.#filter };
return {
...this.#filter,
relays: this.#relays.size > 0 ? [...this.#relays] : undefined
};
}
/**
@ -232,6 +202,7 @@ export class RequestFilterBuilder {
authors(authors?: Array<HexKey>) {
if (!authors) return this;
this.#filter.authors = appendDedupe(this.#filter.authors, authors);
this.#filter.authors = this.#filter.authors.filter(a => a.length === 64);
return this;
}
@ -281,6 +252,9 @@ export class RequestFilterBuilder {
.authors([unwrap(link.author)]);
} else {
this.ids([link.id]);
if (link.author) {
this.authors([link.author]);
}
}
link.relays?.forEach(v => this.relay(v));
return this;
@ -298,46 +272,18 @@ export class RequestFilterBuilder {
tags[0][0],
tags.map(v => v[1]),
);
this.relay(removeUndefined(links.map(a => a.relays).flat()));
return this;
}
/**
* Build/expand this filter into a set of relay specific queries
*/
build(model?: RequestRouter, options?: RequestBuilderOptions): Array<BuiltRawReqFilter> {
return this.#buildFromFilter(this.#filter, model, options);
}
#buildFromFilter(f: ReqFilter, model?: RequestRouter, options?: RequestBuilderOptions) {
// use the explicit relay list first
if (this.#relays.size > 0) {
return [...this.#relays].map(r => {
return {
filters: [f],
relay: r,
strategy: RequestStrategy.ExplicitRelays,
};
});
build(model?: RequestRouter, options?: RequestBuilderOptions): Array<ReqFilter> {
if (model) {
return model.forRequest(this.filter, options?.outboxPickN);
}
// If any authors are set use the gossip model to fetch data for each author
if (f.authors && model) {
const split = model.forRequest(f, options?.outboxPickN);
return split.map(a => {
return {
filters: [a.filter],
relay: a.relay,
strategy: RequestStrategy.AuthorsRelays,
};
});
}
return [
{
filters: [f],
relay: "",
strategy: RequestStrategy.DefaultRelays,
},
];
return [this.filter];
}
}

49
packages/system/src/request-router.ts
View File

@ -1,21 +1,7 @@
import { unwrap } from "@snort/shared";
import { NostrEvent, ReqFilter } from "./nostr";
import { FlatReqFilter } from "./query-optimizer";
export interface RelayTaggedFilter {
relay: string;
filter: ReqFilter;
}
export interface RelayTaggedFlatFilters {
relay: string;
filters: Array<FlatReqFilter>;
}
export interface RelayTaggedFilters {
relay: string;
filters: Array<ReqFilter>;
}
/**
* Request router managed splitting of requests to one or more relays, and which relay to send events to.
*/
@ -35,7 +21,7 @@ export interface RequestRouter {
* @param pickN Number of relays to pick
* @returns
*/
forRequest(filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter>;
forRequest(filter: ReqFilter, pickN?: number): Array<ReqFilter>;
/**
* Split a request filter to one or more relays.
@ -43,34 +29,37 @@ export interface RequestRouter {
* @param pickN Number of relays to pick
* @returns
*/
forFlatRequest(filter: Array<FlatReqFilter>, pickN?: number): Array<RelayTaggedFlatFilters>;
forFlatRequest(filter: Array<FlatReqFilter>, pickN?: number): Array<FlatReqFilter>;
/**
* Same as forRequest, but merges the results
* @param filters
*/
forAllRequest(filters: Array<ReqFilter>): Array<ReqFilter>;
}
export abstract class BaseRequestRouter implements RequestRouter {
abstract forReply(ev: NostrEvent, pickN?: number): Promise<Array<string>>;
abstract forRequest(filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter>;
abstract forFlatRequest(filter: FlatReqFilter[], pickN?: number): Array<RelayTaggedFlatFilters>;
abstract forRequest(filter: ReqFilter, pickN?: number): Array<ReqFilter>;
abstract forFlatRequest(filter: FlatReqFilter[], pickN?: number): Array<FlatReqFilter>;
forAllRequest(filters: Array<ReqFilter>) {
const allSplit = filters
.map(a => this.forRequest(a))
.reduce((acc, v) => {
for (const vn of v) {
const existing = acc.get(vn.relay);
if (existing) {
existing.push(vn.filter);
} else {
acc.set(vn.relay, [vn.filter]);
for (const r of (vn.relays?.length ?? 0) > 0 ? unwrap(vn.relays) : [""]) {
const existing = acc.get(r);
if (existing) {
existing.push(vn);
} else {
acc.set(r, [vn]);
}
}
}
return acc;
}, new Map<string, Array<ReqFilter>>());
return [...allSplit.entries()].map(([k, v]) => {
return {
relay: k,
filters: v,
} as RelayTaggedFilters;
});
return [...allSplit.values()].flat()
}
}