diff --git a/packages/system-query/.gitignore b/packages/system-query/.gitignore index e673575a..d6c6ec7e 100644 --- a/packages/system-query/.gitignore +++ b/packages/system-query/.gitignore @@ -1,2 +1,3 @@ .idea/ -target/ \ No newline at end of file +target/ +*.txt \ No newline at end of file diff --git a/packages/system-query/Cargo.lock b/packages/system-query/Cargo.lock index 036190d1..c1a9a84a 100644 --- a/packages/system-query/Cargo.lock +++ b/packages/system-query/Cargo.lock @@ -30,6 +30,17 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "itertools" version = "0.11.0" @@ -39,6 +50,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "js-sys" version = "0.3.64" @@ -48,6 +65,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + [[package]] name = "log" version = "0.4.20" @@ -60,6 +83,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.66" @@ -78,6 +107,42 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + [[package]] name = "scoped-tls" version = "1.0.1" @@ -115,6 +180,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "syn" version = "2.0.31" @@ -131,8 +207,10 @@ name = "system-query" version = "0.1.0" dependencies = [ "itertools", + "rand", "serde", "serde-wasm-bindgen", + "serde_json", "wasm-bindgen", "wasm-bindgen-test", ] @@ -143,6 +221,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.87" diff --git a/packages/system-query/Cargo.toml b/packages/system-query/Cargo.toml index 0ced6b86..17f087db 100644 --- a/packages/system-query/Cargo.toml +++ b/packages/system-query/Cargo.toml @@ -9,9 +9,11 @@ crate-type = ["cdylib"] [dependencies] itertools = "0.11.0" +rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } serde-wasm-bindgen = "0.5.0" wasm-bindgen = "0.2.87" +serde_json = "1.0.105" [dev-dependencies] wasm-bindgen-test = "0.3.37" diff --git a/packages/system-query/pkg/system_query.d.ts b/packages/system-query/pkg/system_query.d.ts index 3935fee6..1b244910 100644 --- a/packages/system-query/pkg/system_query.d.ts +++ b/packages/system-query/pkg/system_query.d.ts @@ -11,6 +11,12 @@ export function diff_filters(prev: any, next: any): any; * @returns {any} */ export function expand_filter(val: any): any; +/** +* @param {any} prev +* @param {any} next +* @returns {any} +*/ +export function get_diff(prev: any, next: any): any; export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; @@ -18,6 +24,7 @@ export interface InitOutput { readonly memory: WebAssembly.Memory; readonly diff_filters: (a: number, b: number, c: number) => void; readonly expand_filter: (a: number, b: number) => void; + readonly get_diff: (a: number, b: number, c: number) => void; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_add_to_stack_pointer: (a: number) => number; diff --git a/packages/system-query/pkg/system_query.js b/packages/system-query/pkg/system_query.js index 1b85b0e1..2cb5f0aa 100644 --- a/packages/system-query/pkg/system_query.js +++ b/packages/system-query/pkg/system_query.js @@ -229,6 +229,27 @@ export function expand_filter(val) { } } +/** +* @param {any} prev +* @param {any} next +* @returns {any} +*/ +export function get_diff(prev, next) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.get_diff(retptr, addHeapObject(prev), addHeapObject(next)); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var r2 = getInt32Memory0()[retptr / 4 + 2]; + if (r2) { + throw takeObject(r1); + } + return takeObject(r0); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + function handleError(f, args) { try { return f.apply(this, args); diff --git a/packages/system-query/pkg/system_query_bg.wasm b/packages/system-query/pkg/system_query_bg.wasm index a01a0ca8..1a585083 100644 Binary files a/packages/system-query/pkg/system_query_bg.wasm and b/packages/system-query/pkg/system_query_bg.wasm differ diff --git a/packages/system-query/pkg/system_query_bg.wasm.d.ts b/packages/system-query/pkg/system_query_bg.wasm.d.ts index de6a2575..ac811304 100644 --- a/packages/system-query/pkg/system_query_bg.wasm.d.ts +++ b/packages/system-query/pkg/system_query_bg.wasm.d.ts @@ -3,6 +3,7 @@ export const memory: WebAssembly.Memory; export function diff_filters(a: number, b: number, c: number): void; export function expand_filter(a: number, b: number): void; +export function get_diff(a: number, b: number, c: number): void; export function __wbindgen_malloc(a: number, b: number): number; export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number; export function __wbindgen_add_to_stack_pointer(a: number): number; diff --git a/packages/system-query/src/diff.rs b/packages/system-query/src/diff.rs index 948362aa..0e1a35d5 100644 --- a/packages/system-query/src/diff.rs +++ b/packages/system-query/src/diff.rs @@ -1,8 +1,8 @@ -use itertools::Itertools; use crate::FlatReqFilter; +use itertools::Itertools; -pub fn diff_filter(prev: &Vec, next: &Vec) -> Vec:: { - let mut added: Vec:: = vec![]; +pub fn diff_filter(prev: &Vec, next: &Vec) -> Vec { + let mut added: Vec = vec![]; for n in next.iter() { if !prev.iter().contains(&n) { @@ -19,38 +19,34 @@ mod tests { #[test] fn simple_diff_same() { - let prev = vec![ - FlatReqFilter { - id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - } - ]; - let next = vec![ - FlatReqFilter { - id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - } - ]; + let prev = vec![FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }]; + let next = vec![FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }]; let result = diff_filter(&prev, &next); assert_eq!(result, vec![]) @@ -58,22 +54,20 @@ mod tests { #[test] fn simple_diff_add() { - let prev = vec![ - FlatReqFilter { - id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - } - ]; + let prev = vec![FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }]; let next = vec![ FlatReqFilter { id: Some("a".to_owned()), @@ -106,8 +100,9 @@ mod tests { ]; let result = diff_filter(&prev, &next); - assert_eq!(result, vec![ - FlatReqFilter { + assert_eq!( + result, + vec![FlatReqFilter { id: Some("b".to_owned()), author: None, kind: None, @@ -120,48 +115,45 @@ mod tests { since: None, until: None, limit: None, - } - ]) + }] + ) } #[test] fn simple_diff_replace() { - let prev = vec![ - FlatReqFilter { - id: Some("a".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - } - ]; - let next = vec![ - FlatReqFilter { - id: Some("b".to_owned()), - author: None, - kind: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - }, - ]; + let prev = vec![FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }]; + let next = vec![FlatReqFilter { + id: Some("b".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }]; let result = diff_filter(&prev, &next); - assert_eq!(result, vec![ - FlatReqFilter { + assert_eq!( + result, + vec![FlatReqFilter { id: Some("b".to_owned()), author: None, kind: None, @@ -174,7 +166,7 @@ mod tests { since: None, until: None, limit: None, - } - ]) + }] + ) } -} \ No newline at end of file +} diff --git a/packages/system-query/src/expand.rs b/packages/system-query/src/expand.rs index ea619c12..46eb37f3 100644 --- a/packages/system-query/src/expand.rs +++ b/packages/system-query/src/expand.rs @@ -1,5 +1,5 @@ -use itertools::Itertools; use crate::{FlatReqFilter, ReqFilter}; +use itertools::Itertools; #[derive(Clone)] enum StringOrNumberEntry<'a> { @@ -12,39 +12,66 @@ pub fn expand_filter(filter: &ReqFilter) -> Vec { let mut inputs: Vec> = vec![]; if let Some(ids) = &filter.ids { - let t_ids = ids.iter().map(|z| StringOrNumberEntry::String(("id", z))).collect_vec(); + let t_ids = ids + .iter() + .map(|z| StringOrNumberEntry::String(("id", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(authors) = &filter.authors { - let t_ids = authors.iter().map(|z| StringOrNumberEntry::String(("author", z))).collect_vec(); + let t_ids = authors + .iter() + .map(|z| StringOrNumberEntry::String(("author", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(kinds) = &filter.kinds { - let t_ids = kinds.iter().map(|z| StringOrNumberEntry::Number(("kind", z))).collect_vec(); + let t_ids = kinds + .iter() + .map(|z| StringOrNumberEntry::Number(("kind", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(e_tags) = &filter.e_tag { - let t_ids = e_tags.iter().map(|z| StringOrNumberEntry::String(("e_tag", z))).collect_vec(); + let t_ids = e_tags + .iter() + .map(|z| StringOrNumberEntry::String(("e_tag", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(p_tags) = &filter.p_tag { - let t_ids = p_tags.iter().map(|z| StringOrNumberEntry::String(("p_tag", z))).collect_vec(); + let t_ids = p_tags + .iter() + .map(|z| StringOrNumberEntry::String(("p_tag", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(d_tags) = &filter.d_tag { - let t_ids = d_tags.iter().map(|z| StringOrNumberEntry::String(("d_tag", z))).collect_vec(); + let t_ids = d_tags + .iter() + .map(|z| StringOrNumberEntry::String(("d_tag", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(t_tags) = &filter.t_tag { - let t_ids = t_tags.iter().map(|z| StringOrNumberEntry::String(("t_tag", z))).collect_vec(); + let t_ids = t_tags + .iter() + .map(|z| StringOrNumberEntry::String(("t_tag", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(r_tags) = &filter.r_tag { - let t_ids = r_tags.iter().map(|z| StringOrNumberEntry::String(("r_tag", z))).collect_vec(); + let t_ids = r_tags + .iter() + .map(|z| StringOrNumberEntry::String(("r_tag", z))) + .collect_vec(); inputs.push(t_ids); } if let Some(search) = &filter.search { - let t_ids = search.iter().map(|z| StringOrNumberEntry::String(("search", z))).collect_vec(); + let t_ids = search + .iter() + .map(|z| StringOrNumberEntry::String(("search", z))) + .collect_vec(); inputs.push(t_ids); } @@ -155,28 +182,260 @@ mod tests { let output = expand_filter(&input); output.iter().take(5).for_each(|x| println!("{:?}", x)); let expected = vec![ - FlatReqFilter { author: Some("a".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("a".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, - FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(1), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(1), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(2), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(2), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(3), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("a".to_owned()), + kind: Some(3), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(1), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(1), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(2), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(2), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(3), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("b".to_owned()), + kind: Some(3), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(1), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(1), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(2), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(2), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(3), + id: Some("x".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, + FlatReqFilter { + author: Some("c".to_owned()), + kind: Some(3), + id: Some("y".to_owned()), + p_tag: Some("a".to_owned()), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: Some(99), + until: None, + limit: Some(10), + e_tag: None, + }, ]; assert_eq!(output.len(), expected.len()); - output - .iter() - .for_each(|a| assert!(expected.contains(a))); + output.iter().for_each(|a| assert!(expected.contains(a))); } -} \ No newline at end of file +} diff --git a/packages/system-query/src/lib.rs b/packages/system-query/src/lib.rs index bcdc5f46..75a8f131 100644 --- a/packages/system-query/src/lib.rs +++ b/packages/system-query/src/lib.rs @@ -1,12 +1,12 @@ -extern crate wasm_bindgen; - +use std::fmt::{Debug}; use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; -mod expand; mod diff; +mod expand; +mod merge; -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[derive(PartialEq, Clone, Serialize, Deserialize)] pub struct ReqFilter { #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] pub ids: Option>, @@ -34,7 +34,13 @@ pub struct ReqFilter { pub limit: Option, } -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +impl Debug for ReqFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&serde_json::to_string(self).unwrap().to_owned()) + } +} + +#[derive(PartialEq, Clone, Serialize, Deserialize)] pub struct FlatReqFilter { #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] id: Option, @@ -62,6 +68,12 @@ pub struct FlatReqFilter { limit: Option, } +impl Debug for FlatReqFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&serde_json::to_string(self).unwrap().to_owned()) + } +} + #[wasm_bindgen] pub fn diff_filters(prev: JsValue, next: JsValue) -> Result { let prev_parsed: Vec = serde_wasm_bindgen::from_value(prev)?; @@ -75,4 +87,116 @@ pub fn expand_filter(val: JsValue) -> Result { let parsed: ReqFilter = serde_wasm_bindgen::from_value(val)?; let result = expand::expand_filter(&parsed); Ok(serde_wasm_bindgen::to_value(&result)?) -} \ No newline at end of file +} + +#[wasm_bindgen] +pub fn get_diff(prev: JsValue, next: JsValue) -> Result { + let prev_parsed: Vec = serde_wasm_bindgen::from_value(prev)?; + let next_parsed: Vec = serde_wasm_bindgen::from_value(next)?; + let expanded_prev: Vec = prev_parsed + .iter() + .flat_map(|v| expand::expand_filter(v)) + .collect(); + let expanded_next: Vec = next_parsed + .iter() + .flat_map(|v| expand::expand_filter(v)) + .collect(); + let result = diff::diff_filter(&expanded_prev, &expanded_next); + Ok(serde_wasm_bindgen::to_value(&result)?) +} + +#[wasm_bindgen] +pub fn flat_merge(val: JsValue) -> Result { + let val_parsed: Vec = serde_wasm_bindgen::from_value(val)?; + let result = merge::flat_merge(&val_parsed); + Ok(serde_wasm_bindgen::to_value(&result)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use itertools::Itertools; + use std::cmp::Ordering; + + #[test] + fn flat_merge_expanded() { + let input = vec![ + ReqFilter { + ids: None, + kinds: Some(vec![1, 6969, 6]), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + authors: Some(vec![ + "kieran".to_string(), + "snort".to_string(), + "c".to_string(), + "d".to_string(), + "e".to_string(), + ]), + since: Some(1), + until: Some(100), + search: None, + limit: None, + }, + ReqFilter { + ids: None, + kinds: Some(vec![4]), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + authors: Some(vec!["kieran".to_string()]), + limit: None, + }, + ReqFilter { + ids: None, + authors: None, + kinds: Some(vec![4]), + e_tag: None, + p_tag: Some(vec!["kieran".to_string()]), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ReqFilter { + ids: None, + kinds: Some(vec![1000]), + authors: Some(vec!["snort".to_string()]), + p_tag: Some(vec!["kieran".to_string()]), + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + e_tag: None, + limit: None, + }, + ]; + + let expanded = input + .iter() + .flat_map(|v| expand::expand_filter(v)) + .sorted_by(|_, _| { + if rand::random() { + Ordering::Less + } else { + Ordering::Greater + } + }) + .collect_vec(); + let expanded_flat = merge::flat_merge(&expanded); + assert_eq!(expanded_flat, input); + } +} diff --git a/packages/system-query/src/merge.rs b/packages/system-query/src/merge.rs new file mode 100644 index 00000000..47a8ba81 --- /dev/null +++ b/packages/system-query/src/merge.rs @@ -0,0 +1,512 @@ +use crate::{FlatReqFilter, ReqFilter}; +use itertools::Itertools; +use std::cmp::Ordering; + +pub fn flat_merge(all: &Vec) -> Vec { + let mut ret: Vec = vec![]; + + let merge_sets: Vec> = vec![vec![all.first().unwrap()]]; + let merge_sets = all + .iter() + .skip(1) + .sorted_by(|a, b| match distance(&a, &b) { + 0 => Ordering::Equal, + 1 => Ordering::Less, + _ => Ordering::Greater, + }) + .fold(merge_sets, |mut acc, x| { + let mut did_match = false; + for y in acc.iter_mut() { + if y.iter().all(|z| can_merge_filters(z, x)) { + y.push(x); + did_match = true; + break; + } + } + if !did_match { + acc.push(vec![x]); + } + acc + }); + + for s in merge_sets.iter() { + ret.push(merge_set(s)); + } + ret +} + +fn merge_set(set: &Vec<&FlatReqFilter>) -> ReqFilter { + let ret = ReqFilter { + ids: None, + authors: None, + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + set.iter().fold(ret, |mut acc, x| { + array_prop_append(&x.id, &mut acc.ids); + array_prop_append(&x.author, &mut acc.authors); + array_prop_append(&x.kind, &mut acc.kinds); + array_prop_append(&x.e_tag, &mut acc.e_tag); + array_prop_append(&x.p_tag, &mut acc.p_tag); + array_prop_append(&x.t_tag, &mut acc.t_tag); + array_prop_append(&x.d_tag, &mut acc.d_tag); + array_prop_append(&x.r_tag, &mut acc.r_tag); + array_prop_append(&x.search, &mut acc.search); + acc.since = x.since; + acc.until = x.until; + acc.limit = x.limit; + + acc + }) +} + +fn can_merge_filters(a: &FlatReqFilter, b: &FlatReqFilter) -> bool { + if a.since != b.since || a.until != b.until || a.limit != b.limit || a.search != b.search { + return false; + } + + distance(a, b) <= 1 +} + +/// Calculate the distance in terms of similarity for merging +/// +/// The goal of this function is to find 2 filters which are very similar where +/// one filter may have a single property change like so: +/// +/// ```javascript +/// const a = { "kinds": 1, "authors": "a", "since": 99 }; +/// const b = { "kinds": 1, "authors": "b", "since": 99 }; +/// ``` +/// In this case these 2 filters could be merged because their distance is `1` +/// ```javascript +/// const result = { "kinds": [1], "authors": ["a", "b"], "since": 99 }; +/// ``` +fn distance(a: &FlatReqFilter, b: &FlatReqFilter) -> u32 { + let mut ret = 0u32; + + ret += prop_dist(&a.id, &b.id); + ret += prop_dist(&a.kind, &b.kind); + ret += prop_dist(&a.author, &b.author); + ret += prop_dist(&a.e_tag, &b.e_tag); + ret += prop_dist(&a.p_tag, &b.p_tag); + ret += prop_dist(&a.d_tag, &b.d_tag); + ret += prop_dist(&a.r_tag, &b.r_tag); + ret += prop_dist(&a.t_tag, &b.t_tag); + ret += prop_dist(&a.search, &b.search); + + ret +} + +#[inline(always)] +fn prop_dist(a: &Option, b: &Option) -> u32 { + if (a.is_some() && b.is_none()) || (a.is_none() && b.is_some()) { + return 10; + } else if a.is_some() && a != b { + return 1; + } + 0 +} + +#[inline(always)] +fn array_prop_append(val: &Option, arr: &mut Option>) { + if let Some(ap) = val { + if arr.is_none() { + *arr = Some(vec![ap.clone()]) + } else if !arr.as_ref().unwrap().contains(ap) { + arr.as_mut().unwrap().push(ap.clone()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn distance() { + let a = FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + let b = FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + let c = FlatReqFilter { + id: Some("c".to_owned()), + author: None, + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + let d = FlatReqFilter { + id: Some("a".to_owned()), + author: None, + kind: Some(1), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + let e = FlatReqFilter { + id: Some("e".to_owned()), + author: None, + kind: Some(1), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + assert_eq!(super::distance(&a, &b), 0); + assert_eq!(super::distance(&a, &c), 1); + assert_eq!(super::distance(&a, &d), 10); + assert_eq!(super::distance(&a, &e), 11); + } + + #[test] + fn merge_set() { + let a = FlatReqFilter { + id: Some("0".to_owned()), + author: Some("a".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(10), + }; + let b = FlatReqFilter { + id: Some("0".to_owned()), + author: Some("b".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(10), + }; + + let output = ReqFilter { + ids: Some(vec!["0".to_owned()]), + authors: Some(vec!["a".to_owned(), "b".to_owned()]), + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(10), + }; + assert_eq!(super::merge_set(&vec![&a, &b]), output); + } + + #[test] + fn can_merge_filters() { + let a = FlatReqFilter { + id: Some("0".to_owned()), + author: Some("a".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(10), + }; + let b = FlatReqFilter { + id: Some("0".to_owned()), + author: Some("b".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(10), + }; + let c = FlatReqFilter { + id: Some("0".to_owned()), + author: Some("b".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(100), + }; + assert!(super::can_merge_filters(&a, &b)); + assert!(!super::can_merge_filters(&b, &c)); + } + + #[test] + fn flat_merge() { + let input = vec![ + FlatReqFilter { + id: Some("0".to_owned()), + author: Some("a".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: Some("0".to_owned()), + author: Some("b".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: None, + author: None, + kind: Some(1), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: None, + author: None, + kind: Some(2), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: None, + author: None, + kind: Some(2), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: Some("0".to_owned()), + author: Some("c".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: None, + author: Some("c".to_owned()), + kind: Some(1), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + FlatReqFilter { + id: None, + author: Some("c".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(100), + }, + FlatReqFilter { + id: Some("1".to_owned()), + author: Some("c".to_owned()), + kind: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ]; + let output = vec![ + ReqFilter { + ids: Some(vec!["0".to_owned()]), + authors: Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ReqFilter { + ids: None, + authors: None, + kinds: Some(vec![1, 2]), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ReqFilter { + ids: None, + authors: Some(vec!["c".to_owned()]), + kinds: Some(vec![1]), + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ReqFilter { + ids: None, + authors: Some(vec!["c".to_owned()]), + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: Some(100), + }, + ReqFilter { + ids: Some(vec!["1".to_owned()]), + authors: Some(vec!["c".to_owned()]), + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }, + ]; + + assert_eq!(super::flat_merge(&input), output) + } +} diff --git a/packages/system-query/system-query.iml b/packages/system-query/system-query.iml new file mode 100644 index 00000000..2fecef3b --- /dev/null +++ b/packages/system-query/system-query.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index b0786757..90f8ecf6 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -241,7 +241,7 @@ export class NostrSystem extends ExternalStore implements System return existing; } const filters = !req.options?.skipDiff - ? req.buildDiff(this.#relayCache, existing.flatFilters) + ? req.buildDiff(this.#relayCache, existing.filters) : req.build(this.#relayCache); if (filters.length === 0 && !!req.options?.skipDiff) { return existing; diff --git a/packages/system/src/query.ts b/packages/system/src/query.ts index 0be31a46..065b0468 100644 --- a/packages/system/src/query.ts +++ b/packages/system/src/query.ts @@ -4,9 +4,7 @@ import { unixNowMs, unwrap } from "@snort/shared"; import { Connection, ReqFilter, Nips, TaggedNostrEvent } from "."; import { NoteStore } from "./note-collection"; -import { flatMerge } from "./request-merger"; import { BuiltRawReqFilter } from "./request-builder"; -import { FlatReqFilter, expandFilter } from "./request-expander"; import { eventMatchesFilter } from "./request-matcher"; /** @@ -19,7 +17,6 @@ class QueryTrace { eose?: number; close?: number; #wasForceClosed = false; - readonly flatFilters: Array; readonly #fnClose: (id: string) => void; readonly #fnProgress: () => void; @@ -34,7 +31,6 @@ class QueryTrace { this.start = unixNowMs(); this.#fnClose = fnClose; this.#fnProgress = fnProgress; - this.flatFilters = filters.flatMap(expandFilter); } sentToRelay() { @@ -166,11 +162,7 @@ export class Query implements QueryBase { * Recompute the complete set of compressed filters from all query traces */ get filters() { - return flatMerge(this.flatFilters); - } - - get flatFilters() { - return this.#tracing.flatMap(a => a.flatFilters); + return this.#tracing.flatMap(a => a.filters); } get feed() { diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts index 747a97a4..311d41e5 100644 --- a/packages/system/src/request-builder.ts +++ b/packages/system/src/request-builder.ts @@ -1,12 +1,12 @@ import debug from "debug"; import { v4 as uuid } from "uuid"; import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared"; +import { get_diff }from "@snort/system-query"; import { ReqFilter, u256, HexKey, EventKind } from "."; -import { diffFilters } from "./request-splitter"; import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model"; import { flatMerge, mergeSimilar } from "./request-merger"; -import { FlatReqFilter, expandFilter } from "./request-expander"; +import { FlatReqFilter } from "./request-expander"; /** * Which strategy is used when building REQ filters @@ -103,15 +103,16 @@ export class RequestBuilder { /** * Detects a change in request from a previous set of filters */ - buildDiff(relays: RelayCache, prev: Array): Array { + buildDiff(relays: RelayCache, prev: Array): Array { const start = unixNowMs(); - const next = this.#builders.flatMap(f => expandFilter(f.filter)); - const diff = diffFilters(prev, next); + //const next = this.#builders.flatMap(f => expandFilter(f.filter)); + //const diff = diffFilters(prev, next); + const diff = get_diff(prev, this.buildRaw()) as Array; const ts = unixNowMs() - start; this.#log("buildDiff %s %d ms", this.id, ts); - if (diff.changed) { - return splitFlatByWriteRelays(relays, diff.added).map(a => { + if (diff.length > 0) { + return splitFlatByWriteRelays(relays, diff).map(a => { return { strategy: RequestStrategy.AuthorsRelays, filters: flatMerge(a.filters),