diff --git a/packages/system-query/Cargo.toml b/packages/system-query/Cargo.toml index 17f087dbf..fb15f8163 100644 --- a/packages/system-query/Cargo.toml +++ b/packages/system-query/Cargo.toml @@ -9,11 +9,11 @@ crate-type = ["cdylib"] [dependencies] itertools = "0.11.0" -rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } serde-wasm-bindgen = "0.5.0" wasm-bindgen = "0.2.87" -serde_json = "1.0.105" [dev-dependencies] +rand = "0.8.5" wasm-bindgen-test = "0.3.37" +serde_json = "1.0.105" diff --git a/packages/system-query/pkg/system_query.d.ts b/packages/system-query/pkg/system_query.d.ts index 1b2449104..4b24908a0 100644 --- a/packages/system-query/pkg/system_query.d.ts +++ b/packages/system-query/pkg/system_query.d.ts @@ -17,6 +17,11 @@ export function expand_filter(val: any): any; * @returns {any} */ export function get_diff(prev: any, next: any): any; +/** +* @param {any} val +* @returns {any} +*/ +export function flat_merge(val: any): any; export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; @@ -25,6 +30,7 @@ export interface InitOutput { readonly diff_filters: (a: number, b: number, c: number) => void; readonly expand_filter: (a: number, b: number) => void; readonly get_diff: (a: number, b: number, c: number) => void; + readonly flat_merge: (a: number, b: number) => void; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_add_to_stack_pointer: (a: number) => number; diff --git a/packages/system-query/pkg/system_query.js b/packages/system-query/pkg/system_query.js index 2cb5f0aa0..9fb7deef2 100644 --- a/packages/system-query/pkg/system_query.js +++ b/packages/system-query/pkg/system_query.js @@ -250,6 +250,26 @@ export function get_diff(prev, next) { } } +/** +* @param {any} val +* @returns {any} +*/ +export function flat_merge(val) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.flat_merge(retptr, addHeapObject(val)); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var r2 = getInt32Memory0()[retptr / 4 + 2]; + if (r2) { + throw takeObject(r1); + } + return takeObject(r0); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + function handleError(f, args) { try { return f.apply(this, args); diff --git a/packages/system-query/pkg/system_query_bg.wasm b/packages/system-query/pkg/system_query_bg.wasm index 1a5850832..394d0f797 100644 Binary files a/packages/system-query/pkg/system_query_bg.wasm and b/packages/system-query/pkg/system_query_bg.wasm differ diff --git a/packages/system-query/pkg/system_query_bg.wasm.d.ts b/packages/system-query/pkg/system_query_bg.wasm.d.ts index ac811304a..506a78650 100644 --- a/packages/system-query/pkg/system_query_bg.wasm.d.ts +++ b/packages/system-query/pkg/system_query_bg.wasm.d.ts @@ -4,6 +4,7 @@ export const memory: WebAssembly.Memory; export function diff_filters(a: number, b: number, c: number): void; export function expand_filter(a: number, b: number): void; export function get_diff(a: number, b: number, c: number): void; +export function flat_merge(a: number, b: number): void; export function __wbindgen_malloc(a: number, b: number): number; export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number; export function __wbindgen_add_to_stack_pointer(a: number): number; diff --git a/packages/system-query/src/expand.rs b/packages/system-query/src/expand.rs index 46eb37f34..4871a0fe8 100644 --- a/packages/system-query/src/expand.rs +++ b/packages/system-query/src/expand.rs @@ -161,14 +161,19 @@ pub fn expand_filter(filter: &ReqFilter) -> Vec { mod tests { use super::*; use crate::ReqFilter; + use std::collections::HashSet; #[test] fn test_expand_filter() { let input = ReqFilter { - authors: Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), - kinds: Some(vec![1, 2, 3]), - ids: Some(vec!["x".to_owned(), "y".to_owned()]), - p_tag: Some(vec!["a".to_owned()]), + authors: Some(HashSet::from([ + "a".to_owned(), + "b".to_owned(), + "c".to_owned(), + ])), + kinds: Some(HashSet::from([1, 2, 3])), + ids: Some(HashSet::from(["x".to_owned(), "y".to_owned()])), + p_tag: Some(HashSet::from(["a".to_owned()])), t_tag: None, d_tag: None, r_tag: None, diff --git a/packages/system-query/src/filter.rs b/packages/system-query/src/filter.rs new file mode 100644 index 000000000..43f812ef6 --- /dev/null +++ b/packages/system-query/src/filter.rs @@ -0,0 +1,287 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +#[cfg(test)] +use std::fmt::Debug; +use std::hash::Hash; + +#[derive(PartialEq, Clone, Serialize, Deserialize)] +pub struct ReqFilter { + #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] + pub ids: Option>, + #[serde(rename = "authors", skip_serializing_if = "Option::is_none")] + pub authors: Option>, + #[serde(rename = "kinds", skip_serializing_if = "Option::is_none")] + pub kinds: Option>, + #[serde(rename = "#e", skip_serializing_if = "Option::is_none")] + pub e_tag: Option>, + #[serde(rename = "#p", skip_serializing_if = "Option::is_none")] + pub p_tag: Option>, + #[serde(rename = "#t", skip_serializing_if = "Option::is_none")] + pub t_tag: Option>, + #[serde(rename = "#d", skip_serializing_if = "Option::is_none")] + pub d_tag: Option>, + #[serde(rename = "#r", skip_serializing_if = "Option::is_none")] + pub r_tag: Option>, + #[serde(rename = "search", skip_serializing_if = "Option::is_none")] + pub search: Option>, + #[serde(rename = "since", skip_serializing_if = "Option::is_none")] + pub since: Option, + #[serde(rename = "until", skip_serializing_if = "Option::is_none")] + pub until: Option, + #[serde(rename = "limit", skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +#[cfg(test)] +impl Debug for ReqFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&serde_json::to_string(self).unwrap().to_owned()) + } +} + +#[derive(PartialEq, PartialOrd, Clone, Serialize, Deserialize)] +pub struct FlatReqFilter { + #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "authors", skip_serializing_if = "Option::is_none")] + pub author: Option, + #[serde(rename = "kinds", skip_serializing_if = "Option::is_none")] + pub kind: Option, + #[serde(rename = "#e", skip_serializing_if = "Option::is_none")] + pub e_tag: Option, + #[serde(rename = "#p", skip_serializing_if = "Option::is_none")] + pub p_tag: Option, + #[serde(rename = "#t", skip_serializing_if = "Option::is_none")] + pub t_tag: Option, + #[serde(rename = "#d", skip_serializing_if = "Option::is_none")] + pub d_tag: Option, + #[serde(rename = "#r", skip_serializing_if = "Option::is_none")] + pub r_tag: Option, + #[serde(rename = "search", skip_serializing_if = "Option::is_none")] + pub search: Option, + #[serde(rename = "since", skip_serializing_if = "Option::is_none")] + pub since: Option, + #[serde(rename = "until", skip_serializing_if = "Option::is_none")] + pub until: Option, + #[serde(rename = "limit", skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +#[cfg(test)] +impl Debug for FlatReqFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&serde_json::to_string(self).unwrap().to_owned()) + } +} + +pub trait Distance { + /// Calculate the distance in terms of similarity for merging + /// + /// The goal of this function is to find 2 filters which are very similar where + /// one filter may have a single property change like so: + /// + /// ```javascript + /// const a = { "kinds": 1, "authors": "a", "since": 99 }; + /// const b = { "kinds": 1, "authors": "b", "since": 99 }; + /// ``` + /// In this case these 2 filters could be merged because their distance is `1` + /// ```javascript + /// const result = { "kinds": [1], "authors": ["a", "b"], "since": 99 }; + /// ``` + fn distance(&self, other: &Self) -> u32; +} + +pub trait CanMerge { + fn can_merge(&self, other: &Self) -> bool; +} + +impl Distance for FlatReqFilter { + fn distance(&self, b: &Self) -> u32 { + let mut ret = 0u32; + + ret += prop_dist(&self.id, &b.id); + ret += prop_dist(&self.kind, &b.kind); + ret += prop_dist(&self.author, &b.author); + ret += prop_dist(&self.e_tag, &b.e_tag); + ret += prop_dist(&self.p_tag, &b.p_tag); + ret += prop_dist(&self.d_tag, &b.d_tag); + ret += prop_dist(&self.r_tag, &b.r_tag); + ret += prop_dist(&self.t_tag, &b.t_tag); + ret += prop_dist(&self.search, &b.search); + + ret + } +} + +impl CanMerge for FlatReqFilter { + fn can_merge(&self, other: &Self) -> bool { + if self.since != other.since + || self.until != other.until + || self.limit != other.limit + || self.search != other.search + { + return false; + } + + self.distance(other) <= 1 + } +} + +impl From> for ReqFilter { + fn from(value: Vec<&FlatReqFilter>) -> Self { + let ret = ReqFilter { + ids: None, + authors: None, + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + value.iter().fold(ret, |mut acc, x| { + array_prop_append(&x.id, &mut acc.ids); + array_prop_append(&x.author, &mut acc.authors); + array_prop_append(&x.kind, &mut acc.kinds); + array_prop_append(&x.e_tag, &mut acc.e_tag); + array_prop_append(&x.p_tag, &mut acc.p_tag); + array_prop_append(&x.t_tag, &mut acc.t_tag); + array_prop_append(&x.d_tag, &mut acc.d_tag); + array_prop_append(&x.r_tag, &mut acc.r_tag); + array_prop_append(&x.search, &mut acc.search); + acc.since = x.since; + acc.until = x.until; + acc.limit = x.limit; + + acc + }) + } +} + +impl From> for ReqFilter { + fn from(value: Vec<&ReqFilter>) -> Self { + let ret = ReqFilter { + ids: None, + authors: None, + kinds: None, + e_tag: None, + p_tag: None, + t_tag: None, + d_tag: None, + r_tag: None, + search: None, + since: None, + until: None, + limit: None, + }; + value.iter().fold(ret, |mut acc, x| { + array_prop_append_vec(&x.ids, &mut acc.ids); + array_prop_append_vec(&x.authors, &mut acc.authors); + array_prop_append_vec(&x.kinds, &mut acc.kinds); + array_prop_append_vec(&x.e_tag, &mut acc.e_tag); + array_prop_append_vec(&x.p_tag, &mut acc.p_tag); + array_prop_append_vec(&x.t_tag, &mut acc.t_tag); + array_prop_append_vec(&x.d_tag, &mut acc.d_tag); + array_prop_append_vec(&x.r_tag, &mut acc.r_tag); + array_prop_append_vec(&x.search, &mut acc.search); + acc.since = x.since; + acc.until = x.until; + acc.limit = x.limit; + + acc + }) + } +} + +impl Distance for ReqFilter { + fn distance(&self, b: &Self) -> u32 { + let mut ret = 0u32; + + ret += prop_dist_vec(&self.ids, &b.ids); + ret += prop_dist_vec(&self.kinds, &b.kinds); + ret += prop_dist_vec(&self.authors, &b.authors); + ret += prop_dist_vec(&self.e_tag, &b.e_tag); + ret += prop_dist_vec(&self.p_tag, &b.p_tag); + ret += prop_dist_vec(&self.d_tag, &b.d_tag); + ret += prop_dist_vec(&self.r_tag, &b.r_tag); + ret += prop_dist_vec(&self.t_tag, &b.t_tag); + ret += prop_dist_vec(&self.search, &b.search); + + ret + } +} + +impl CanMerge for ReqFilter { + fn can_merge(&self, other: &Self) -> bool { + if self.since != other.since + || self.until != other.until + || self.limit != other.limit + || self.search != other.search + { + return false; + } + + self.distance(other) <= 1 + } +} + +#[inline(always)] +fn prop_dist(a: &Option, b: &Option) -> u32 { + if (a.is_some() && b.is_none()) || (a.is_none() && b.is_some()) { + return 10; + } else if a.is_some() && a != b { + return 1; + } + 0 +} + +#[inline(always)] +fn prop_dist_vec(a: &Option>, b: &Option>) -> u32 { + if (a.is_some() && b.is_none()) || (a.is_none() && b.is_some()) { + return 10; + } + match (a, b) { + (Some(aa), Some(bb)) => { + if aa.len() != bb.len() { + 1 + } else if aa == bb { + 0 + } else { + 1 + } + } + (None, None) => 0, + _ => panic!("Should not reach here!"), + } +} + +#[inline(always)] +fn array_prop_append(val: &Option, arr: &mut Option>) { + if let Some(ap) = val { + if arr.is_none() { + *arr = Some(HashSet::from([ap.clone()])) + } else { + arr.as_mut().unwrap().insert(ap.clone()); + } + } +} + +#[inline(always)] +fn array_prop_append_vec( + val: &Option>, + arr: &mut Option>, +) { + if let Some(ap) = val { + if arr.is_none() { + *arr = Some(ap.clone()) + } else { + ap.iter().for_each(|v| { + arr.as_mut().unwrap().insert((*v).clone()); + }); + } + } +} diff --git a/packages/system-query/src/lib.rs b/packages/system-query/src/lib.rs index 75a8f131c..0b1186088 100644 --- a/packages/system-query/src/lib.rs +++ b/packages/system-query/src/lib.rs @@ -1,79 +1,11 @@ -use std::fmt::{Debug}; -use serde::{Deserialize, Serialize}; +use crate::filter::{FlatReqFilter, ReqFilter}; use wasm_bindgen::prelude::*; mod diff; mod expand; +mod filter; mod merge; -#[derive(PartialEq, Clone, Serialize, Deserialize)] -pub struct ReqFilter { - #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] - pub ids: Option>, - #[serde(rename = "authors", skip_serializing_if = "Option::is_none")] - pub authors: Option>, - #[serde(rename = "kinds", skip_serializing_if = "Option::is_none")] - pub kinds: Option>, - #[serde(rename = "#e", skip_serializing_if = "Option::is_none")] - pub e_tag: Option>, - #[serde(rename = "#p", skip_serializing_if = "Option::is_none")] - pub p_tag: Option>, - #[serde(rename = "#t", skip_serializing_if = "Option::is_none")] - pub t_tag: Option>, - #[serde(rename = "#d", skip_serializing_if = "Option::is_none")] - pub d_tag: Option>, - #[serde(rename = "#r", skip_serializing_if = "Option::is_none")] - pub r_tag: Option>, - #[serde(rename = "search", skip_serializing_if = "Option::is_none")] - pub search: Option>, - #[serde(rename = "since", skip_serializing_if = "Option::is_none")] - pub since: Option, - #[serde(rename = "until", skip_serializing_if = "Option::is_none")] - pub until: Option, - #[serde(rename = "limit", skip_serializing_if = "Option::is_none")] - pub limit: Option, -} - -impl Debug for ReqFilter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&serde_json::to_string(self).unwrap().to_owned()) - } -} - -#[derive(PartialEq, Clone, Serialize, Deserialize)] -pub struct FlatReqFilter { - #[serde(rename = "ids", skip_serializing_if = "Option::is_none")] - id: Option, - #[serde(rename = "authors", skip_serializing_if = "Option::is_none")] - author: Option, - #[serde(rename = "kinds", skip_serializing_if = "Option::is_none")] - kind: Option, - #[serde(rename = "#e", skip_serializing_if = "Option::is_none")] - e_tag: Option, - #[serde(rename = "#p", skip_serializing_if = "Option::is_none")] - p_tag: Option, - #[serde(rename = "#t", skip_serializing_if = "Option::is_none")] - t_tag: Option, - #[serde(rename = "#d", skip_serializing_if = "Option::is_none")] - d_tag: Option, - #[serde(rename = "#r", skip_serializing_if = "Option::is_none")] - r_tag: Option, - #[serde(rename = "search", skip_serializing_if = "Option::is_none")] - search: Option, - #[serde(rename = "since", skip_serializing_if = "Option::is_none")] - since: Option, - #[serde(rename = "until", skip_serializing_if = "Option::is_none")] - until: Option, - #[serde(rename = "limit", skip_serializing_if = "Option::is_none")] - limit: Option, -} - -impl Debug for FlatReqFilter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&serde_json::to_string(self).unwrap().to_owned()) - } -} - #[wasm_bindgen] pub fn diff_filters(prev: JsValue, next: JsValue) -> Result { let prev_parsed: Vec = serde_wasm_bindgen::from_value(prev)?; @@ -108,7 +40,7 @@ pub fn get_diff(prev: JsValue, next: JsValue) -> Result { #[wasm_bindgen] pub fn flat_merge(val: JsValue) -> Result { let val_parsed: Vec = serde_wasm_bindgen::from_value(val)?; - let result = merge::flat_merge(&val_parsed); + let result = merge::merge::(val_parsed.iter().collect()); Ok(serde_wasm_bindgen::to_value(&result)?) } @@ -117,25 +49,26 @@ mod tests { use super::*; use itertools::Itertools; use std::cmp::Ordering; + use std::collections::HashSet; #[test] fn flat_merge_expanded() { let input = vec![ ReqFilter { ids: None, - kinds: Some(vec![1, 6969, 6]), + kinds: Some(HashSet::from([1, 6969, 6])), e_tag: None, p_tag: None, t_tag: None, d_tag: None, r_tag: None, - authors: Some(vec![ + authors: Some(HashSet::from([ "kieran".to_string(), "snort".to_string(), "c".to_string(), "d".to_string(), "e".to_string(), - ]), + ])), since: Some(1), until: Some(100), search: None, @@ -143,7 +76,7 @@ mod tests { }, ReqFilter { ids: None, - kinds: Some(vec![4]), + kinds: Some(HashSet::from([4])), e_tag: None, p_tag: None, t_tag: None, @@ -152,15 +85,15 @@ mod tests { search: None, since: None, until: None, - authors: Some(vec!["kieran".to_string()]), + authors: Some(HashSet::from(["kieran".to_string()])), limit: None, }, ReqFilter { ids: None, authors: None, - kinds: Some(vec![4]), + kinds: Some(HashSet::from([4])), e_tag: None, - p_tag: Some(vec!["kieran".to_string()]), + p_tag: Some(HashSet::from(["kieran".to_string()])), t_tag: None, d_tag: None, r_tag: None, @@ -171,9 +104,9 @@ mod tests { }, ReqFilter { ids: None, - kinds: Some(vec![1000]), - authors: Some(vec!["snort".to_string()]), - p_tag: Some(vec!["kieran".to_string()]), + kinds: Some(HashSet::from([1000])), + authors: Some(HashSet::from(["snort".to_string()])), + p_tag: Some(HashSet::from(["kieran".to_string()])), t_tag: None, d_tag: None, r_tag: None, @@ -196,7 +129,8 @@ mod tests { } }) .collect_vec(); - let expanded_flat = merge::flat_merge(&expanded); - assert_eq!(expanded_flat, input); + let merged_expanded: Vec = merge::merge(expanded.iter().collect()); + assert_eq!(merged_expanded.len(), input.len()); + assert!(merged_expanded.iter().all(|v| input.contains(v))); } } diff --git a/packages/system-query/src/merge.rs b/packages/system-query/src/merge.rs index 47a8ba815..3b2455789 100644 --- a/packages/system-query/src/merge.rs +++ b/packages/system-query/src/merge.rs @@ -1,134 +1,59 @@ -use crate::{FlatReqFilter, ReqFilter}; -use itertools::Itertools; -use std::cmp::Ordering; +use crate::filter::CanMerge; -pub fn flat_merge(all: &Vec) -> Vec { - let mut ret: Vec = vec![]; - - let merge_sets: Vec> = vec![vec![all.first().unwrap()]]; - let merge_sets = all - .iter() - .skip(1) - .sorted_by(|a, b| match distance(&a, &b) { - 0 => Ordering::Equal, - 1 => Ordering::Less, - _ => Ordering::Greater, - }) - .fold(merge_sets, |mut acc, x| { - let mut did_match = false; - for y in acc.iter_mut() { - if y.iter().all(|z| can_merge_filters(z, x)) { - y.push(x); - did_match = true; - break; - } - } - if !did_match { - acc.push(vec![x]); - } - acc - }); - - for s in merge_sets.iter() { - ret.push(merge_set(s)); - } - ret -} - -fn merge_set(set: &Vec<&FlatReqFilter>) -> ReqFilter { - let ret = ReqFilter { - ids: None, - authors: None, - kinds: None, - e_tag: None, - p_tag: None, - t_tag: None, - d_tag: None, - r_tag: None, - search: None, - since: None, - until: None, - limit: None, - }; - set.iter().fold(ret, |mut acc, x| { - array_prop_append(&x.id, &mut acc.ids); - array_prop_append(&x.author, &mut acc.authors); - array_prop_append(&x.kind, &mut acc.kinds); - array_prop_append(&x.e_tag, &mut acc.e_tag); - array_prop_append(&x.p_tag, &mut acc.p_tag); - array_prop_append(&x.t_tag, &mut acc.t_tag); - array_prop_append(&x.d_tag, &mut acc.d_tag); - array_prop_append(&x.r_tag, &mut acc.r_tag); - array_prop_append(&x.search, &mut acc.search); - acc.since = x.since; - acc.until = x.until; - acc.limit = x.limit; - - acc - }) -} - -fn can_merge_filters(a: &FlatReqFilter, b: &FlatReqFilter) -> bool { - if a.since != b.since || a.until != b.until || a.limit != b.limit || a.search != b.search { - return false; - } - - distance(a, b) <= 1 -} - -/// Calculate the distance in terms of similarity for merging -/// -/// The goal of this function is to find 2 filters which are very similar where -/// one filter may have a single property change like so: -/// -/// ```javascript -/// const a = { "kinds": 1, "authors": "a", "since": 99 }; -/// const b = { "kinds": 1, "authors": "b", "since": 99 }; -/// ``` -/// In this case these 2 filters could be merged because their distance is `1` -/// ```javascript -/// const result = { "kinds": [1], "authors": ["a", "b"], "since": 99 }; -/// ``` -fn distance(a: &FlatReqFilter, b: &FlatReqFilter) -> u32 { - let mut ret = 0u32; - - ret += prop_dist(&a.id, &b.id); - ret += prop_dist(&a.kind, &b.kind); - ret += prop_dist(&a.author, &b.author); - ret += prop_dist(&a.e_tag, &b.e_tag); - ret += prop_dist(&a.p_tag, &b.p_tag); - ret += prop_dist(&a.d_tag, &b.d_tag); - ret += prop_dist(&a.r_tag, &b.r_tag); - ret += prop_dist(&a.t_tag, &b.t_tag); - ret += prop_dist(&a.search, &b.search); - - ret -} - -#[inline(always)] -fn prop_dist(a: &Option, b: &Option) -> u32 { - if (a.is_some() && b.is_none()) || (a.is_none() && b.is_some()) { - return 10; - } else if a.is_some() && a != b { - return 1; - } - 0 -} - -#[inline(always)] -fn array_prop_append(val: &Option, arr: &mut Option>) { - if let Some(ap) = val { - if arr.is_none() { - *arr = Some(vec![ap.clone()]) - } else if !arr.as_ref().unwrap().contains(ap) { - arr.as_mut().unwrap().push(ap.clone()); +pub fn merge<'a, T, Z>(all: Vec<&'a T>) -> Vec +where + T: CanMerge, + for<'b> Z: CanMerge + From> + From>, +{ + let mut ret: Vec = merge_once(all); + loop { + let last_len = ret.len(); + ret = merge_once(ret.iter().collect()); + if last_len == ret.len() { + break; } } + ret +} + +fn merge_once<'a, T, Z>(all: Vec<&'a T>) -> Vec +where + T: CanMerge, + for<'b> Z: CanMerge + From> + From>, +{ + let mut ret: Vec = vec![]; + if all.is_empty() { + return ret; + } + + let merge_sets: Vec> = vec![vec![all.first().unwrap()]]; + let merge_sets = all.iter().skip(1).fold(merge_sets, |mut acc, x| { + let mut did_match = false; + for y in acc.iter_mut() { + if y.iter().all(|z| z.can_merge(x)) { + y.push(x); + did_match = true; + break; + } + } + if !did_match { + acc.push(vec![x]); + } + acc + }); + + for s in merge_sets { + ret.push(Z::from(s)); + } + + ret } #[cfg(test)] mod tests { use super::*; + use crate::filter::{Distance, FlatReqFilter, ReqFilter}; + use std::collections::HashSet; #[test] fn distance() { @@ -202,10 +127,10 @@ mod tests { until: None, limit: None, }; - assert_eq!(super::distance(&a, &b), 0); - assert_eq!(super::distance(&a, &c), 1); - assert_eq!(super::distance(&a, &d), 10); - assert_eq!(super::distance(&a, &e), 11); + assert_eq!(a.distance(&b), 0); + assert_eq!(a.distance(&c), 1); + assert_eq!(a.distance(&d), 10); + assert_eq!(a.distance(&e), 11); } #[test] @@ -240,8 +165,8 @@ mod tests { }; let output = ReqFilter { - ids: Some(vec!["0".to_owned()]), - authors: Some(vec!["a".to_owned(), "b".to_owned()]), + ids: Some(HashSet::from(["0".to_owned()])), + authors: Some(HashSet::from(["a".to_owned(), "b".to_owned()])), kinds: None, e_tag: None, p_tag: None, @@ -253,7 +178,7 @@ mod tests { until: None, limit: Some(10), }; - assert_eq!(super::merge_set(&vec![&a, &b]), output); + assert_eq!(ReqFilter::from(vec![&a, &b]), output); } #[test] @@ -300,8 +225,8 @@ mod tests { until: None, limit: Some(100), }; - assert!(super::can_merge_filters(&a, &b)); - assert!(!super::can_merge_filters(&b, &c)); + assert!(&a.can_merge(&b)); + assert!(!&b.can_merge(&c)); } #[test] @@ -436,8 +361,12 @@ mod tests { ]; let output = vec![ ReqFilter { - ids: Some(vec!["0".to_owned()]), - authors: Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), + ids: Some(HashSet::from(["0".to_owned()])), + authors: Some(HashSet::from([ + "a".to_owned(), + "b".to_owned(), + "c".to_owned(), + ])), kinds: None, e_tag: None, p_tag: None, @@ -452,7 +381,7 @@ mod tests { ReqFilter { ids: None, authors: None, - kinds: Some(vec![1, 2]), + kinds: Some(HashSet::from([1, 2])), e_tag: None, p_tag: None, t_tag: None, @@ -465,8 +394,8 @@ mod tests { }, ReqFilter { ids: None, - authors: Some(vec!["c".to_owned()]), - kinds: Some(vec![1]), + authors: Some(HashSet::from(["c".to_owned()])), + kinds: Some(HashSet::from([1])), e_tag: None, p_tag: None, t_tag: None, @@ -479,7 +408,7 @@ mod tests { }, ReqFilter { ids: None, - authors: Some(vec!["c".to_owned()]), + authors: Some(HashSet::from(["c".to_owned()])), kinds: None, e_tag: None, p_tag: None, @@ -492,8 +421,8 @@ mod tests { limit: Some(100), }, ReqFilter { - ids: Some(vec!["1".to_owned()]), - authors: Some(vec!["c".to_owned()]), + ids: Some(HashSet::from(["1".to_owned()])), + authors: Some(HashSet::from(["c".to_owned()])), kinds: None, e_tag: None, p_tag: None, @@ -507,6 +436,9 @@ mod tests { }, ]; - assert_eq!(super::flat_merge(&input), output) + assert_eq!( + merge::(input.iter().collect()), + output + ) } } diff --git a/packages/system/src/nostr-system.ts b/packages/system/src/nostr-system.ts index 90f8ecf68..ec22de6c6 100644 --- a/packages/system/src/nostr-system.ts +++ b/packages/system/src/nostr-system.ts @@ -4,7 +4,7 @@ import { unwrap, sanitizeRelayUrl, ExternalStore, FeedCache } from "@snort/share import { NostrEvent, TaggedNostrEvent } from "./nostr"; import { AuthHandler, Connection, RelaySettings, ConnectionStateSnapshot } from "./connection"; import { Query } from "./query"; -import { NoteCollection, NoteStore, NoteStoreHook, NoteStoreSnapshotData } from "./note-collection"; +import { NoteCollection, NoteStore, NoteStoreSnapshotData } from "./note-collection"; import { BuiltRawReqFilter, RequestBuilder } from "./request-builder"; import { RelayMetricHandler } from "./relay-metric-handler"; import { @@ -258,8 +258,12 @@ export class NostrSystem extends ExternalStore implements System const filters = req.build(this.#relayCache); const q = new Query(req.id, req.instance, store, req.options?.leaveOpen); if (filters.some(a => a.filters.some(b => b.ids))) { + const expectIds = new Set(filters.flatMap(a => a.filters).flatMap(a => a.ids ?? [])); q.feed.onEvent(async evs => { - await this.#eventsCache.bulkSet(evs); + const toSet = evs.filter(a => expectIds.has(a.id) && this.#eventsCache.getFromCache(a.id) === undefined); + if(toSet.length > 0) { + await this.#eventsCache.bulkSet(toSet); + } }); } this.Queries.set(req.id, q); diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts index 311d41e57..9a7c49aad 100644 --- a/packages/system/src/request-builder.ts +++ b/packages/system/src/request-builder.ts @@ -1,7 +1,7 @@ import debug from "debug"; import { v4 as uuid } from "uuid"; import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared"; -import { get_diff }from "@snort/system-query"; +import { flat_merge, get_diff }from "@snort/system-query"; import { ReqFilter, u256, HexKey, EventKind } from "."; import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model"; @@ -115,7 +115,7 @@ export class RequestBuilder { return splitFlatByWriteRelays(relays, diff).map(a => { return { strategy: RequestStrategy.AuthorsRelays, - filters: flatMerge(a.filters), + filters: flat_merge(a.filters) as Array, relay: a.relay, }; });