Kieran
/
snort
5
14
Fork 14
Code Issues 53 Pull Requests 1 Packages Projects Releases 24 Wiki Activity

Merger

This commit is contained in:
Kieran 2023-09-08 17:25:36 +01:00
parent 08bc4cafa9
commit 3e0fae2c33
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
15 changed files with 1156 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/system-query/.gitignore vendored
View File

@ -1,2 +1,3 @@
.idea/
target/
target/
*.txt

84
packages/system-query/Cargo.lock generated
View File

@ -30,6 +30,17 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "getrandom"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "itertools"
version = "0.11.0"
@ -39,6 +50,12 @@ dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "js-sys"
version = "0.3.64"
@ -48,6 +65,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "log"
version = "0.4.20"
@ -60,6 +83,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro2"
version = "1.0.66"
@ -78,6 +107,42 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "ryu"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "scoped-tls"
version = "1.0.1"
@ -115,6 +180,17 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "2.0.31"
@ -131,8 +207,10 @@ name = "system-query"
version = "0.1.0"
dependencies = [
"itertools",
"rand",
"serde",
"serde-wasm-bindgen",
"serde_json",
"wasm-bindgen",
"wasm-bindgen-test",
]
@ -143,6 +221,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.87"

2
packages/system-query/Cargo.toml
View File

@ -9,9 +9,11 @@ crate-type = ["cdylib"]
[dependencies]
itertools = "0.11.0"
rand = "0.8.5"
serde = { version = "1.0.188", features = ["derive"] }
serde-wasm-bindgen = "0.5.0"
wasm-bindgen = "0.2.87"
serde_json = "1.0.105"
[dev-dependencies]
wasm-bindgen-test = "0.3.37"

7
packages/system-query/pkg/system_query.d.ts vendored
View File

@ -11,6 +11,12 @@ export function diff_filters(prev: any, next: any): any;
* @returns {any}
*/
export function expand_filter(val: any): any;
/**
* @param {any} prev
* @param {any} next
* @returns {any}
*/
export function get_diff(prev: any, next: any): any;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
@ -18,6 +24,7 @@ export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly diff_filters: (a: number, b: number, c: number) => void;
readonly expand_filter: (a: number, b: number) => void;
readonly get_diff: (a: number, b: number, c: number) => void;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
readonly __wbindgen_add_to_stack_pointer: (a: number) => number;

21
packages/system-query/pkg/system_query.js
View File

@ -229,6 +229,27 @@ export function expand_filter(val) {
}
}
/**
* @param {any} prev
* @param {any} next
* @returns {any}
*/
export function get_diff(prev, next) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.get_diff(retptr, addHeapObject(prev), addHeapObject(next));
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
var r2 = getInt32Memory0()[retptr / 4 + 2];
if (r2) {
throw takeObject(r1);
}
return takeObject(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
function handleError(f, args) {
try {
return f.apply(this, args);

BIN
packages/system-query/pkg/system_query_bg.wasm
View File

Binary file not shown.

1
packages/system-query/pkg/system_query_bg.wasm.d.ts vendored
View File

@ -3,6 +3,7 @@
export const memory: WebAssembly.Memory;
export function diff_filters(a: number, b: number, c: number): void;
export function expand_filter(a: number, b: number): void;
export function get_diff(a: number, b: number, c: number): void;
export function __wbindgen_malloc(a: number, b: number): number;
export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;
export function __wbindgen_add_to_stack_pointer(a: number): number;

176
packages/system-query/src/diff.rs
View File

@ -1,8 +1,8 @@
use itertools::Itertools;
use crate::FlatReqFilter;
use itertools::Itertools;
pub fn diff_filter(prev: &Vec<FlatReqFilter>, next: &Vec<FlatReqFilter>) -> Vec::<FlatReqFilter> {
let mut added: Vec::<FlatReqFilter> = vec![];
pub fn diff_filter(prev: &Vec<FlatReqFilter>, next: &Vec<FlatReqFilter>) -> Vec<FlatReqFilter> {
let mut added: Vec<FlatReqFilter> = vec![];
for n in next.iter() {
if !prev.iter().contains(&n) {
@ -19,38 +19,34 @@ mod tests {
#[test]
fn simple_diff_same() {
let prev = vec![
FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}
];
let next = vec![
FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}
];
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}];
let next = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}];
let result = diff_filter(&prev, &next);
assert_eq!(result, vec![])
@ -58,22 +54,20 @@ mod tests {
#[test]
fn simple_diff_add() {
let prev = vec![
FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}
];
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}];
let next = vec![
FlatReqFilter {
id: Some("a".to_owned()),
@ -106,8 +100,9 @@ mod tests {
];
let result = diff_filter(&prev, &next);
assert_eq!(result, vec![
FlatReqFilter {
assert_eq!(
result,
vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
@ -120,48 +115,45 @@ mod tests {
since: None,
until: None,
limit: None,
}
])
}]
)
}
#[test]
fn simple_diff_replace() {
let prev = vec![
FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}
];
let next = vec![
FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
];
let prev = vec![FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}];
let next = vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
}];
let result = diff_filter(&prev, &next);
assert_eq!(result, vec![
FlatReqFilter {
assert_eq!(
result,
vec![FlatReqFilter {
id: Some("b".to_owned()),
author: None,
kind: None,
@ -174,7 +166,7 @@ mod tests {
since: None,
until: None,
limit: None,
}
])
}]
)
}
}
}

323
packages/system-query/src/expand.rs
View File

@ -1,5 +1,5 @@
use itertools::Itertools;
use crate::{FlatReqFilter, ReqFilter};
use itertools::Itertools;
#[derive(Clone)]
enum StringOrNumberEntry<'a> {
@ -12,39 +12,66 @@ pub fn expand_filter(filter: &ReqFilter) -> Vec<FlatReqFilter> {
let mut inputs: Vec<Vec<StringOrNumberEntry>> = vec![];
if let Some(ids) = &filter.ids {
let t_ids = ids.iter().map(|z| StringOrNumberEntry::String(("id", z))).collect_vec();
let t_ids = ids
.iter()
.map(|z| StringOrNumberEntry::String(("id", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(authors) = &filter.authors {
let t_ids = authors.iter().map(|z| StringOrNumberEntry::String(("author", z))).collect_vec();
let t_ids = authors
.iter()
.map(|z| StringOrNumberEntry::String(("author", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(kinds) = &filter.kinds {
let t_ids = kinds.iter().map(|z| StringOrNumberEntry::Number(("kind", z))).collect_vec();
let t_ids = kinds
.iter()
.map(|z| StringOrNumberEntry::Number(("kind", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(e_tags) = &filter.e_tag {
let t_ids = e_tags.iter().map(|z| StringOrNumberEntry::String(("e_tag", z))).collect_vec();
let t_ids = e_tags
.iter()
.map(|z| StringOrNumberEntry::String(("e_tag", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(p_tags) = &filter.p_tag {
let t_ids = p_tags.iter().map(|z| StringOrNumberEntry::String(("p_tag", z))).collect_vec();
let t_ids = p_tags
.iter()
.map(|z| StringOrNumberEntry::String(("p_tag", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(d_tags) = &filter.d_tag {
let t_ids = d_tags.iter().map(|z| StringOrNumberEntry::String(("d_tag", z))).collect_vec();
let t_ids = d_tags
.iter()
.map(|z| StringOrNumberEntry::String(("d_tag", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(t_tags) = &filter.t_tag {
let t_ids = t_tags.iter().map(|z| StringOrNumberEntry::String(("t_tag", z))).collect_vec();
let t_ids = t_tags
.iter()
.map(|z| StringOrNumberEntry::String(("t_tag", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(r_tags) = &filter.r_tag {
let t_ids = r_tags.iter().map(|z| StringOrNumberEntry::String(("r_tag", z))).collect_vec();
let t_ids = r_tags
.iter()
.map(|z| StringOrNumberEntry::String(("r_tag", z)))
.collect_vec();
inputs.push(t_ids);
}
if let Some(search) = &filter.search {
let t_ids = search.iter().map(|z| StringOrNumberEntry::String(("search", z))).collect_vec();
let t_ids = search
.iter()
.map(|z| StringOrNumberEntry::String(("search", z)))
.collect_vec();
inputs.push(t_ids);
}
@ -155,28 +182,260 @@ mod tests {
let output = expand_filter(&input);
output.iter().take(5).for_each(|x| println!("{:?}", x));
let expected = vec![
FlatReqFilter { author: Some("a".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("a".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("a".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("a".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("b".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(1), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(2), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("x".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter { author: Some("c".to_owned()), kind: Some(3), id: Some("y".to_owned()), p_tag: Some("a".to_owned()), t_tag: None, d_tag: None, r_tag: None, search: None, since: Some(99), until: None, limit: Some(10), e_tag: None },
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("a".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("b".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(1),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(1),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(2),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(2),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(3),
id: Some("x".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
FlatReqFilter {
author: Some("c".to_owned()),
kind: Some(3),
id: Some("y".to_owned()),
p_tag: Some("a".to_owned()),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: Some(99),
until: None,
limit: Some(10),
e_tag: None,
},
];
assert_eq!(output.len(), expected.len());
output
.iter()
.for_each(|a| assert!(expected.contains(a)));
output.iter().for_each(|a| assert!(expected.contains(a)));
}
}
}

136
packages/system-query/src/lib.rs
View File

@ -1,12 +1,12 @@
extern crate wasm_bindgen;
use std::fmt::{Debug};
use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
mod expand;
mod diff;
mod expand;
mod merge;
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
#[derive(PartialEq, Clone, Serialize, Deserialize)]
pub struct ReqFilter {
#[serde(rename = "ids", skip_serializing_if = "Option::is_none")]
pub ids: Option<Vec<String>>,
@ -34,7 +34,13 @@ pub struct ReqFilter {
pub limit: Option<i32>,
}
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
impl Debug for ReqFilter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&serde_json::to_string(self).unwrap().to_owned())
}
}
#[derive(PartialEq, Clone, Serialize, Deserialize)]
pub struct FlatReqFilter {
#[serde(rename = "ids", skip_serializing_if = "Option::is_none")]
id: Option<String>,
@ -62,6 +68,12 @@ pub struct FlatReqFilter {
limit: Option<i32>,
}
impl Debug for FlatReqFilter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&serde_json::to_string(self).unwrap().to_owned())
}
}
#[wasm_bindgen]
pub fn diff_filters(prev: JsValue, next: JsValue) -> Result<JsValue, JsValue> {
let prev_parsed: Vec<FlatReqFilter> = serde_wasm_bindgen::from_value(prev)?;
@ -75,4 +87,116 @@ pub fn expand_filter(val: JsValue) -> Result<JsValue, JsValue> {
let parsed: ReqFilter = serde_wasm_bindgen::from_value(val)?;
let result = expand::expand_filter(&parsed);
Ok(serde_wasm_bindgen::to_value(&result)?)
}
}
#[wasm_bindgen]
pub fn get_diff(prev: JsValue, next: JsValue) -> Result<JsValue, JsValue> {
let prev_parsed: Vec<ReqFilter> = serde_wasm_bindgen::from_value(prev)?;
let next_parsed: Vec<ReqFilter> = serde_wasm_bindgen::from_value(next)?;
let expanded_prev: Vec<FlatReqFilter> = prev_parsed
.iter()
.flat_map(|v| expand::expand_filter(v))
.collect();
let expanded_next: Vec<FlatReqFilter> = next_parsed
.iter()
.flat_map(|v| expand::expand_filter(v))
.collect();
let result = diff::diff_filter(&expanded_prev, &expanded_next);
Ok(serde_wasm_bindgen::to_value(&result)?)
}
#[wasm_bindgen]
pub fn flat_merge(val: JsValue) -> Result<JsValue, JsValue> {
let val_parsed: Vec<FlatReqFilter> = serde_wasm_bindgen::from_value(val)?;
let result = merge::flat_merge(&val_parsed);
Ok(serde_wasm_bindgen::to_value(&result)?)
}
#[cfg(test)]
mod tests {
use super::*;
use itertools::Itertools;
use std::cmp::Ordering;
#[test]
fn flat_merge_expanded() {
let input = vec![
ReqFilter {
ids: None,
kinds: Some(vec![1, 6969, 6]),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
authors: Some(vec![
"kieran".to_string(),
"snort".to_string(),
"c".to_string(),
"d".to_string(),
"e".to_string(),
]),
since: Some(1),
until: Some(100),
search: None,
limit: None,
},
ReqFilter {
ids: None,
kinds: Some(vec![4]),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
authors: Some(vec!["kieran".to_string()]),
limit: None,
},
ReqFilter {
ids: None,
authors: None,
kinds: Some(vec![4]),
e_tag: None,
p_tag: Some(vec!["kieran".to_string()]),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
ReqFilter {
ids: None,
kinds: Some(vec![1000]),
authors: Some(vec!["snort".to_string()]),
p_tag: Some(vec!["kieran".to_string()]),
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
e_tag: None,
limit: None,
},
];
let expanded = input
.iter()
.flat_map(|v| expand::expand_filter(v))
.sorted_by(|_, _| {
if rand::random() {
Ordering::Less
} else {
Ordering::Greater
}
})
.collect_vec();
let expanded_flat = merge::flat_merge(&expanded);
assert_eq!(expanded_flat, input);
}
}

512
packages/system-query/src/merge.rs Normal file
View File

@ -0,0 +1,512 @@
use crate::{FlatReqFilter, ReqFilter};
use itertools::Itertools;
use std::cmp::Ordering;
pub fn flat_merge(all: &Vec<FlatReqFilter>) -> Vec<ReqFilter> {
let mut ret: Vec<ReqFilter> = vec![];
let merge_sets: Vec<Vec<&FlatReqFilter>> = vec![vec![all.first().unwrap()]];
let merge_sets = all
.iter()
.skip(1)
.sorted_by(|a, b| match distance(&a, &b) {
0 => Ordering::Equal,
1 => Ordering::Less,
_ => Ordering::Greater,
})
.fold(merge_sets, |mut acc, x| {
let mut did_match = false;
for y in acc.iter_mut() {
if y.iter().all(|z| can_merge_filters(z, x)) {
y.push(x);
did_match = true;
break;
}
}
if !did_match {
acc.push(vec![x]);
}
acc
});
for s in merge_sets.iter() {
ret.push(merge_set(s));
}
ret
}
fn merge_set(set: &Vec<&FlatReqFilter>) -> ReqFilter {
let ret = ReqFilter {
ids: None,
authors: None,
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
set.iter().fold(ret, |mut acc, x| {
array_prop_append(&x.id, &mut acc.ids);
array_prop_append(&x.author, &mut acc.authors);
array_prop_append(&x.kind, &mut acc.kinds);
array_prop_append(&x.e_tag, &mut acc.e_tag);
array_prop_append(&x.p_tag, &mut acc.p_tag);
array_prop_append(&x.t_tag, &mut acc.t_tag);
array_prop_append(&x.d_tag, &mut acc.d_tag);
array_prop_append(&x.r_tag, &mut acc.r_tag);
array_prop_append(&x.search, &mut acc.search);
acc.since = x.since;
acc.until = x.until;
acc.limit = x.limit;
acc
})
}
fn can_merge_filters(a: &FlatReqFilter, b: &FlatReqFilter) -> bool {
if a.since != b.since || a.until != b.until || a.limit != b.limit || a.search != b.search {
return false;
}
distance(a, b) <= 1
}
/// Calculate the distance in terms of similarity for merging
///
/// The goal of this function is to find 2 filters which are very similar where
/// one filter may have a single property change like so:
///
/// ```javascript
/// const a = { "kinds": 1, "authors": "a", "since": 99 };
/// const b = { "kinds": 1, "authors": "b", "since": 99 };
/// ```
/// In this case these 2 filters could be merged because their distance is `1`
/// ```javascript
/// const result = { "kinds": [1], "authors": ["a", "b"], "since": 99 };
/// ```
fn distance(a: &FlatReqFilter, b: &FlatReqFilter) -> u32 {
let mut ret = 0u32;
ret += prop_dist(&a.id, &b.id);
ret += prop_dist(&a.kind, &b.kind);
ret += prop_dist(&a.author, &b.author);
ret += prop_dist(&a.e_tag, &b.e_tag);
ret += prop_dist(&a.p_tag, &b.p_tag);
ret += prop_dist(&a.d_tag, &b.d_tag);
ret += prop_dist(&a.r_tag, &b.r_tag);
ret += prop_dist(&a.t_tag, &b.t_tag);
ret += prop_dist(&a.search, &b.search);
ret
}
#[inline(always)]
fn prop_dist<T: Eq>(a: &Option<T>, b: &Option<T>) -> u32 {
if (a.is_some() && b.is_none()) || (a.is_none() && b.is_some()) {
return 10;
} else if a.is_some() && a != b {
return 1;
}
0
}
#[inline(always)]
fn array_prop_append<T: Clone + Eq>(val: &Option<T>, arr: &mut Option<Vec<T>>) {
if let Some(ap) = val {
if arr.is_none() {
*arr = Some(vec![ap.clone()])
} else if !arr.as_ref().unwrap().contains(ap) {
arr.as_mut().unwrap().push(ap.clone());
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn distance() {
let a = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let b = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let c = FlatReqFilter {
id: Some("c".to_owned()),
author: None,
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let d = FlatReqFilter {
id: Some("a".to_owned()),
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
let e = FlatReqFilter {
id: Some("e".to_owned()),
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
};
assert_eq!(super::distance(&a, &b), 0);
assert_eq!(super::distance(&a, &c), 1);
assert_eq!(super::distance(&a, &d), 10);
assert_eq!(super::distance(&a, &e), 11);
}
#[test]
fn merge_set() {
let a = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
};
let b = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
};
let output = ReqFilter {
ids: Some(vec!["0".to_owned()]),
authors: Some(vec!["a".to_owned(), "b".to_owned()]),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
};
assert_eq!(super::merge_set(&vec![&a, &b]), output);
}
#[test]
fn can_merge_filters() {
let a = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
};
let b = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(10),
};
let c = FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
};
assert!(super::can_merge_filters(&a, &b));
assert!(!super::can_merge_filters(&b, &c));
}
#[test]
fn flat_merge() {
let input = vec![
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("a".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("b".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: None,
author: None,
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: None,
author: None,
kind: Some(2),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: None,
author: None,
kind: Some(2),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: Some("0".to_owned()),
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: None,
author: Some("c".to_owned()),
kind: Some(1),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
FlatReqFilter {
id: None,
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
},
FlatReqFilter {
id: Some("1".to_owned()),
author: Some("c".to_owned()),
kind: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
];
let output = vec![
ReqFilter {
ids: Some(vec!["0".to_owned()]),
authors: Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
ReqFilter {
ids: None,
authors: None,
kinds: Some(vec![1, 2]),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
ReqFilter {
ids: None,
authors: Some(vec!["c".to_owned()]),
kinds: Some(vec![1]),
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
ReqFilter {
ids: None,
authors: Some(vec!["c".to_owned()]),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: Some(100),
},
ReqFilter {
ids: Some(vec!["1".to_owned()]),
authors: Some(vec!["c".to_owned()]),
kinds: None,
e_tag: None,
p_tag: None,
t_tag: None,
d_tag: None,
r_tag: None,
search: None,
since: None,
until: None,
limit: None,
},
];
assert_eq!(super::flat_merge(&input), output)
}
}

12
packages/system-query/system-query.iml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="RUST_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
packages/system/src/nostr-system.ts
View File

@ -241,7 +241,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
return existing;
}
const filters = !req.options?.skipDiff
? req.buildDiff(this.#relayCache, existing.flatFilters)
? req.buildDiff(this.#relayCache, existing.filters)
: req.build(this.#relayCache);
if (filters.length === 0 && !!req.options?.skipDiff) {
return existing;

10
packages/system/src/query.ts
View File

@ -4,9 +4,7 @@ import { unixNowMs, unwrap } from "@snort/shared";
import { Connection, ReqFilter, Nips, TaggedNostrEvent } from ".";
import { NoteStore } from "./note-collection";
import { flatMerge } from "./request-merger";
import { BuiltRawReqFilter } from "./request-builder";
import { FlatReqFilter, expandFilter } from "./request-expander";
import { eventMatchesFilter } from "./request-matcher";
/**
@ -19,7 +17,6 @@ class QueryTrace {
eose?: number;
close?: number;
#wasForceClosed = false;
readonly flatFilters: Array<FlatReqFilter>;
readonly #fnClose: (id: string) => void;
readonly #fnProgress: () => void;
@ -34,7 +31,6 @@ class QueryTrace {
this.start = unixNowMs();
this.#fnClose = fnClose;
this.#fnProgress = fnProgress;
this.flatFilters = filters.flatMap(expandFilter);
}
sentToRelay() {
@ -166,11 +162,7 @@ export class Query implements QueryBase {
* Recompute the complete set of compressed filters from all query traces
*/
get filters() {
return flatMerge(this.flatFilters);
}
get flatFilters() {
return this.#tracing.flatMap(a => a.flatFilters);
return this.#tracing.flatMap(a => a.filters);
}
get feed() {

15
packages/system/src/request-builder.ts
View File

@ -1,12 +1,12 @@
import debug from "debug";
import { v4 as uuid } from "uuid";
import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared";
import { get_diff }from "@snort/system-query";
import { ReqFilter, u256, HexKey, EventKind } from ".";
import { diffFilters } from "./request-splitter";
import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model";
import { flatMerge, mergeSimilar } from "./request-merger";
import { FlatReqFilter, expandFilter } from "./request-expander";
import { FlatReqFilter } from "./request-expander";
/**
* Which strategy is used when building REQ filters
@ -103,15 +103,16 @@ export class RequestBuilder {
/**
* Detects a change in request from a previous set of filters
*/
buildDiff(relays: RelayCache, prev: Array<FlatReqFilter>): Array<BuiltRawReqFilter> {
buildDiff(relays: RelayCache, prev: Array<ReqFilter>): Array<BuiltRawReqFilter> {
const start = unixNowMs();
const next = this.#builders.flatMap(f => expandFilter(f.filter));
const diff = diffFilters(prev, next);
//const next = this.#builders.flatMap(f => expandFilter(f.filter));
//const diff = diffFilters(prev, next);
const diff = get_diff(prev, this.buildRaw()) as Array<FlatReqFilter>;
const ts = unixNowMs() - start;
this.#log("buildDiff %s %d ms", this.id, ts);
if (diff.changed) {
return splitFlatByWriteRelays(relays, diff.added).map(a => {
if (diff.length > 0) {
return splitFlatByWriteRelays(relays, diff).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
filters: flatMerge(a.filters),