optimize

packages/system/jest.config.js

@@ -3,7 +3,7 @@ module.exports = {
   bail: true,
   preset: "ts-jest",
   testEnvironment: "jsdom",
-  roots: ["src"],
+  roots: ["src", "tests"],
-  moduleDirectories: ["src"],
+  moduleDirectories: ["src", "node_modules"],
   setupFiles: ["./tests/setupTests.ts"],
 };

packages/system/src/Connection.ts

@@ -4,7 +4,7 @@ import { DefaultConnectTimeout } from "./Const";
 import { ConnectionStats } from "./ConnectionStats";
 import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
 import { RelayInfo } from "./RelayInfo";
-import { unwrap } from "./Util";
+import { unwrap } from "./Utils";
 import ExternalStore from "./ExternalStore";
 
 export type AuthHandler = (challenge: string, relay: string) => Promise<NostrEvent | undefined>;

packages/system/src/EventBuilder.ts

@@ -1,6 +1,6 @@
 import { EventKind, HexKey, NostrPrefix, NostrEvent } from ".";
 import { HashtagRegex } from "./Const";
-import { getPublicKey, unixNow } from "./Util";
+import { getPublicKey, unixNow } from "./Utils";
 import { EventExt } from "./EventExt";
 import { parseNostrLink } from "./NostrLink";
 

packages/system/src/EventExt.ts

@@ -2,7 +2,7 @@ import * as secp from "@noble/curves/secp256k1";
 import * as utils from "@noble/curves/abstract/utils";
 import { EventKind, HexKey, NostrEvent, Tag } from ".";
 import base64 from "@protobufjs/base64";
-import { sha256, unixNow } from "./Util";
+import { sha256, unixNow } from "./Utils";
 
 export interface Thread {
   root?: Tag;

packages/system/src/EventPublisher.ts

@@ -13,7 +13,7 @@ import {
   UserMetadata,
 } from ".";
 
-import { unwrap } from "./Util";
+import { unwrap } from "./Utils";
 import { EventBuilder } from "./EventBuilder";
 import { EventExt } from "./EventExt";
 import { barrierQueue, processWorkQueue, WorkQueueItem } from "./WorkQueue";

packages/system/src/GossipModel.ts

@@ -1,5 +1,5 @@
 import { FullRelaySettings, ReqFilter } from ".";
-import { unwrap } from "./Util";
+import { unwrap } from "./Utils";
 import debug from "debug";
 
 const PickNRelays = 2;

packages/system/src/NostrLink.ts

@@ -1,4 +1,4 @@
-import { bech32ToHex, hexToBech32 } from "./Util";
+import { bech32ToHex, hexToBech32 } from "./Utils";
 import { NostrPrefix, decodeTLV, TLVEntryType } from ".";
 
 export interface NostrLink {

packages/system/src/NostrSystem.ts

@@ -7,7 +7,7 @@ import { Query } from "./Query";
 import { RelayCache } from "./GossipModel";
 import { NoteStore } from "./NoteCollection";
 import { BuiltRawReqFilter, RequestBuilder } from "./RequestBuilder";
-import { unwrap, sanitizeRelayUrl } from "./Util";
+import { unwrap, sanitizeRelayUrl } from "./Utils";
 import { SystemInterface, SystemSnapshot } from ".";
 
 /**
@@ -122,15 +122,13 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
     const existing = this.Queries.get(req.id);
     if (existing) {
       const filters = !req.options?.skipDiff
-        ? req.buildDiff(this.#relayCache, existing.filters)
+        ? req.buildDiff(this.#relayCache, existing.flatFilters)
         : req.build(this.#relayCache);
       if (filters.length === 0 && !!req.options?.skipDiff) {
         return existing;
       } else {
         for (const subQ of filters) {
-          this.SendQuery(existing, subQ).then(qta =>
+          this.SendQuery(existing, subQ);
-            qta.forEach(v => this.#log("New QT from diff %s %s %O from: %O", req.id, v.id, v.filters, existing.filters))
-          );
         }
         this.notifyChange();
         return existing;
@@ -142,9 +140,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
       const q = new Query(req.id, store, req.options?.leaveOpen);
       this.Queries.set(req.id, q);
       for (const subQ of filters) {
-        this.SendQuery(q, subQ).then(qta =>
+        this.SendQuery(q, subQ);
-          qta.forEach(v => this.#log("New QT from diff %s %s %O", req.id, v.id, v.filters))
-        );
       }
       this.notifyChange();
       return q;

packages/system/src/NoteCollection.ts

@@ -1,5 +1,5 @@
 import { TaggedRawEvent, u256 } from ".";
-import { appendDedupe, findTag } from "./Util";
+import { appendDedupe, findTag } from "./Utils";
 
 export interface StoreSnapshot<TSnapshot> {
   data: TSnapshot | undefined;

packages/system/src/ProfileCache.ts

@@ -1,7 +1,7 @@
 import { EventKind, HexKey, SystemInterface, TaggedRawEvent, PubkeyReplaceableNoteStore, RequestBuilder } from ".";
 import { ProfileCacheExpire } from "./Const";
 import { CacheStore, mapEventToProfile, MetadataCache } from "./cache";
-import { unixNowMs } from "./Util";
+import { unixNowMs } from "./Utils";
 import debug from "debug";
 
 export class ProfileLoaderService {

packages/system/src/Query.ts

@@ -1,7 +1,7 @@
 import { v4 as uuid } from "uuid";
 import debug from "debug";
 import { Connection, ReqFilter, Nips, TaggedRawEvent } from ".";
-import { unixNowMs, unwrap } from "./Util";
+import { unixNowMs, unwrap } from "./Utils";
 import { NoteStore } from "./NoteCollection";
 import { flatMerge } from "./RequestMerger";
 import { BuiltRawReqFilter } from "./RequestBuilder";
@@ -137,7 +137,6 @@ export class Query implements QueryBase {
   #feed: NoteStore;
 
   #log = debug("Query");
-  #allFilters: Array<ReqFilter> = [];
 
   constructor(id: string, feed: NoteStore, leaveOpen?: boolean) {
     this.id = id;
@@ -154,7 +153,11 @@ export class Query implements QueryBase {
    * Recompute the complete set of compressed filters from all query traces
    */
   get filters() {
-    return this.#allFilters;
+    return flatMerge(this.flatFilters);
+  }
+
+  get flatFilters() {
+    return this.#tracing.flatMap(a => a.filters).flatMap(expandFilter);
   }
 
   get feed() {
@@ -271,14 +274,7 @@ export class Query implements QueryBase {
       () => this.#onProgress()
     );
     this.#tracing.push(qt);
-    this.#reComputeFilters();
     c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
     return qt;
   }
-
-  #reComputeFilters() {
-    console.time("reComputeFilters");
-    this.#allFilters = flatMerge(this.#tracing.flatMap(a => a.filters).flatMap(expandFilter));
-    console.timeEnd("reComputeFilters");
-  }
 }

packages/system/src/RequestBuilder.ts

@@ -1,8 +1,10 @@
 import { ReqFilter, u256, HexKey, EventKind } from ".";
-import { appendDedupe, dedupe } from "./Util";
+import { appendDedupe, dedupe, unixNowMs } from "./Utils";
 import { diffFilters } from "./RequestSplitter";
 import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel";
 import { mergeSimilar } from "./RequestMerger";
+import { FlatReqFilter, expandFilter } from "./RequestExpander";
+import debug from "debug";
 
 /**
  * Which strategy is used when building REQ filters
@@ -92,10 +94,18 @@ export class RequestBuilder {
    * @param q All previous filters merged
    * @returns
    */
-  buildDiff(relays: RelayCache, filters: Array<ReqFilter>): Array<BuiltRawReqFilter> {
+  buildDiff(relays: RelayCache, filters: Array<FlatReqFilter>): Array<BuiltRawReqFilter> {
-    const next = this.buildRaw();
+    const start = unixNowMs();
+    const next = this.#builders.flatMap(f => expandFilter(f.filter))
     const diff = diffFilters(filters, next);
+    const ts = (unixNowMs() - start);
+    const log = debug("buildDiff");
+    log("%s %d ms", this.id, ts);
+    if (ts > 200) {
+      console.warn(diff, filters);
+    }
     if (diff.changed) {
+      log(diff);
       return splitAllByWriteRelays(relays, diff.added).map(a => {
         return {
           strategy: RequestStrategy.AuthorsRelays,

packages/system/src/RequestMerger.ts

@@ -1,3 +1,4 @@
+import { distance } from "./Utils";
 import { ReqFilter } from ".";
 import { FlatReqFilter } from "./RequestExpander";
 
@@ -16,74 +17,10 @@ export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter |
       }
     }
   }
-  let flag = false;
+  return distance(a, b) <= 1;
-  if (!equalProp(a.ids, b.ids)) {
-    flag = true;
-  }
-  if (!equalProp(a.authors, b.authors)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a.kinds, b.kinds)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a.limit, b.limit)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a.until, b.until)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a.since, b.since)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a.search, b.search)) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a["#e"], b["#e"])) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a["#p"], b["#p"])) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a["#d"], b["#d"])) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a["#r"], b["#r"])) {
-    if (flag) return false;
-    flag = true;
-  }
-  if (!equalProp(a["#t"], b["#t"])) {
-    if (flag) return false;
-    flag = true;
-  }
-  return true;
-}
-
-function equalProp(a: string | number | Array<string | number> | undefined, b: string | number | Array<string | number> | undefined) {
-  if ((a !== undefined && b === undefined) || (a === undefined && b !== undefined)) {
-    return false;
-  }
-  if (Array.isArray(a) && Array.isArray(b)) {
-    if (a.length !== b.length) {
-      return false;
-    }
-    if (!a.every(v => b.includes(v))) {
-      return false;
-    }
-  }
-  return a === b;
 }
 
 export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
-  console.time("mergeSimilar");
   const ret = [];
 
   const fCopy = [...filters];
@@ -92,14 +29,13 @@ export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
     const mergeSet = [current];
     for (let i = 0; i < fCopy.length; i++) {
       const f = fCopy[i];
-      if (mergeSet.every(v => canMergeFilters(v, f))) {
+      if (!mergeSet.some(v => !canMergeFilters(v, f))) {
         mergeSet.push(fCopy.splice(i, 1)[0]);
         i--;
       }
     }
     ret.push(simpleMerge(mergeSet));
   }
-  console.timeEnd("mergeSimilar");
   return ret;
 }
 
@@ -117,7 +53,8 @@ export function simpleMerge(filters: Array<ReqFilter>) {
         if (result[key] === undefined) {
           result[key] = [...value];
         } else {
-          result[key] = [...new Set([...result[key], ...value])];
+          const toAdd = value.filter(a => !result[key].includes(a));
+          result[key].push(...toAdd);
         }
       } else {
         result[key] = value;
@@ -162,31 +99,25 @@ export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
  * @returns
  */
 export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
-  console.time("flatMerge");
   let ret: Array<ReqFilter> = [];
 
   // to compute filters which can be merged we need to calucate the distance change between each filter
   // then we can merge filters which are exactly 1 change diff from each other
 
   function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
-    const result: any = {};
+    return filters.reduce((acc, a) => {
-
+      Object.entries(a).forEach(([k, v]) => {
-    filters.forEach(f => {
+        if (DiscriminatorKeys.includes(k)) {
-      const filter = f as Record<string, string | number>;
+          acc[k] = v;
-      Object.entries(filter).forEach(([key, value]) => {
-        if (!DiscriminatorKeys.includes(key)) {
-          if (result[key] === undefined) {
-            result[key] = [value];
-          } else {
-            result[key] = [...new Set([...result[key], value])];
-          }
         } else {
-          result[key] = value;
+          acc[k] ??= [];
+          if (!acc[k].includes(v)) {
+            acc[k].push(v);
+          }
         }
-      });
+      })
-    });
+      return acc;
-
+    }, {} as any) as ReqFilter;
-    return result as ReqFilter;
   }
 
   // reducer, kinda verbose
@@ -212,7 +143,5 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
     }
     ret = n;
   }
-  console.timeEnd("flatMerge");
-  console.debug(ret);
   return ret;
 }

packages/system/src/RequestSplitter.ts

@@ -1,18 +1,15 @@
-import { ReqFilter } from ".";
+import { flatFilterEq } from "./Utils";
-import { flatReqFilterEq } from "./Util";
+import { FlatReqFilter } from "./RequestExpander";
-import { expandFilter } from "./RequestExpander";
 import { flatMerge } from "./RequestMerger";
 
-export function diffFilters(prev: Array<ReqFilter>, next: Array<ReqFilter>) {
+export function diffFilters(prev: Array<FlatReqFilter>, next: Array<FlatReqFilter>, calcRemoved?: boolean) {
-  const prevExpanded = prev.flatMap(expandFilter);
+  const added = next.filter(a => !prev.some(b => flatFilterEq(a, b)));
-  const nextExpanded = next.flatMap(expandFilter);
+  const removed = calcRemoved ? prev.filter(a => !next.some(b => flatFilterEq(a, b))) : [];
-
-  const added = flatMerge(nextExpanded.filter(a => !prevExpanded.some(b => flatReqFilterEq(a, b))));
-  const removed = flatMerge(prevExpanded.filter(a => !nextExpanded.some(b => flatReqFilterEq(a, b))));
 
+  const changed = added.length > 0 || removed.length > 0;
   return {
-    added,
+    added: changed ? flatMerge(added) : [],
-    removed,
+    removed: changed ? flatMerge(removed) : [],
-    changed: added.length > 0 || removed.length > 0,
+    changed,
   };
 }

packages/system/src/Tag.ts

@@ -1,5 +1,5 @@
 import { HexKey, u256 } from "./Nostr";
-import { unwrap } from "./Util";
+import { unwrap } from "./Utils";
 
 export default class Tag {
   Original: string[];

packages/system/src/Utils.ts

@@ -2,7 +2,7 @@ import * as utils from "@noble/curves/abstract/utils";
 import * as secp from "@noble/curves/secp256k1";
 import { sha256 as sha2 } from "@noble/hashes/sha256";
 import { bech32 } from "bech32";
-import { NostrEvent, u256 } from "./Nostr";
+import { NostrEvent, ReqFilter, u256 } from "./Nostr";
 import { FlatReqFilter } from "RequestExpander";
 
 export function unwrap<T>(v: T | undefined | null): T {
@@ -55,7 +55,22 @@ export function deepEqual(x: any, y: any): boolean {
     : x === y;
 }
 
-export function flatReqFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
+export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
+  return equalProp(a.ids, b.ids)
+    && equalProp(a.kinds, b.kinds)
+    && equalProp(a.authors, b.authors)
+    && equalProp(a.limit, b.limit)
+    && equalProp(a.since, b.since)
+    && equalProp(a.until, b.until)
+    && equalProp(a.search, b.search)
+    && equalProp(a["#e"], b["#e"])
+    && equalProp(a["#p"], b["#p"])
+    && equalProp(a["#t"], b["#t"])
+    && equalProp(a["#d"], b["#d"])
+    && equalProp(a["#r"], b["#r"]);
+}
+
+export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
   return a.ids === b.ids
     && a.kinds === b.kinds
     && a.authors === b.authors
@@ -70,6 +85,55 @@ export function flatReqFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
     && a["#r"] === b["#r"];
 }
 
+export function equalProp(a: string | number | Array<string | number> | undefined, b: string | number | Array<string | number> | undefined) {
+  if ((a !== undefined && b === undefined) || (a === undefined && b !== undefined)) {
+    return false;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      return false;
+    }
+    if (!a.every(v => b.includes(v))) {
+      return false;
+    }
+  }
+  return a === b;
+}
+
+/**
+ * Compute the "distance" between two objects by comparing their difference in properties
+ * Missing/Added keys result in +10 distance
+ * This is not recursive
+ */
+export function distance(a: any, b: any): number {
+  const keys1 = Object.keys(a);
+  const keys2 = Object.keys(b);
+  const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
+
+  let distance = 0;
+  for (const key of maxKeys) {
+    if (key in a && key in b) {
+      if (Array.isArray(a[key]) && Array.isArray(b[key])) {
+        const aa = a[key] as Array<string | number>;
+        const bb = b[key] as Array<string | number>;
+        if (aa.length === bb.length) {
+          if (aa.some(v => !bb.includes(v))) {
+            distance++;
+          }
+        } else {
+          distance++;
+        }
+      } else if (a[key] !== b[key]) {
+        distance++;
+      }
+    } else {
+      distance += 10;
+    }
+  }
+
+  return distance;
+}
+
 export function dedupe<T>(v: Array<T>) {
   return [...new Set(v)];
 }

packages/system/src/cache/index.ts

@@ -1,5 +1,5 @@
 import { HexKey, NostrEvent, UserMetadata } from "..";
-import { hexToBech32, unixNowMs } from "../Util";
+import { hexToBech32, unixNowMs } from "../Utils";
 
 export interface MetadataCache extends UserMetadata {
   /**
@@ -36,13 +36,21 @@ export interface MetadataCache extends UserMetadata {
 export function mapEventToProfile(ev: NostrEvent) {
   try {
     const data: UserMetadata = JSON.parse(ev.content);
-    return {
+    let ret = {
       ...data,
       pubkey: ev.pubkey,
       npub: hexToBech32("npub", ev.pubkey),
       created: ev.created_at,
       loaded: unixNowMs(),
     } as MetadataCache;
+
+    // sanitize non-string/number
+    for (const [k, v] of Object.entries(ret)) {
+      if (typeof v !== "number" && typeof v !== "string") {
+        (ret as any)[k] = undefined;
+      }
+    }
+    return ret;
   } catch (e) {
     console.error("Failed to parse JSON", ev, e);
   }

packages/system/tests/RequestBuilder.test.ts

@@ -1,6 +1,7 @@
 import { RelayCache } from "../src/GossipModel";
 import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder";
 import { describe, expect } from "@jest/globals";
+import { expandFilter } from "../src/RequestExpander";
 
 const DummyCache = {
   get: (pk?: string) => {
@@ -88,7 +89,7 @@ describe("RequestBuilder", () => {
     f0.authors(["a"]);
     expect(a).toEqual([{}]);
 
-    const b = rb.buildDiff(DummyCache, a);
+    const b = rb.buildDiff(DummyCache, a.flatMap(expandFilter));
     expect(b).toMatchObject([
       {
         filters: [{ authors: ["a"] }],

packages/system/tests/RequestMerger.test.ts

@@ -1,7 +1,6 @@
 import { ReqFilter } from "../src";
-import { filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/RequestMerger";
+import { canMergeFilters, filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/RequestMerger";
 import { FlatReqFilter, expandFilter } from "../src/RequestExpander";
-import { distance } from "../src/Util";
 
 describe("RequestMerger", () => {
   it("should simple merge authors", () => {
@@ -105,6 +104,77 @@ describe("flatMerge", () => {
     ] as Array<ReqFilter>;
 
     const dut = flatMerge(input.flatMap(expandFilter).sort(() => (Math.random() > 0.5 ? 1 : -1)));
-    expect(dut.every(a => input.some(b => distance(b, a) === 0))).toEqual(true);
+    expect(dut.every(a => input.some(b => canMergeFilters(b, a) === false))).toEqual(true);
   });
 });
+
+describe('canMerge', () => {
+  it("should have 0 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "a",
+    };
+    expect(canMergeFilters(a, b)).toEqual(true);
+  });
+  it("should have 1 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "b",
+    };
+    expect(canMergeFilters(a, b)).toEqual(true);
+  });
+  it("should have 10 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "a",
+      kinds: 1,
+    };
+    expect(canMergeFilters(a, b)).toEqual(false);
+  });
+  it("should have 11 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "b",
+      kinds: 1,
+    };
+    expect(canMergeFilters(a, b)).toEqual(false);
+  });
+  it("should have 1 distance, arrays", () => {
+    const a = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    const b = {
+      since: 1,
+      until: 100,
+      kinds: [6969],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    expect(canMergeFilters(a, b)).toEqual(true);
+  });
+  it("should have 1 distance, array change extra", () => {
+    const a = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["f", "kieran", "snort", "c", "d"],
+    };
+    const b = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    expect(canMergeFilters(a, b)).toEqual(true);
+  });
+})

packages/system/tests/RequestSplitter.test.ts

@@ -1,12 +1,13 @@
 import { ReqFilter } from "../src";
 import { describe, expect } from "@jest/globals";
 import { diffFilters } from "../src/RequestSplitter";
+import { expandFilter } from "../src/RequestExpander";
 
 describe("RequestSplitter", () => {
   test("single filter add value", () => {
     const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"] }];
     const b: Array<ReqFilter> = [{ kinds: [0], authors: ["a", "b"] }];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [{ kinds: [0], authors: ["b"] }],
       removed: [],
@@ -16,7 +17,7 @@ describe("RequestSplitter", () => {
   test("single filter remove value", () => {
     const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"] }];
     const b: Array<ReqFilter> = [{ kinds: [0], authors: ["b"] }];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [{ kinds: [0], authors: ["b"] }],
       removed: [{ kinds: [0], authors: ["a"] }],
@@ -26,7 +27,7 @@ describe("RequestSplitter", () => {
   test("single filter change critical key", () => {
     const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"], since: 100 }];
     const b: Array<ReqFilter> = [{ kinds: [0], authors: ["a", "b"], since: 101 }];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [{ kinds: [0], authors: ["a", "b"], since: 101 }],
       removed: [{ kinds: [0], authors: ["a"], since: 100 }],
@@ -42,7 +43,7 @@ describe("RequestSplitter", () => {
       { kinds: [0], authors: ["a", "b"] },
       { kinds: [69], authors: ["a", "c"] },
     ];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [
         { kinds: [0], authors: ["b"] },
@@ -61,7 +62,7 @@ describe("RequestSplitter", () => {
       { kinds: [0], authors: ["b"] },
       { kinds: [69], authors: ["c"] },
     ];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [
         { kinds: [0], authors: ["b"] },
@@ -77,7 +78,7 @@ describe("RequestSplitter", () => {
       { kinds: [0], authors: ["a"] },
       { kinds: [69], authors: ["c"] },
     ];
-    const diff = diffFilters(a, b);
+    const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
     expect(diff).toEqual({
       added: [{ kinds: [69], authors: ["c"] }],
       removed: [],

packages/system/tests/Util.test.ts

@@ -1,76 +1,5 @@
-import { distance } from "../src/Util";
+import { NostrPrefix } from "../src/Links";
-
+import { parseNostrLink, tryParseNostrLink } from "../src/NostrLink";
-describe("distance", () => {
-  it("should have 0 distance", () => {
-    const a = {
-      ids: "a",
-    };
-    const b = {
-      ids: "a",
-    };
-    expect(distance(a, b)).toEqual(0);
-  });
-  it("should have 1 distance", () => {
-    const a = {
-      ids: "a",
-    };
-    const b = {
-      ids: "b",
-    };
-    expect(distance(a, b)).toEqual(1);
-  });
-  it("should have 10 distance", () => {
-    const a = {
-      ids: "a",
-    };
-    const b = {
-      ids: "a",
-      kinds: 1,
-    };
-    expect(distance(a, b)).toEqual(10);
-  });
-  it("should have 11 distance", () => {
-    const a = {
-      ids: "a",
-    };
-    const b = {
-      ids: "b",
-      kinds: 1,
-    };
-    expect(distance(a, b)).toEqual(11);
-  });
-  it("should have 1 distance, arrays", () => {
-    const a = {
-      since: 1,
-      until: 100,
-      kinds: [1],
-      authors: ["kieran", "snort", "c", "d", "e"],
-    };
-    const b = {
-      since: 1,
-      until: 100,
-      kinds: [6969],
-      authors: ["kieran", "snort", "c", "d", "e"],
-    };
-    expect(distance(a, b)).toEqual(1);
-  });
-  it("should have 1 distance, array change extra", () => {
-    const a = {
-      since: 1,
-      until: 100,
-      kinds: [1],
-      authors: ["f", "kieran", "snort", "c", "d"],
-    };
-    const b = {
-      since: 1,
-      until: 100,
-      kinds: [1],
-      authors: ["kieran", "snort", "c", "d", "e"],
-    };
-    expect(distance(a, b)).toEqual(1);
-  });
-});
-
 
 describe("tryParseNostrLink", () => {
   it("is a valid nostr link", () => {

packages/system/tsconfig.json

@@ -9,10 +9,10 @@
     "strict": true,
     "declaration": true,
     "declarationMap": true,
-    "sourceMap": true,
+    "inlineSourceMap": true,
     "outDir": "dist",
     "skipLibCheck": true
   },
-  "include": ["src"],
+  "include": ["src/**/*.ts"],
   "files": ["src/index.ts"]
 }