feat: tools pages

Various other fixes:
- Better handeling of limit/since/before merging
- Expose timeout through request builder
- Expose PickN through request builder
- Fix tests

packages/system/src/query-optimizer/index.ts

@@ -18,6 +18,7 @@ export interface FlatReqFilter {
   since?: number;
   until?: number;
   limit?: number;
+  resultSetId: string;
 }
 
 export interface QueryOptimizer {

packages/system/src/query-optimizer/request-expander.ts

@@ -1,3 +1,4 @@
+import { sha256 } from "@snort/shared";
 import { FlatReqFilter } from ".";
 import { ReqFilter } from "../nostr";
 
@@ -7,29 +8,52 @@ import { ReqFilter } from "../nostr";
 export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
   const ret: Array<FlatReqFilter> = [];
   const src = Object.entries(f);
-  const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]);
-  const props = src.filter(([, v]) => !Array.isArray(v));
 
-  function generateCombinations(index: number, currentCombination: FlatReqFilter) {
-    if (index === keys.length) {
-      ret.push(currentCombination);
+  const id = resultSetId(f);
+
+  // Filter entries that are arrays and keep the rest as is
+  const arrays: [string, Array<string> | Array<number>][] = src.filter(([, value]) => Array.isArray(value)) as [
+    string,
+    Array<string> | Array<number>,
+  ][];
+  const constants = Object.fromEntries(src.filter(([, value]) => !Array.isArray(value))) as {
+    [key: string]: string | number | undefined;
+  };
+
+  // Recursive function to compute cartesian product
+  function cartesianProduct(arr: [string, Array<string> | Array<number>][], temp: [string, any][] = []) {
+    if (arr.length === 0) {
+      ret.push(createFilterObject(temp, constants, id));
       return;
     }
-
-    const key = keys[index];
-    const values = (f as Record<string, Array<string | number>>)[key];
-
-    for (let i = 0; i < values.length; i++) {
-      const value = values[i];
-      const updatedCombination = { ...currentCombination, [key]: value };
-      generateCombinations(index + 1, updatedCombination);
+    for (let i = 0; i < arr[0][1].length; i++) {
+      cartesianProduct(arr.slice(1), temp.concat([[arr[0][0], arr[0][1][i]]]));
     }
   }
 
-  generateCombinations(0, {
-    keys: keys.length,
-    ...Object.fromEntries(props),
-  });
+  // Create filter object from the combination
+  function createFilterObject(
+    combination: [string, any][],
+    constants: { [key: string]: string | number | undefined },
+    resultId: string,
+  ) {
+    let filterObject = { ...Object.fromEntries(combination), ...constants } as FlatReqFilter;
+    filterObject.resultSetId = resultId;
+    return filterObject;
+  }
 
+  cartesianProduct(arrays);
   return ret;
 }
+
+function resultSetId(f: ReqFilter) {
+  if (f.limit !== undefined || f.since !== undefined || f.until !== undefined) {
+    const arrays = Object.entries(f)
+      .filter(([, a]) => Array.isArray(a))
+      .map(a => a as [string, Array<string | number>])
+      .sort();
+    const input = arrays.map(([, a]) => a.join(",")).join(",");
+    return sha256(input);
+  }
+  return "";
+}

packages/system/src/query-optimizer/request-merger.ts

@@ -2,18 +2,9 @@ import { distance } from "@snort/shared";
 import { ReqFilter } from "..";
 import { FlatReqFilter } from ".";
 
-/**
- * Keys which can change the entire meaning of the filter outside the array types
- */
-const DiscriminatorKeys = ["since", "until", "limit", "search"];
-
 export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
-  const aObj = a as Record<string, string | number | undefined>;
-  const bObj = b as Record<string, string | number | undefined>;
-  for (const key of DiscriminatorKeys) {
-    if (aObj[key] !== bObj[key]) {
-      return false;
-    }
+  if (a.resultSetId !== b.resultSetId) {
+    return false;
   }
   return distance(a, b) <= 1;
 }
@@ -101,12 +92,11 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
 
   // to compute filters which can be merged we need to calucate the distance change between each filter
   // then we can merge filters which are exactly 1 change diff from each other
-
   function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
     return filters.reduce((acc, a) => {
       Object.entries(a).forEach(([k, v]) => {
-        if (k === "keys" || v === undefined) return;
-        if (DiscriminatorKeys.includes(k)) {
+        if (v === undefined) return;
+        if (k === "since" || k === "until" || k === "limit" || k === "search" || k === "resultSetId") {
           acc[k] = v;
         } else {
           acc[k] ??= [];
@@ -142,5 +132,6 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
     }
     ret = n;
   }
+  ret.forEach(a => delete a["resultSetId"]);
   return ret;
 }