improve diff filters

fix tests
expander/compressor filter mangler

packages/app/src/Cache/DMCache.ts

@@ -1,13 +1,13 @@
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 import { db } from "Db";
 import FeedCache from "./FeedCache";
 
-class DMCache extends FeedCache<RawEvent> {
+class DMCache extends FeedCache<NostrEvent> {
   constructor() {
     super("DMCache", db.dms);
   }
 
-  key(of: RawEvent): string {
+  key(of: NostrEvent): string {
     return of.id;
   }
 
@@ -23,11 +23,11 @@ class DMCache extends FeedCache<RawEvent> {
     return ret;
   }
 
-  allDms(): Array<RawEvent> {
+  allDms(): Array<NostrEvent> {
     return [...this.cache.values()];
   }
 
-  takeSnapshot(): Array<RawEvent> {
+  takeSnapshot(): Array<NostrEvent> {
     return this.allDms();
   }
 }

packages/app/src/Cache/index.ts

@@ -1,4 +1,4 @@
-import { HexKey, RawEvent, UserMetadata } from "System";
+import { HexKey, NostrEvent, UserMetadata } from "System";
 import { hexToBech32, unixNowMs } from "SnortUtils";
 import { DmCache } from "./DMCache";
 import { InteractionCache } from "./EventInteractionCache";
@@ -37,7 +37,7 @@ export interface MetadataCache extends UserMetadata {
   isNostrAddressValid: boolean;
 }
 
-export function mapEventToProfile(ev: RawEvent) {
+export function mapEventToProfile(ev: NostrEvent) {
   try {
     const data: UserMetadata = JSON.parse(ev.content);
     return {

packages/app/src/Db/index.ts

@@ -1,5 +1,5 @@
 import Dexie, { Table } from "dexie";
-import { FullRelaySettings, HexKey, RawEvent, u256 } from "System";
+import { FullRelaySettings, HexKey, NostrEvent, u256 } from "System";
 import { MetadataCache } from "Cache";
 
 export const NAME = "snortDB";
@@ -48,8 +48,8 @@ export class SnortDB extends Dexie {
   users!: Table<MetadataCache>;
   relayMetrics!: Table<RelayMetrics>;
   userRelays!: Table<UsersRelays>;
-  events!: Table<RawEvent>;
+  events!: Table<NostrEvent>;
-  dms!: Table<RawEvent>;
+  dms!: Table<NostrEvent>;
   eventInteraction!: Table<EventInteraction>;
 
   constructor() {

packages/app/src/Element/NostrFileHeader.tsx

@@ -1,5 +1,5 @@
 import { FormattedMessage } from "react-intl";
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 
 import { findTag, NostrLink } from "SnortUtils";
 import useEventFeed from "Feed/EventFeed";
@@ -14,7 +14,7 @@ export default function NostrFileHeader({ link }: { link: NostrLink }) {
   return <NostrFileElement ev={ev.data} />;
 }
 
-export function NostrFileElement({ ev }: { ev: RawEvent }) {
+export function NostrFileElement({ ev }: { ev: NostrEvent }) {
   // assume image or embed which can be rendered by the hypertext kind
   // todo: make use of hash
   // todo: use magnet or other links if present

packages/app/src/Element/NoteReaction.tsx

@@ -1,7 +1,7 @@
 import "./NoteReaction.css";
 import { Link } from "react-router-dom";
 import { useMemo } from "react";
-import { EventKind, RawEvent, TaggedRawEvent, NostrPrefix } from "System";
+import { EventKind, NostrEvent, TaggedRawEvent, NostrPrefix } from "System";
 
 import Note from "Element/Note";
 import ProfileImage from "Element/ProfileImage";
@@ -43,7 +43,7 @@ export default function NoteReaction(props: NoteReactionProps) {
   function extractRoot() {
     if (ev?.kind === EventKind.Repost && ev.content.length > 0 && ev.content !== "#[0]") {
       try {
-        const r: RawEvent = JSON.parse(ev.content);
+        const r: NostrEvent = JSON.parse(ev.content);
         return r as TaggedRawEvent;
       } catch (e) {
         console.error("Could not load reposted content", e);

packages/app/src/Element/PubkeyList.tsx

@@ -1,8 +1,8 @@
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 import { dedupe } from "SnortUtils";
 import FollowListBase from "./FollowListBase";
 
-export default function PubkeyList({ ev, className }: { ev: RawEvent; className?: string }) {
+export default function PubkeyList({ ev, className }: { ev: NostrEvent; className?: string }) {
   const ids = dedupe(ev.tags.filter(a => a[0] === "p").map(a => a[1]));
   return <FollowListBase pubkeys={ids} showAbout={true} className={className} />;
 }

packages/app/src/Element/SendSats.tsx

@@ -2,7 +2,7 @@ import "./SendSats.css";
 import React, { useEffect, useMemo, useState } from "react";
 import { useIntl, FormattedMessage } from "react-intl";
 
-import { HexKey, RawEvent } from "System";
+import { HexKey, NostrEvent } from "System";
 import { System } from "index";
 import { formatShort } from "Number";
 import Icon from "Icons/Icon";
@@ -125,7 +125,7 @@ export default function SendSats(props: SendSatsProps) {
   async function loadInvoice() {
     if (!amount || !handler || !publisher) return null;
 
-    let zap: RawEvent | undefined;
+    let zap: NostrEvent | undefined;
     if (author && zapType !== ZapType.NonZap) {
       const relays = Object.keys(login.relays.item);
 

packages/app/src/Element/SubDebug.tsx

@@ -5,7 +5,7 @@ import useRelayState from "Feed/RelayState";
 import Tabs, { Tab } from "Element/Tabs";
 import { unwrap } from "SnortUtils";
 import useSystemState from "Hooks/useSystemState";
-import { RawReqFilter } from "System";
+import { ReqFilter } from "System";
 import { useCopy } from "useCopy";
 import { System } from "index";
 
@@ -18,7 +18,7 @@ function Queries() {
   const qs = useSystemState();
   const { copy } = useCopy();
 
-  function countElements(filters: Array<RawReqFilter>) {
+  function countElements(filters: Array<ReqFilter>) {
     let total = 0;
     for (const f of filters) {
       for (const v of Object.values(f)) {
@@ -30,12 +30,7 @@ function Queries() {
     return total;
   }
 
-  function queryInfo(q: {
+  function queryInfo(q: { id: string; filters: Array<ReqFilter>; closing: boolean; subFilters: Array<ReqFilter> }) {
-    id: string;
-    filters: Array<RawReqFilter>;
-    closing: boolean;
-    subFilters: Array<RawReqFilter>;
-  }) {
     return (
       <div key={q.id}>
         {q.closing ? <s>{q.id}</s> : <>{q.id}</>}

packages/app/src/Element/TrendingPosts.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from "react";
-import { RawEvent, TaggedRawEvent } from "System";
+import { NostrEvent, TaggedRawEvent } from "System";
 import { FormattedMessage } from "react-intl";
 
 import PageSpinner from "Element/PageSpinner";
@@ -7,7 +7,7 @@ import Note from "Element/Note";
 import NostrBandApi from "External/NostrBand";
 
 export default function TrendingNotes() {
-  const [posts, setPosts] = useState<Array<RawEvent>>();
+  const [posts, setPosts] = useState<Array<NostrEvent>>();
 
   async function loadTrendingNotes() {
     const api = new NostrBandApi();

packages/app/src/Element/WriteDm.tsx

@@ -1,4 +1,4 @@
-import { encodeTLV, NostrPrefix, RawEvent } from "System";
+import { encodeTLV, NostrPrefix, NostrEvent } from "System";
 import useEventPublisher from "Feed/EventPublisher";
 import Icon from "Icons/Icon";
 import Spinner from "Icons/Spinner";
@@ -11,7 +11,7 @@ export default function WriteDm({ chatPubKey }: { chatPubKey: string }) {
   const [msg, setMsg] = useState("");
   const [sending, setSending] = useState(false);
   const [uploading, setUploading] = useState(false);
-  const [otherEvents, setOtherEvents] = useState<Array<RawEvent>>([]);
+  const [otherEvents, setOtherEvents] = useState<Array<NostrEvent>>([]);
   const [error, setError] = useState("");
   const publisher = useEventPublisher();
   const uploader = useFileUpload();

packages/app/src/Element/ZapstrEmbed.tsx

@@ -1,12 +1,12 @@
 import "./ZapstrEmbed.css";
 import { Link } from "react-router-dom";
-import { encodeTLV, NostrPrefix, RawEvent } from "System";
+import { encodeTLV, NostrPrefix, NostrEvent } from "System";
 
 import { ProxyImg } from "Element/ProxyImg";
 import ProfileImage from "Element/ProfileImage";
 import { FormattedMessage } from "react-intl";
 
-export default function ZapstrEmbed({ ev }: { ev: RawEvent }) {
+export default function ZapstrEmbed({ ev }: { ev: NostrEvent }) {
   const media = ev.tags.find(a => a[0] === "media");
   const cover = ev.tags.find(a => a[0] === "cover");
   const subject = ev.tags.find(a => a[0] === "subject");

packages/app/src/External/NostrBand.ts

@@ -1,4 +1,4 @@
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 
 export interface TrendingUser {
   pubkey: string;
@@ -9,8 +9,8 @@ export interface TrendingUserResponse {
 }
 
 export interface TrendingNote {
-  event: RawEvent;
+  event: NostrEvent;
-  author: RawEvent; // kind0 event
+  author: NostrEvent; // kind0 event
 }
 
 export interface TrendingNoteResponse {

packages/app/src/LNURL.ts

@@ -1,4 +1,4 @@
-import { HexKey, RawEvent } from "System";
+import { HexKey, NostrEvent } from "System";
 import { EmailRegex } from "Const";
 import { bech32ToText, unwrap } from "SnortUtils";
 
@@ -119,7 +119,7 @@ export class LNURL {
    * @param zap
    * @returns
    */
-  async getInvoice(amount: number, comment?: string, zap?: RawEvent) {
+  async getInvoice(amount: number, comment?: string, zap?: NostrEvent) {
     const callback = new URL(unwrap(this.#service?.callback));
     const query = new Map<string, string>();
 

packages/app/src/Pages/MessagesPage.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo, useState } from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 import { useNavigate } from "react-router-dom";
-import { HexKey, RawEvent, NostrPrefix } from "System";
+import { HexKey, NostrEvent, NostrPrefix } from "System";
 
 import UnreadCount from "Element/UnreadCount";
 import ProfileImage, { getDisplayName } from "Element/ProfileImage";
@@ -162,30 +162,30 @@ export function setLastReadDm(pk: HexKey) {
   window.localStorage.setItem(k, now.toString());
 }
 
-export function dmTo(e: RawEvent) {
+export function dmTo(e: NostrEvent) {
   const firstP = e.tags.find(b => b[0] === "p");
   return unwrap(firstP?.[1]);
 }
 
-export function isToSelf(e: Readonly<RawEvent>, pk: HexKey) {
+export function isToSelf(e: Readonly<NostrEvent>, pk: HexKey) {
   return e.pubkey === pk && dmTo(e) === pk;
 }
 
-export function dmsInChat(dms: readonly RawEvent[], pk: HexKey) {
+export function dmsInChat(dms: readonly NostrEvent[], pk: HexKey) {
   return dms.filter(a => a.pubkey === pk || dmTo(a) === pk);
 }
 
-export function totalUnread(dms: RawEvent[], myPubKey: HexKey) {
+export function totalUnread(dms: NostrEvent[], myPubKey: HexKey) {
   return extractChats(dms, myPubKey).reduce((acc, v) => (acc += v.unreadMessages), 0);
 }
 
-function unreadDms(dms: RawEvent[], myPubKey: HexKey, pk: HexKey) {
+function unreadDms(dms: NostrEvent[], myPubKey: HexKey, pk: HexKey) {
   if (pk === myPubKey) return 0;
   const lastRead = lastReadDm(pk);
   return dmsInChat(dms, pk).filter(a => a.created_at >= lastRead && a.pubkey !== myPubKey).length;
 }
 
-function newestMessage(dms: readonly RawEvent[], myPubKey: HexKey, pk: HexKey) {
+function newestMessage(dms: readonly NostrEvent[], myPubKey: HexKey, pk: HexKey) {
   if (pk === myPubKey) {
     return dmsInChat(
       dms.filter(d => isToSelf(d, myPubKey)),
@@ -196,11 +196,11 @@ function newestMessage(dms: readonly RawEvent[], myPubKey: HexKey, pk: HexKey) {
   return dmsInChat(dms, pk).reduce((acc, v) => (acc = v.created_at > acc ? v.created_at : acc), 0);
 }
 
-export function dmsForLogin(dms: readonly RawEvent[], myPubKey: HexKey) {
+export function dmsForLogin(dms: readonly NostrEvent[], myPubKey: HexKey) {
   return dms.filter(a => a.pubkey === myPubKey || (a.pubkey !== myPubKey && dmTo(a) === myPubKey));
 }
 
-export function extractChats(dms: RawEvent[], myPubKey: HexKey) {
+export function extractChats(dms: NostrEvent[], myPubKey: HexKey) {
   const myDms = dmsForLogin(dms, myPubKey);
   const keys = myDms.map(a => [a.pubkey, dmTo(a)]).flat();
   const filteredKeys = dedupe(keys);

packages/app/src/SnortUtils/index.ts

@@ -15,7 +15,7 @@ import {
   NostrPrefix,
   decodeTLV,
   TLVEntryType,
-  RawEvent,
+  NostrEvent,
 } from "System";
 import { MetadataCache } from "Cache";
 import NostrLink from "Element/NostrLink";
@@ -482,7 +482,7 @@ export function chunks<T>(arr: T[], length: number) {
   return result;
 }
 
-export function findTag(e: RawEvent, tag: string) {
+export function findTag(e: NostrEvent, tag: string) {
   const maybeTag = e.tags.find(evTag => {
     return evTag[0] === tag;
   });

packages/app/src/State/NoteCreator.ts

@@ -1,19 +1,19 @@
 import { createSlice, PayloadAction } from "@reduxjs/toolkit";
-import { RawEvent, TaggedRawEvent } from "System";
+import { NostrEvent, TaggedRawEvent } from "System";
 
 interface NoteCreatorStore {
   show: boolean;
   note: string;
   error: string;
   active: boolean;
-  preview?: RawEvent;
+  preview?: NostrEvent;
   replyTo?: TaggedRawEvent;
   showAdvanced: boolean;
   selectedCustomRelays: false | Array<string>;
   zapForward: string;
   sensitive: string;
   pollOptions?: Array<string>;
-  otherEvents: Array<RawEvent>;
+  otherEvents: Array<NostrEvent>;
 }
 
 const InitState: NoteCreatorStore = {
@@ -44,7 +44,7 @@ const NoteCreatorSlice = createSlice({
     setActive: (state, action: PayloadAction<boolean>) => {
       state.active = action.payload;
     },
-    setPreview: (state, action: PayloadAction<RawEvent | undefined>) => {
+    setPreview: (state, action: PayloadAction<NostrEvent | undefined>) => {
       state.preview = action.payload;
     },
     setReplyTo: (state, action: PayloadAction<TaggedRawEvent | undefined>) => {
@@ -65,7 +65,7 @@ const NoteCreatorSlice = createSlice({
     setPollOptions: (state, action: PayloadAction<Array<string> | undefined>) => {
       state.pollOptions = action.payload;
     },
-    setOtherEvents: (state, action: PayloadAction<Array<RawEvent>>) => {
+    setOtherEvents: (state, action: PayloadAction<Array<NostrEvent>>) => {
       state.otherEvents = action.payload;
     },
     reset: () => InitState,

packages/app/src/State/ReBroadcast.ts

@@ -1,10 +1,10 @@
 import { createSlice, PayloadAction } from "@reduxjs/toolkit";
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 
 interface ReBroadcastStore {
   show: boolean;
   selectedCustomRelays: false | Array<string>;
-  note?: RawEvent;
+  note?: NostrEvent;
 }
 
 const InitState: ReBroadcastStore = {
@@ -19,7 +19,7 @@ const ReBroadcastSlice = createSlice({
     setShow: (state, action: PayloadAction<boolean>) => {
       state.show = action.payload;
     },
-    setNote: (state, action: PayloadAction<RawEvent>) => {
+    setNote: (state, action: PayloadAction<NostrEvent>) => {
       state.note = action.payload;
     },
     setSelectedCustomRelays: (state, action: PayloadAction<false | Array<string>>) => {

packages/app/src/System/Connection.ts

@@ -2,12 +2,12 @@ import { v4 as uuid } from "uuid";
 
 import { DefaultConnectTimeout } from "./Const";
 import { ConnectionStats } from "./ConnectionStats";
-import { RawEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
+import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
 import { RelayInfo } from "./RelayInfo";
 import { unwrap } from "./Util";
 import ExternalStore from "ExternalStore";
 
-export type AuthHandler = (challenge: string, relay: string) => Promise<RawEvent | undefined>;
+export type AuthHandler = (challenge: string, relay: string) => Promise<NostrEvent | undefined>;
 
 /**
  * Relay settings
@@ -232,7 +232,7 @@ export class Connection extends ExternalStore<ConnectionStateSnapshot> {
   /**
    * Send event on this connection
    */
-  SendEvent(e: RawEvent) {
+  SendEvent(e: NostrEvent) {
     if (!this.Settings.write) {
       return;
     }
@@ -245,7 +245,7 @@ export class Connection extends ExternalStore<ConnectionStateSnapshot> {
   /**
    * Send event on this connection and wait for OK response
    */
-  async SendAsync(e: RawEvent, timeout = 5000) {
+  async SendAsync(e: NostrEvent, timeout = 5000) {
     return new Promise<void>(resolve => {
       if (!this.Settings.write) {
         resolve();

packages/app/src/System/EventBuilder.ts

@@ -1,4 +1,4 @@
-import { EventKind, HexKey, NostrPrefix, RawEvent } from "System";
+import { EventKind, HexKey, NostrPrefix, NostrEvent } from "System";
 import { HashtagRegex } from "Const";
 import { getPublicKey, parseNostrLink, unixNow } from "SnortUtils";
 import { EventExt } from "./EventExt";
@@ -63,7 +63,7 @@ export class EventBuilder {
       kind: this.#kind,
       created_at: this.#createdAt ?? unixNow(),
       tags: this.#tags,
-    } as RawEvent;
+    } as NostrEvent;
     ev.id = EventExt.createId(ev);
     return ev;
   }

packages/app/src/System/EventExt.ts

@@ -1,6 +1,6 @@
 import * as secp from "@noble/curves/secp256k1";
 import * as utils from "@noble/curves/abstract/utils";
-import { EventKind, HexKey, RawEvent, Tag } from "System";
+import { EventKind, HexKey, NostrEvent, Tag } from "System";
 import base64 from "@protobufjs/base64";
 import { sha256, unixNow } from "SnortUtils";
 
@@ -15,7 +15,7 @@ export abstract class EventExt {
   /**
    * Get the pub key of the creator of this event NIP-26
    */
-  static getRootPubKey(e: RawEvent): HexKey {
+  static getRootPubKey(e: NostrEvent): HexKey {
     const delegation = e.tags.find(a => a[0] === "delegation");
     if (delegation?.[1]) {
       return delegation[1];
@@ -26,7 +26,7 @@ export abstract class EventExt {
   /**
    * Sign this message with a private key
    */
-  static async sign(e: RawEvent, key: HexKey) {
+  static async sign(e: NostrEvent, key: HexKey) {
     e.id = this.createId(e);
 
     const sig = await secp.schnorr.sign(e.id, key);
@@ -40,13 +40,13 @@ export abstract class EventExt {
    * Check the signature of this message
    * @returns True if valid signature
    */
-  static async verify(e: RawEvent) {
+  static async verify(e: NostrEvent) {
     const id = this.createId(e);
     const result = await secp.schnorr.verify(e.sig, id, e.pubkey);
     return result;
   }
 
-  static createId(e: RawEvent) {
+  static createId(e: NostrEvent) {
     const payload = [0, e.pubkey, e.created_at, e.kind, e.tags, e.content];
 
     const hash = sha256(JSON.stringify(payload));
@@ -69,10 +69,10 @@ export abstract class EventExt {
       tags: [],
       id: "",
       sig: "",
-    } as RawEvent;
+    } as NostrEvent;
   }
 
-  static extractThread(ev: RawEvent) {
+  static extractThread(ev: NostrEvent) {
     const isThread = ev.tags.some(a => (a[0] === "e" && a[3] !== "mention") || a[0] == "a");
     if (!isThread) {
       return undefined;

packages/app/src/System/EventPublisher.ts

@@ -5,7 +5,7 @@ import {
   FullRelaySettings,
   HexKey,
   Lists,
-  RawEvent,
+  NostrEvent,
   RelaySettings,
   SystemInterface,
   TaggedRawEvent,
@@ -27,7 +27,7 @@ declare global {
   interface Window {
     nostr?: {
       getPublicKey: () => Promise<HexKey>;
-      signEvent: <T extends RawEvent>(event: T) => Promise<T>;
+      signEvent: <T extends NostrEvent>(event: T) => Promise<T>;
 
       getRelays?: () => Promise<Record<string, { read: boolean; write: boolean }>>;
 
@@ -113,7 +113,7 @@ export class EventPublisher {
     return await this.#sign(eb);
   }
 
-  broadcast(ev: RawEvent) {
+  broadcast(ev: NostrEvent) {
     console.debug(ev);
     this.#system.BroadcastEvent(ev);
   }
@@ -123,7 +123,7 @@ export class EventPublisher {
    * If a user removes all the DefaultRelays from their relay list and saves that relay list,
    * When they open the site again we wont see that updated relay list and so it will appear to reset back to the previous state
    */
-  broadcastForBootstrap(ev: RawEvent) {
+  broadcastForBootstrap(ev: NostrEvent) {
     for (const [k] of DefaultRelays) {
       this.#system.WriteOnceToRelay(k, ev);
     }
@@ -132,7 +132,7 @@ export class EventPublisher {
   /**
    * Write event to all given relays.
    */
-  broadcastAll(ev: RawEvent, relays: string[]) {
+  broadcastAll(ev: NostrEvent, relays: string[]) {
     for (const k of relays) {
       this.#system.WriteOnceToRelay(k, ev);
     }
@@ -249,7 +249,7 @@ export class EventPublisher {
     return await this.#sign(eb);
   }
 
-  async react(evRef: RawEvent, content = "+") {
+  async react(evRef: NostrEvent, content = "+") {
     const eb = this.#eb(EventKind.Reaction);
     eb.content(content);
     eb.tag(["e", evRef.id]);
@@ -298,14 +298,14 @@ export class EventPublisher {
   /**
    * Repost a note (NIP-18)
    */
-  async repost(note: RawEvent) {
+  async repost(note: NostrEvent) {
     const eb = this.#eb(EventKind.Repost);
     eb.tag(["e", note.id, ""]);
     eb.tag(["p", note.pubkey]);
     return await this.#sign(eb);
   }
 
-  async decryptDm(note: RawEvent) {
+  async decryptDm(note: NostrEvent) {
     if (note.pubkey !== this.#pubKey && !note.tags.some(a => a[1] === this.#pubKey)) {
       throw new Error("Can't decrypt, DM does not belong to this user");
     }

packages/app/src/System/GossipModel.ts

@@ -1,4 +1,4 @@
-import { FullRelaySettings, RawReqFilter } from "System";
+import { FullRelaySettings, ReqFilter } from "System";
 import { unwrap } from "SnortUtils";
 import debug from "debug";
 
@@ -6,19 +6,19 @@ const PickNRelays = 2;
 
 export interface RelayTaggedFilter {
   relay: string;
-  filter: RawReqFilter;
+  filter: ReqFilter;
 }
 
 export interface RelayTaggedFilters {
   relay: string;
-  filters: Array<RawReqFilter>;
+  filters: Array<ReqFilter>;
 }
 
 export interface RelayCache {
   get(pubkey?: string): Array<FullRelaySettings> | undefined;
 }
 
-export function splitAllByWriteRelays(cache: RelayCache, filters: Array<RawReqFilter>) {
+export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilter>) {
   const allSplit = filters
     .map(a => splitByWriteRelays(cache, a))
     .reduce((acc, v) => {
@@ -31,7 +31,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array<RawReqFi
         }
       }
       return acc;
-    }, new Map<string, Array<RawReqFilter>>());
+    }, new Map<string, Array<ReqFilter>>());
 
   return [...allSplit.entries()].map(([k, v]) => {
     return {
@@ -46,7 +46,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array<RawReqFi
  * @param filter
  * @returns
  */
-export function splitByWriteRelays(cache: RelayCache, filter: RawReqFilter): Array<RelayTaggedFilter> {
+export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<RelayTaggedFilter> {
   if ((filter.authors?.length ?? 0) === 0)
     return [
       {

packages/app/src/System/Nostr.ts

@@ -1,6 +1,6 @@
 import { RelaySettings } from "./Connection";
 
-export type RawEvent = {
+export interface NostrEvent {
   id: u256;
   pubkey: HexKey;
   created_at: number;
@@ -8,9 +8,9 @@ export type RawEvent = {
   tags: Array<Array<string>>;
   content: string;
   sig: string;
-};
+}
 
-export interface TaggedRawEvent extends RawEvent {
+export interface TaggedRawEvent extends NostrEvent {
   /**
    * A list of relays this event was seen on
    */
@@ -32,12 +32,12 @@ export type MaybeHexKey = HexKey | undefined;
  */
 export type u256 = string;
 
-export type ReqCommand = [cmd: "REQ", id: string, ...filters: Array<RawReqFilter>];
+export type ReqCommand = [cmd: "REQ", id: string, ...filters: Array<ReqFilter>];
 
 /**
  * Raw REQ filter object
  */
-export type RawReqFilter = {
+export interface ReqFilter {
   ids?: u256[];
   authors?: u256[];
   kinds?: number[];
@@ -50,7 +50,7 @@ export type RawReqFilter = {
   since?: number;
   until?: number;
   limit?: number;
-};
+}
 
 /**
  * Medatadata event content

packages/app/src/System/NostrSystem.ts

@@ -2,7 +2,7 @@ import debug from "debug";
 import { v4 as uuid } from "uuid";
 
 import ExternalStore from "ExternalStore";
-import { RawEvent, RawReqFilter, TaggedRawEvent } from "./Nostr";
+import { NostrEvent, ReqFilter, TaggedRawEvent } from "./Nostr";
 import { AuthHandler, Connection, RelaySettings, ConnectionStateSnapshot } from "./Connection";
 import { Query, QueryBase } from "./Query";
 import { RelayCache } from "./GossipModel";
@@ -194,7 +194,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
   /**
    * Send events to writable relays
    */
-  BroadcastEvent(ev: RawEvent) {
+  BroadcastEvent(ev: NostrEvent) {
     for (const [, s] of this.#sockets) {
       s.SendEvent(ev);
     }
@@ -203,7 +203,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
   /**
    * Write an event to a relay then disconnect
    */
-  async WriteOnceToRelay(address: string, ev: RawEvent) {
+  async WriteOnceToRelay(address: string, ev: NostrEvent) {
     return new Promise<void>((resolve, reject) => {
       const c = new Connection(address, { write: true, read: false }, this.HandleAuth, true);
 

packages/app/src/System/Query.ts

@@ -1,6 +1,6 @@
 import { v4 as uuid } from "uuid";
 import debug from "debug";
-import { Connection, RawReqFilter, Nips, TaggedRawEvent } from "System";
+import { Connection, ReqFilter, Nips, TaggedRawEvent } from "System";
 import { unixNowMs, unwrap } from "SnortUtils";
 import { NoteStore } from "./NoteCollection";
 import { simpleMerge } from "./RequestMerger";
@@ -22,7 +22,7 @@ class QueryTrace {
 
   constructor(
     readonly relay: string,
-    readonly filters: Array<RawReqFilter>,
+    readonly filters: Array<ReqFilter>,
     readonly connId: string,
     fnClose: (id: string) => void,
     fnProgress: () => void
@@ -94,7 +94,7 @@ export interface QueryBase {
   /**
    * The query payload (REQ filters)
    */
-  filters: Array<RawReqFilter>;
+  filters: Array<ReqFilter>;
 
   /**
    * List of relays to send this query to

packages/app/src/System/RequestBuilder.ts

@@ -1,7 +1,8 @@
-import { RawReqFilter, u256, HexKey, EventKind } from "System";
+import { ReqFilter, u256, HexKey, EventKind } from "System";
 import { appendDedupe, dedupe } from "SnortUtils";
 import { diffFilters } from "./RequestSplitter";
 import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel";
+import { mergeSimilar } from "./RequestMerger";
 
 /**
  * Which strategy is used when building REQ filters
@@ -28,7 +29,7 @@ export enum RequestStrategy {
  * A built REQ filter ready for sending to System
  */
 export interface BuiltRawReqFilter {
-  filters: Array<RawReqFilter>;
+  filters: Array<ReqFilter>;
   relay: string;
   strategy: RequestStrategy;
 }
@@ -77,7 +78,7 @@ export class RequestBuilder {
     return this;
   }
 
-  buildRaw(): Array<RawReqFilter> {
+  buildRaw(): Array<ReqFilter> {
     return this.#builders.map(f => f.filter);
   }
 
@@ -91,11 +92,11 @@ export class RequestBuilder {
    * @param q All previous filters merged
    * @returns
    */
-  buildDiff(relays: RelayCache, filters: Array<RawReqFilter>): Array<BuiltRawReqFilter> {
+  buildDiff(relays: RelayCache, filters: Array<ReqFilter>): Array<BuiltRawReqFilter> {
     const next = this.buildRaw();
     const diff = diffFilters(filters, next);
     if (diff.changed) {
-      return splitAllByWriteRelays(relays, diff.filters).map(a => {
+      return splitAllByWriteRelays(relays, diff.added).map(a => {
         return {
           strategy: RequestStrategy.AuthorsRelays,
           filters: a.filters,
@@ -124,7 +125,7 @@ export class RequestBuilder {
 
     const filtersSquashed = [...relayMerged.values()].map(a => {
       return {
-        filters: a.flatMap(b => b.filters),
+        filters: mergeSimilar(a.flatMap(b => b.filters)),
         relay: a[0].relay,
         strategy: a[0].strategy,
       } as BuiltRawReqFilter;
@@ -138,7 +139,7 @@ export class RequestBuilder {
  * Builder class for a single request filter
  */
 export class RequestFilterBuilder {
-  #filter: RawReqFilter = {};
+  #filter: ReqFilter = {};
   #relayHints = new Map<u256, Array<string>>();
 
   get filter() {

packages/app/src/System/RequestExpander.test.ts

@@ -0,0 +1,34 @@
+import { expandFilter } from "./RequestExpander";
+
+describe("RequestExpander", () => {
+  test("expand filter", () => {
+    const a = {
+      authors: ["a", "b", "c"],
+      kinds: [1, 2, 3],
+      ids: ["x", "y"],
+      "#p": ["a"],
+      since: 99,
+      limit: 10,
+    };
+    expect(expandFilter(a)).toEqual([
+      { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 },
+      { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 },
+    ]);
+  });
+});

packages/app/src/System/RequestExpander.ts

@@ -0,0 +1,48 @@
+import { u256, ReqFilter } from "./Nostr";
+
+export interface FlatReqFilter {
+  ids?: u256;
+  authors?: u256;
+  kinds?: number;
+  "#e"?: u256;
+  "#p"?: u256;
+  "#t"?: string;
+  "#d"?: string;
+  "#r"?: string;
+  search?: string;
+  since?: number;
+  until?: number;
+  limit?: number;
+}
+
+/**
+ * Expand a filter into its most fine grained form
+ */
+export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
+  const ret: Array<FlatReqFilter> = [];
+  const src = Object.entries(f);
+  const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]);
+  const props = src.filter(([, v]) => !Array.isArray(v));
+
+  function generateCombinations(index: number, currentCombination: FlatReqFilter) {
+    if (index === keys.length) {
+      ret.push(currentCombination);
+      return;
+    }
+
+    const key = keys[index];
+    const values = (f as Record<string, Array<string | number>>)[key];
+
+    for (let i = 0; i < values.length; i++) {
+      const value = values[i];
+      const updatedCombination = { ...currentCombination, [key]: value };
+      generateCombinations(index + 1, updatedCombination);
+    }
+  }
+
+  generateCombinations(0, {
+    ...Object.fromEntries(props),
+  });
+
+  return ret;
+}

packages/app/src/System/RequestMatcher.ts

@@ -1,6 +1,6 @@
-import { RawEvent, RawReqFilter } from "./Nostr";
+import { NostrEvent, ReqFilter } from "./Nostr";
 
-export function eventMatchesFilter(ev: RawEvent, filter: RawReqFilter) {
+export function eventMatchesFilter(ev: NostrEvent, filter: ReqFilter) {
   if (!(filter.ids?.includes(ev.id) ?? false)) {
     return false;
   }

packages/app/src/System/RequestMerger.test.ts

@@ -1,17 +1,19 @@
-import { RawReqFilter } from "System";
+import { ReqFilter } from "System";
-import { filterIncludes, mergeSimilar, simpleMerge } from "./RequestMerger";
+import { filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "./RequestMerger";
+import { FlatReqFilter, expandFilter } from "./RequestExpander";
+import { distance } from "./Util";
 
 describe("RequestMerger", () => {
   it("should simple merge authors", () => {
     const a = {
       authors: ["a"],
-    } as RawReqFilter;
+    } as ReqFilter;
     const b = {
       authors: ["b"],
-    } as RawReqFilter;
+    } as ReqFilter;
 
     const merged = mergeSimilar([a, b]);
-    expect(merged).toMatchObject([
+    expect(merged).toEqual([
       {
         authors: ["a", "b"],
       },
@@ -21,17 +23,17 @@ describe("RequestMerger", () => {
   it("should append non-mergable filters", () => {
     const a = {
       authors: ["a"],
-    } as RawReqFilter;
+    } as ReqFilter;
     const b = {
       authors: ["b"],
-    } as RawReqFilter;
+    } as ReqFilter;
     const c = {
       limit: 5,
       authors: ["a"],
     };
 
     const merged = mergeSimilar([a, b, c]);
-    expect(merged).toMatchObject([
+    expect(merged).toEqual([
       {
         authors: ["a", "b"],
       },
@@ -46,11 +48,11 @@ describe("RequestMerger", () => {
     const bigger = {
       authors: ["a", "b", "c"],
       since: 99,
-    } as RawReqFilter;
+    } as ReqFilter;
     const smaller = {
       authors: ["c"],
       since: 100,
-    } as RawReqFilter;
+    } as ReqFilter;
     expect(filterIncludes(bigger, smaller)).toBe(true);
   });
 
@@ -58,14 +60,50 @@ describe("RequestMerger", () => {
     const a = {
       authors: ["a", "b", "c"],
       since: 99,
-    } as RawReqFilter;
+    } as ReqFilter;
     const b = {
       authors: ["c", "d", "e"],
       since: 100,
-    } as RawReqFilter;
+    } as ReqFilter;
     expect(simpleMerge([a, b])).toEqual({
       authors: ["a", "b", "c", "d", "e"],
       since: 100,
     });
   });
 });
+
+describe("flatMerge", () => {
+  it("should flat merge simple", () => {
+    const input = [
+      { ids: 0, authors: "a" },
+      { ids: 0, authors: "b" },
+      { kinds: 1 },
+      { kinds: 2 },
+      { ids: 0, authors: "c" },
+      { authors: "c", kinds: 1 },
+      { authors: "c", limit: 100 },
+      { ids: 1, authors: "c" },
+    ] as Array<FlatReqFilter>;
+    const output = [
+      { ids: [0], authors: ["a", "b", "c"] },
+      { kinds: [1, 2] },
+      { authors: ["c"], kinds: [1] },
+      { authors: ["c"], limit: 100 },
+      { ids: [1], authors: ["c"] },
+    ] as Array<ReqFilter>;
+
+    expect(flatMerge(input)).toEqual(output);
+  });
+
+  it("should expand and flat merge complex same", () => {
+    const input = [
+      { kinds: [1, 6969, 6], authors: ["kieran", "snort", "c", "d", "e"], since: 1, until: 100 },
+      { kinds: [4], authors: ["kieran"] },
+      { kinds: [4], "#p": ["kieran"] },
+      { kinds: [1000], authors: ["snort"], "#p": ["kieran"] },
+    ] as Array<ReqFilter>;
+
+    const dut = flatMerge(input.flatMap(expandFilter).sort(() => (Math.random() > 0.5 ? 1 : -1)));
+    expect(dut.every(a => input.some(b => distance(b, a) === 0))).toEqual(true);
+  });
+});

packages/app/src/System/RequestMerger.ts

@@ -1,12 +1,40 @@
-import { RawReqFilter } from "System";
+import { ReqFilter } from "System";
+import { FlatReqFilter } from "./RequestExpander";
+import { distance } from "./Util";
 
-export function mergeSimilar(filters: Array<RawReqFilter>): Array<RawReqFilter> {
+/**
-  const hasCriticalKeySet = (a: RawReqFilter) => {
+ * Keys which can change the entire meaning of the filter outside the array types
-    return a.limit !== undefined || a.since !== undefined || a.until !== undefined;
+ */
-  };
+const DiscriminatorKeys = ["since", "until", "limit", "search"];
-  const canEasilyMerge = filters.filter(a => !hasCriticalKeySet(a));
+
-  const cannotMerge = filters.filter(a => hasCriticalKeySet(a));
+export function canMergeFilters(a: any, b: any): boolean {
-  return [...(canEasilyMerge.length > 0 ? [simpleMerge(canEasilyMerge)] : []), ...cannotMerge];
+  for (const key of DiscriminatorKeys) {
+    if (key in a || key in b) {
+      if (a[key] !== b[key]) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
+  const ret = [];
+
+  while (filters.length > 0) {
+    const current = filters.shift()!;
+    const mergeSet = [current];
+    for (let i = 0; i < filters.length; i++) {
+      const f = filters[i];
+      if (mergeSet.every(v => canMergeFilters(v, f) && distance(v, f) === 1)) {
+        mergeSet.push(filters.splice(i, 1)[0]);
+        i--;
+      }
+    }
+    ret.push(simpleMerge(mergeSet));
+  }
+  return ret;
 }
 
 /**
@@ -14,7 +42,7 @@ export function mergeSimilar(filters: Array<RawReqFilter>): Array<RawReqFilter>
  * @param filters
  * @returns
  */
-export function simpleMerge(filters: Array<RawReqFilter>) {
+export function simpleMerge(filters: Array<ReqFilter>) {
   const result: any = {};
 
   filters.forEach(filter => {
@@ -31,7 +59,7 @@ export function simpleMerge(filters: Array<RawReqFilter>) {
     });
   });
 
-  return result as RawReqFilter;
+  return result as ReqFilter;
 }
 
 /**
@@ -40,7 +68,7 @@ export function simpleMerge(filters: Array<RawReqFilter>) {
  * @param smaller
  * @returns
  */
-export function filterIncludes(bigger: RawReqFilter, smaller: RawReqFilter) {
+export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
   const outside = bigger as Record<string, Array<string | number> | number>;
   for (const [k, v] of Object.entries(smaller)) {
     if (outside[k] === undefined) {
@@ -61,3 +89,61 @@ export function filterIncludes(bigger: RawReqFilter, smaller: RawReqFilter) {
   }
   return true;
 }
+
+/**
+ * Merge expanded flat filters into combined concise filters
+ * @param all
+ * @returns
+ */
+export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
+  let ret: Array<ReqFilter> = [];
+
+  // to compute filters which can be merged we need to calucate the distance change between each filter
+  // then we can merge filters which are exactly 1 change diff from each other
+
+  function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
+    const result: any = {};
+
+    filters.forEach(f => {
+      const filter = f as Record<string, string | number>;
+      Object.entries(filter).forEach(([key, value]) => {
+        if (!DiscriminatorKeys.includes(key)) {
+          if (result[key] === undefined) {
+            result[key] = [value];
+          } else {
+            result[key] = [...new Set([...result[key], value])];
+          }
+        } else {
+          result[key] = value;
+        }
+      });
+    });
+
+    return result as ReqFilter;
+  }
+
+  // reducer, kinda verbose
+  while (all.length > 0) {
+    const currentFilter = all.shift()!;
+    const mergeSet = [currentFilter];
+
+    for (let i = 0; i < all.length; i++) {
+      const f = all[i];
+
+      if (mergeSet.every(a => canMergeFilters(a, f) && distance(a, f) === 1)) {
+        mergeSet.push(all.splice(i, 1)[0]);
+        i--;
+      }
+    }
+    ret.push(mergeFiltersInSet(mergeSet));
+  }
+
+  while (true) {
+    const n = mergeSimilar([...ret]);
+    if (n.length === ret.length) {
+      break;
+    }
+    ret = n;
+  }
+  return ret;
+}

packages/app/src/System/RequestSplitter.test.ts

@@ -1,104 +1,87 @@
-import { RawReqFilter } from "System";
+import { ReqFilter } from "System";
 import { describe, expect } from "@jest/globals";
-import { diffFilters, expandFilter } from "./RequestSplitter";
+import { diffFilters } from "./RequestSplitter";
 
 describe("RequestSplitter", () => {
   test("single filter add value", () => {
-    const a: Array<RawReqFilter> = [{ kinds: [0], authors: ["a"] }];
+    const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"] }];
-    const b: Array<RawReqFilter> = [{ kinds: [0], authors: ["a", "b"] }];
+    const b: Array<ReqFilter> = [{ kinds: [0], authors: ["a", "b"] }];
     const diff = diffFilters(a, b);
-    expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["b"] }], changed: true });
+    expect(diff).toEqual({
+      added: [{ kinds: [0], authors: ["b"] }],
+      removed: [],
+      changed: true,
+    });
   });
   test("single filter remove value", () => {
-    const a: Array<RawReqFilter> = [{ kinds: [0], authors: ["a"] }];
+    const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"] }];
-    const b: Array<RawReqFilter> = [{ kinds: [0], authors: ["b"] }];
+    const b: Array<ReqFilter> = [{ kinds: [0], authors: ["b"] }];
     const diff = diffFilters(a, b);
-    expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["b"] }], changed: true });
+    expect(diff).toEqual({
+      added: [{ kinds: [0], authors: ["b"] }],
+      removed: [{ kinds: [0], authors: ["a"] }],
+      changed: true,
+    });
   });
   test("single filter change critical key", () => {
-    const a: Array<RawReqFilter> = [{ kinds: [0], authors: ["a"], since: 100 }];
+    const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"], since: 100 }];
-    const b: Array<RawReqFilter> = [{ kinds: [0], authors: ["a", "b"], since: 101 }];
+    const b: Array<ReqFilter> = [{ kinds: [0], authors: ["a", "b"], since: 101 }];
     const diff = diffFilters(a, b);
-    expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["a", "b"], since: 101 }], changed: true });
+    expect(diff).toEqual({
+      added: [{ kinds: [0], authors: ["a", "b"], since: 101 }],
+      removed: [{ kinds: [0], authors: ["a"], since: 100 }],
+      changed: true,
+    });
   });
   test("multiple filter add value", () => {
-    const a: Array<RawReqFilter> = [
+    const a: Array<ReqFilter> = [
       { kinds: [0], authors: ["a"] },
       { kinds: [69], authors: ["a"] },
     ];
-    const b: Array<RawReqFilter> = [
+    const b: Array<ReqFilter> = [
       { kinds: [0], authors: ["a", "b"] },
       { kinds: [69], authors: ["a", "c"] },
     ];
     const diff = diffFilters(a, b);
     expect(diff).toEqual({
-      filters: [
+      added: [
         { kinds: [0], authors: ["b"] },
         { kinds: [69], authors: ["c"] },
       ],
+      removed: [],
       changed: true,
     });
   });
   test("multiple filter remove value", () => {
-    const a: Array<RawReqFilter> = [
+    const a: Array<ReqFilter> = [
       { kinds: [0], authors: ["a"] },
       { kinds: [69], authors: ["a"] },
     ];
-    const b: Array<RawReqFilter> = [
+    const b: Array<ReqFilter> = [
       { kinds: [0], authors: ["b"] },
       { kinds: [69], authors: ["c"] },
     ];
     const diff = diffFilters(a, b);
     expect(diff).toEqual({
-      filters: [
+      added: [
         { kinds: [0], authors: ["b"] },
         { kinds: [69], authors: ["c"] },
       ],
+      removed: [{ kinds: [0, 69], authors: ["a"] }],
       changed: true,
     });
   });
   test("add filter", () => {
-    const a: Array<RawReqFilter> = [{ kinds: [0], authors: ["a"] }];
+    const a: Array<ReqFilter> = [{ kinds: [0], authors: ["a"] }];
-    const b: Array<RawReqFilter> = [
+    const b: Array<ReqFilter> = [
       { kinds: [0], authors: ["a"] },
       { kinds: [69], authors: ["c"] },
     ];
     const diff = diffFilters(a, b);
     expect(diff).toEqual({
-      filters: [
+      added: [{ kinds: [69], authors: ["c"] }],
-        { kinds: [0], authors: ["a"] },
+      removed: [],
-        { kinds: [69], authors: ["c"] },
-      ],
       changed: true,
     });
   });
-  test("expand filter", () => {
-    const a = {
-      authors: ["a", "b", "c"],
-      kinds: [1, 2, 3],
-      ids: ["x", "y"],
-      since: 99,
-      limit: 10,
-    };
-    expect(expandFilter(a)).toEqual([
-      { authors: ["a"], kinds: [1], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["a"], kinds: [1], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["a"], kinds: [2], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["a"], kinds: [2], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["a"], kinds: [3], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["a"], kinds: [3], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [1], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [1], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [2], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [2], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [3], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["b"], kinds: [3], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [1], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [1], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [2], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [2], ids: ["y"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [3], ids: ["x"], since: 99, limit: 10 },
-      { authors: ["c"], kinds: [3], ids: ["y"], since: 99, limit: 10 },
-    ]);
-  });
 });

packages/app/src/System/RequestSplitter.ts

@@ -1,76 +1,18 @@
-import { RawReqFilter } from "System";
+import { ReqFilter } from "System";
+import { deepEqual } from "./Util";
+import { expandFilter } from "./RequestExpander";
+import { flatMerge } from "./RequestMerger";
 
-// Critical keys changing means the entire filter has changed
+export function diffFilters(prev: Array<ReqFilter>, next: Array<ReqFilter>) {
-export const CriticalKeys = ["since", "until", "limit"];
+  const prevExpanded = prev.flatMap(expandFilter);
+  const nextExpanded = next.flatMap(expandFilter);
 
-export function diffFilters(a: Array<RawReqFilter>, b: Array<RawReqFilter>) {
+  const added = flatMerge(nextExpanded.filter(a => !prevExpanded.some(b => deepEqual(a, b))));
-  const result: Array<RawReqFilter> = [];
+  const removed = flatMerge(prevExpanded.filter(a => !nextExpanded.some(b => deepEqual(a, b))));
-  let anyChanged = false;
-  for (const [i, bN] of b.entries()) {
-    const prev: Record<string, string | number | string[] | number[] | undefined> = a[i];
-    if (!prev) {
-      result.push(bN);
-      anyChanged = true;
-    } else {
-      let anyCriticalKeyChanged = false;
-      for (const [k, v] of Object.entries(bN)) {
-        if (Array.isArray(v)) {
-          const prevArray = prev[k] as Array<string | number> | undefined;
-          const thisArray = v as Array<string | number>;
-          const added = thisArray.filter(a => !prevArray?.includes(a));
-          // support adding new values to array, removing values is ignored since we only care about getting new values
-          result[i] = { ...result[i], [k]: added.length === 0 ? prevArray ?? [] : added };
-          if (added.length > 0) {
-            anyChanged = true;
-          }
-        } else if (prev[k] !== v) {
-          result[i] = { ...result[i], [k]: v };
-          if (CriticalKeys.includes(k)) {
-            anyCriticalKeyChanged = anyChanged = true;
-            break;
-          }
-        }
-      }
-      if (anyCriticalKeyChanged) {
-        result[i] = bN;
-      }
-    }
-  }
 
   return {
-    filters: result,
+    added,
-    changed: anyChanged,
+    removed,
+    changed: added.length > 0 || removed.length > 0,
   };
 }
-
-/**
- * Expand a filter into its most fine grained form
- */
-export function expandFilter(f: RawReqFilter): Array<RawReqFilter> {
-  const ret: Array<RawReqFilter> = [];
-  const src = Object.entries(f);
-  const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]);
-  const props = src.filter(([, v]) => !Array.isArray(v));
-
-  function generateCombinations(index: number, currentCombination: RawReqFilter) {
-    if (index === keys.length) {
-      ret.push(currentCombination);
-      return;
-    }
-
-    const key = keys[index];
-    const values = (f as Record<string, Array<string | number>>)[key];
-
-    for (let i = 0; i < values.length; i++) {
-      const value = values[i];
-      const updatedCombination = { ...currentCombination, [key]: [value] };
-      generateCombinations(index + 1, updatedCombination);
-    }
-  }
-
-  generateCombinations(0, {
-    ...Object.fromEntries(props),
-  });
-
-  return ret;
-}

packages/app/src/System/SystemWorker.ts

@@ -2,7 +2,7 @@ import ExternalStore from "ExternalStore";
 import {
   NoteStore,
   Query,
-  RawEvent,
+  NostrEvent,
   RelaySettings,
   RequestBuilder,
   SystemSnapshot,
@@ -51,11 +51,11 @@ export class SystemWorker extends ExternalStore<SystemSnapshot> implements Syste
     throw new Error("Method not implemented.");
   }
 
-  BroadcastEvent(ev: RawEvent): void {
+  BroadcastEvent(ev: NostrEvent): void {
     throw new Error("Method not implemented.");
   }
 
-  WriteOnceToRelay(relay: string, ev: RawEvent): Promise<void> {
+  WriteOnceToRelay(relay: string, ev: NostrEvent): Promise<void> {
     throw new Error("Method not implemented.");
   }
 

packages/app/src/System/Util.test.ts

@@ -0,0 +1,72 @@
+import { distance } from "./Util";
+
+describe("distance", () => {
+  it("should have 0 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "a",
+    };
+    expect(distance(a, b)).toEqual(0);
+  });
+  it("should have 1 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "b",
+    };
+    expect(distance(a, b)).toEqual(1);
+  });
+  it("should have 10 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "a",
+      kinds: 1,
+    };
+    expect(distance(a, b)).toEqual(10);
+  });
+  it("should have 11 distance", () => {
+    const a = {
+      ids: "a",
+    };
+    const b = {
+      ids: "b",
+      kinds: 1,
+    };
+    expect(distance(a, b)).toEqual(11);
+  });
+  it("should have 1 distance, arrays", () => {
+    const a = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    const b = {
+      since: 1,
+      until: 100,
+      kinds: [6969],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    expect(distance(a, b)).toEqual(1);
+  });
+  it("should have 1 distance, array change extra", () => {
+    const a = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["f", "kieran", "snort", "c", "d"],
+    };
+    const b = {
+      since: 1,
+      until: 100,
+      kinds: [1],
+      authors: ["kieran", "snort", "c", "d", "e"],
+    };
+    expect(distance(a, b)).toEqual(1);
+  });
+});

packages/app/src/System/Util.ts

@@ -40,3 +40,47 @@ export function unixNow() {
 export function unixNowMs() {
   return new Date().getTime();
 }
+
+export function deepEqual(x: any, y: any): boolean {
+  const ok = Object.keys,
+    tx = typeof x,
+    ty = typeof y;
+
+  return x && y && tx === "object" && tx === ty
+    ? ok(x).length === ok(y).length && ok(x).every(key => deepEqual(x[key], y[key]))
+    : x === y;
+}
+
+/**
+ * Compute the "distance" between two objects by comparing their difference in properties
+ * Missing/Added keys result in +10 distance
+ * This is not recursive
+ */
+export function distance(a: any, b: any): number {
+  const keys1 = Object.keys(a);
+  const keys2 = Object.keys(b);
+  const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
+
+  let distance = 0;
+  for (const key of maxKeys) {
+    if (key in a && key in b) {
+      if (Array.isArray(a[key]) && Array.isArray(b[key])) {
+        const aa = a[key] as Array<string | number>;
+        const bb = b[key] as Array<string | number>;
+        if (aa.length === bb.length) {
+          if (aa.some(v => !bb.includes(v))) {
+            distance++;
+          }
+        } else {
+          distance++;
+        }
+      } else if (a[key] !== b[key]) {
+        distance++;
+      }
+    } else {
+      distance += 10;
+    }
+  }
+
+  return distance;
+}

packages/app/src/System/index.ts

@@ -9,7 +9,7 @@ import {
   ReplaceableNoteStore,
 } from "./NoteCollection";
 import { Query } from "./Query";
-import { RawEvent, RawReqFilter } from "./Nostr";
+import { NostrEvent, ReqFilter } from "./Nostr";
 
 export * from "./NostrSystem";
 export { default as EventKind } from "./EventKind";
@@ -29,15 +29,15 @@ export interface SystemInterface {
   Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder | null): Query | undefined;
   ConnectToRelay(address: string, options: RelaySettings): Promise<void>;
   DisconnectRelay(address: string): void;
-  BroadcastEvent(ev: RawEvent): void;
+  BroadcastEvent(ev: NostrEvent): void;
-  WriteOnceToRelay(relay: string, ev: RawEvent): Promise<void>;
+  WriteOnceToRelay(relay: string, ev: NostrEvent): Promise<void>;
 }
 
 export interface SystemSnapshot {
   queries: Array<{
     id: string;
-    filters: Array<RawReqFilter>;
+    filters: Array<ReqFilter>;
-    subFilters: Array<RawReqFilter>;
+    subFilters: Array<ReqFilter>;
     closing: boolean;
   }>;
 }

packages/app/src/Upload/index.ts

@@ -1,5 +1,5 @@
 import useLogin from "Hooks/useLogin";
-import { RawEvent } from "System";
+import { NostrEvent } from "System";
 
 import NostrBuild from "Upload/NostrBuild";
 import VoidCat from "Upload/VoidCat";
@@ -14,7 +14,7 @@ export interface UploadResult {
   /**
    * NIP-94 File Header
    */
-  header?: RawEvent;
+  header?: NostrEvent;
 }
 
 /**

packages/app/src/Wallet/NostrWalletConnect.ts

@@ -1,4 +1,4 @@
-import { Connection, EventKind, RawEvent } from "System";
+import { Connection, EventKind, NostrEvent } from "System";
 import { EventBuilder } from "System";
 import { EventExt } from "System/EventExt";
 import { LNWallet, WalletError, WalletErrorCode, WalletInfo, WalletInvoice, WalletInvoiceState } from "Wallet";
@@ -123,7 +123,7 @@ export class NostrConnectWallet implements LNWallet {
     return Promise.resolve([]);
   }
 
-  async #onReply(sub: string, e: RawEvent) {
+  async #onReply(sub: string, e: NostrEvent) {
     if (sub === "info") {
       const pending = this.#commandQueue.get("info");
       if (!pending) {