From ae6618f0ed1de0f18cd4239d70815a1039679b0f Mon Sep 17 00:00:00 2001 From: Kieran Date: Thu, 1 Jun 2023 22:03:28 +0100 Subject: [PATCH] improve diff filters fix tests expander/compressor filter mangler --- packages/app/src/Cache/DMCache.ts | 10 +- packages/app/src/Cache/index.ts | 4 +- packages/app/src/Db/index.ts | 6 +- packages/app/src/Element/NostrFileHeader.tsx | 4 +- packages/app/src/Element/NoteReaction.tsx | 4 +- packages/app/src/Element/PubkeyList.tsx | 4 +- packages/app/src/Element/SendSats.tsx | 4 +- packages/app/src/Element/SubDebug.tsx | 11 +- packages/app/src/Element/TrendingPosts.tsx | 4 +- packages/app/src/Element/WriteDm.tsx | 4 +- packages/app/src/Element/ZapstrEmbed.tsx | 4 +- packages/app/src/External/NostrBand.ts | 6 +- packages/app/src/LNURL.ts | 4 +- packages/app/src/Pages/MessagesPage.tsx | 18 +-- packages/app/src/SnortUtils/index.ts | 4 +- packages/app/src/State/NoteCreator.ts | 10 +- packages/app/src/State/ReBroadcast.ts | 6 +- packages/app/src/System/Connection.ts | 8 +- packages/app/src/System/EventBuilder.ts | 4 +- packages/app/src/System/EventExt.ts | 14 +-- packages/app/src/System/EventPublisher.ts | 16 +-- packages/app/src/System/GossipModel.ts | 12 +- packages/app/src/System/Nostr.ts | 12 +- packages/app/src/System/NostrSystem.ts | 6 +- packages/app/src/System/Query.ts | 6 +- packages/app/src/System/RequestBuilder.ts | 15 +-- .../app/src/System/RequestExpander.test.ts | 34 ++++++ packages/app/src/System/RequestExpander.ts | 48 ++++++++ packages/app/src/System/RequestMatcher.ts | 4 +- packages/app/src/System/RequestMerger.test.ts | 62 ++++++++-- packages/app/src/System/RequestMerger.ts | 108 ++++++++++++++++-- .../app/src/System/RequestSplitter.test.ts | 87 ++++++-------- packages/app/src/System/RequestSplitter.ts | 82 ++----------- packages/app/src/System/SystemWorker.ts | 6 +- packages/app/src/System/Util.test.ts | 72 ++++++++++++ packages/app/src/System/Util.ts | 44 +++++++ packages/app/src/System/index.ts | 10 +- packages/app/src/Upload/index.ts | 4 +- packages/app/src/Wallet/NostrWalletConnect.ts | 4 +- 39 files changed, 504 insertions(+), 261 deletions(-) create mode 100644 packages/app/src/System/RequestExpander.test.ts create mode 100644 packages/app/src/System/RequestExpander.ts create mode 100644 packages/app/src/System/Util.test.ts diff --git a/packages/app/src/Cache/DMCache.ts b/packages/app/src/Cache/DMCache.ts index cacfd3791..7cbce8df7 100644 --- a/packages/app/src/Cache/DMCache.ts +++ b/packages/app/src/Cache/DMCache.ts @@ -1,13 +1,13 @@ -import { RawEvent } from "System"; +import { NostrEvent } from "System"; import { db } from "Db"; import FeedCache from "./FeedCache"; -class DMCache extends FeedCache { +class DMCache extends FeedCache { constructor() { super("DMCache", db.dms); } - key(of: RawEvent): string { + key(of: NostrEvent): string { return of.id; } @@ -23,11 +23,11 @@ class DMCache extends FeedCache { return ret; } - allDms(): Array { + allDms(): Array { return [...this.cache.values()]; } - takeSnapshot(): Array { + takeSnapshot(): Array { return this.allDms(); } } diff --git a/packages/app/src/Cache/index.ts b/packages/app/src/Cache/index.ts index 02941f3d0..0f0cabb49 100644 --- a/packages/app/src/Cache/index.ts +++ b/packages/app/src/Cache/index.ts @@ -1,4 +1,4 @@ -import { HexKey, RawEvent, UserMetadata } from "System"; +import { HexKey, NostrEvent, UserMetadata } from "System"; import { hexToBech32, unixNowMs } from "SnortUtils"; import { DmCache } from "./DMCache"; import { InteractionCache } from "./EventInteractionCache"; @@ -37,7 +37,7 @@ export interface MetadataCache extends UserMetadata { isNostrAddressValid: boolean; } -export function mapEventToProfile(ev: RawEvent) { +export function mapEventToProfile(ev: NostrEvent) { try { const data: UserMetadata = JSON.parse(ev.content); return { diff --git a/packages/app/src/Db/index.ts b/packages/app/src/Db/index.ts index 25f0e7880..9a72e8f39 100644 --- a/packages/app/src/Db/index.ts +++ b/packages/app/src/Db/index.ts @@ -1,5 +1,5 @@ import Dexie, { Table } from "dexie"; -import { FullRelaySettings, HexKey, RawEvent, u256 } from "System"; +import { FullRelaySettings, HexKey, NostrEvent, u256 } from "System"; import { MetadataCache } from "Cache"; export const NAME = "snortDB"; @@ -48,8 +48,8 @@ export class SnortDB extends Dexie { users!: Table; relayMetrics!: Table; userRelays!: Table; - events!: Table; - dms!: Table; + events!: Table; + dms!: Table; eventInteraction!: Table; constructor() { diff --git a/packages/app/src/Element/NostrFileHeader.tsx b/packages/app/src/Element/NostrFileHeader.tsx index 49531b0cc..512d09df8 100644 --- a/packages/app/src/Element/NostrFileHeader.tsx +++ b/packages/app/src/Element/NostrFileHeader.tsx @@ -1,5 +1,5 @@ import { FormattedMessage } from "react-intl"; -import { RawEvent } from "System"; +import { NostrEvent } from "System"; import { findTag, NostrLink } from "SnortUtils"; import useEventFeed from "Feed/EventFeed"; @@ -14,7 +14,7 @@ export default function NostrFileHeader({ link }: { link: NostrLink }) { return ; } -export function NostrFileElement({ ev }: { ev: RawEvent }) { +export function NostrFileElement({ ev }: { ev: NostrEvent }) { // assume image or embed which can be rendered by the hypertext kind // todo: make use of hash // todo: use magnet or other links if present diff --git a/packages/app/src/Element/NoteReaction.tsx b/packages/app/src/Element/NoteReaction.tsx index 236b3772e..1849c904b 100644 --- a/packages/app/src/Element/NoteReaction.tsx +++ b/packages/app/src/Element/NoteReaction.tsx @@ -1,7 +1,7 @@ import "./NoteReaction.css"; import { Link } from "react-router-dom"; import { useMemo } from "react"; -import { EventKind, RawEvent, TaggedRawEvent, NostrPrefix } from "System"; +import { EventKind, NostrEvent, TaggedRawEvent, NostrPrefix } from "System"; import Note from "Element/Note"; import ProfileImage from "Element/ProfileImage"; @@ -43,7 +43,7 @@ export default function NoteReaction(props: NoteReactionProps) { function extractRoot() { if (ev?.kind === EventKind.Repost && ev.content.length > 0 && ev.content !== "#[0]") { try { - const r: RawEvent = JSON.parse(ev.content); + const r: NostrEvent = JSON.parse(ev.content); return r as TaggedRawEvent; } catch (e) { console.error("Could not load reposted content", e); diff --git a/packages/app/src/Element/PubkeyList.tsx b/packages/app/src/Element/PubkeyList.tsx index 17a3b1612..6350af3f1 100644 --- a/packages/app/src/Element/PubkeyList.tsx +++ b/packages/app/src/Element/PubkeyList.tsx @@ -1,8 +1,8 @@ -import { RawEvent } from "System"; +import { NostrEvent } from "System"; import { dedupe } from "SnortUtils"; import FollowListBase from "./FollowListBase"; -export default function PubkeyList({ ev, className }: { ev: RawEvent; className?: string }) { +export default function PubkeyList({ ev, className }: { ev: NostrEvent; className?: string }) { const ids = dedupe(ev.tags.filter(a => a[0] === "p").map(a => a[1])); return ; } diff --git a/packages/app/src/Element/SendSats.tsx b/packages/app/src/Element/SendSats.tsx index 93d8e240b..765c59294 100644 --- a/packages/app/src/Element/SendSats.tsx +++ b/packages/app/src/Element/SendSats.tsx @@ -2,7 +2,7 @@ import "./SendSats.css"; import React, { useEffect, useMemo, useState } from "react"; import { useIntl, FormattedMessage } from "react-intl"; -import { HexKey, RawEvent } from "System"; +import { HexKey, NostrEvent } from "System"; import { System } from "index"; import { formatShort } from "Number"; import Icon from "Icons/Icon"; @@ -125,7 +125,7 @@ export default function SendSats(props: SendSatsProps) { async function loadInvoice() { if (!amount || !handler || !publisher) return null; - let zap: RawEvent | undefined; + let zap: NostrEvent | undefined; if (author && zapType !== ZapType.NonZap) { const relays = Object.keys(login.relays.item); diff --git a/packages/app/src/Element/SubDebug.tsx b/packages/app/src/Element/SubDebug.tsx index acf16e47b..5ce1d17fa 100644 --- a/packages/app/src/Element/SubDebug.tsx +++ b/packages/app/src/Element/SubDebug.tsx @@ -5,7 +5,7 @@ import useRelayState from "Feed/RelayState"; import Tabs, { Tab } from "Element/Tabs"; import { unwrap } from "SnortUtils"; import useSystemState from "Hooks/useSystemState"; -import { RawReqFilter } from "System"; +import { ReqFilter } from "System"; import { useCopy } from "useCopy"; import { System } from "index"; @@ -18,7 +18,7 @@ function Queries() { const qs = useSystemState(); const { copy } = useCopy(); - function countElements(filters: Array) { + function countElements(filters: Array) { let total = 0; for (const f of filters) { for (const v of Object.values(f)) { @@ -30,12 +30,7 @@ function Queries() { return total; } - function queryInfo(q: { - id: string; - filters: Array; - closing: boolean; - subFilters: Array; - }) { + function queryInfo(q: { id: string; filters: Array; closing: boolean; subFilters: Array }) { return (
{q.closing ? {q.id} : <>{q.id}} diff --git a/packages/app/src/Element/TrendingPosts.tsx b/packages/app/src/Element/TrendingPosts.tsx index 98ceee3d3..c46c6e73e 100644 --- a/packages/app/src/Element/TrendingPosts.tsx +++ b/packages/app/src/Element/TrendingPosts.tsx @@ -1,5 +1,5 @@ import { useEffect, useState } from "react"; -import { RawEvent, TaggedRawEvent } from "System"; +import { NostrEvent, TaggedRawEvent } from "System"; import { FormattedMessage } from "react-intl"; import PageSpinner from "Element/PageSpinner"; @@ -7,7 +7,7 @@ import Note from "Element/Note"; import NostrBandApi from "External/NostrBand"; export default function TrendingNotes() { - const [posts, setPosts] = useState>(); + const [posts, setPosts] = useState>(); async function loadTrendingNotes() { const api = new NostrBandApi(); diff --git a/packages/app/src/Element/WriteDm.tsx b/packages/app/src/Element/WriteDm.tsx index 4a594f2ef..445437bd7 100644 --- a/packages/app/src/Element/WriteDm.tsx +++ b/packages/app/src/Element/WriteDm.tsx @@ -1,4 +1,4 @@ -import { encodeTLV, NostrPrefix, RawEvent } from "System"; +import { encodeTLV, NostrPrefix, NostrEvent } from "System"; import useEventPublisher from "Feed/EventPublisher"; import Icon from "Icons/Icon"; import Spinner from "Icons/Spinner"; @@ -11,7 +11,7 @@ export default function WriteDm({ chatPubKey }: { chatPubKey: string }) { const [msg, setMsg] = useState(""); const [sending, setSending] = useState(false); const [uploading, setUploading] = useState(false); - const [otherEvents, setOtherEvents] = useState>([]); + const [otherEvents, setOtherEvents] = useState>([]); const [error, setError] = useState(""); const publisher = useEventPublisher(); const uploader = useFileUpload(); diff --git a/packages/app/src/Element/ZapstrEmbed.tsx b/packages/app/src/Element/ZapstrEmbed.tsx index 84068e394..d5ca7f017 100644 --- a/packages/app/src/Element/ZapstrEmbed.tsx +++ b/packages/app/src/Element/ZapstrEmbed.tsx @@ -1,12 +1,12 @@ import "./ZapstrEmbed.css"; import { Link } from "react-router-dom"; -import { encodeTLV, NostrPrefix, RawEvent } from "System"; +import { encodeTLV, NostrPrefix, NostrEvent } from "System"; import { ProxyImg } from "Element/ProxyImg"; import ProfileImage from "Element/ProfileImage"; import { FormattedMessage } from "react-intl"; -export default function ZapstrEmbed({ ev }: { ev: RawEvent }) { +export default function ZapstrEmbed({ ev }: { ev: NostrEvent }) { const media = ev.tags.find(a => a[0] === "media"); const cover = ev.tags.find(a => a[0] === "cover"); const subject = ev.tags.find(a => a[0] === "subject"); diff --git a/packages/app/src/External/NostrBand.ts b/packages/app/src/External/NostrBand.ts index eeb00c9ac..8615b61e8 100644 --- a/packages/app/src/External/NostrBand.ts +++ b/packages/app/src/External/NostrBand.ts @@ -1,4 +1,4 @@ -import { RawEvent } from "System"; +import { NostrEvent } from "System"; export interface TrendingUser { pubkey: string; @@ -9,8 +9,8 @@ export interface TrendingUserResponse { } export interface TrendingNote { - event: RawEvent; - author: RawEvent; // kind0 event + event: NostrEvent; + author: NostrEvent; // kind0 event } export interface TrendingNoteResponse { diff --git a/packages/app/src/LNURL.ts b/packages/app/src/LNURL.ts index 9a0e75d77..4138bb932 100644 --- a/packages/app/src/LNURL.ts +++ b/packages/app/src/LNURL.ts @@ -1,4 +1,4 @@ -import { HexKey, RawEvent } from "System"; +import { HexKey, NostrEvent } from "System"; import { EmailRegex } from "Const"; import { bech32ToText, unwrap } from "SnortUtils"; @@ -119,7 +119,7 @@ export class LNURL { * @param zap * @returns */ - async getInvoice(amount: number, comment?: string, zap?: RawEvent) { + async getInvoice(amount: number, comment?: string, zap?: NostrEvent) { const callback = new URL(unwrap(this.#service?.callback)); const query = new Map(); diff --git a/packages/app/src/Pages/MessagesPage.tsx b/packages/app/src/Pages/MessagesPage.tsx index 831fddd35..34f83c7d9 100644 --- a/packages/app/src/Pages/MessagesPage.tsx +++ b/packages/app/src/Pages/MessagesPage.tsx @@ -1,7 +1,7 @@ import React, { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { HexKey, RawEvent, NostrPrefix } from "System"; +import { HexKey, NostrEvent, NostrPrefix } from "System"; import UnreadCount from "Element/UnreadCount"; import ProfileImage, { getDisplayName } from "Element/ProfileImage"; @@ -162,30 +162,30 @@ export function setLastReadDm(pk: HexKey) { window.localStorage.setItem(k, now.toString()); } -export function dmTo(e: RawEvent) { +export function dmTo(e: NostrEvent) { const firstP = e.tags.find(b => b[0] === "p"); return unwrap(firstP?.[1]); } -export function isToSelf(e: Readonly, pk: HexKey) { +export function isToSelf(e: Readonly, pk: HexKey) { return e.pubkey === pk && dmTo(e) === pk; } -export function dmsInChat(dms: readonly RawEvent[], pk: HexKey) { +export function dmsInChat(dms: readonly NostrEvent[], pk: HexKey) { return dms.filter(a => a.pubkey === pk || dmTo(a) === pk); } -export function totalUnread(dms: RawEvent[], myPubKey: HexKey) { +export function totalUnread(dms: NostrEvent[], myPubKey: HexKey) { return extractChats(dms, myPubKey).reduce((acc, v) => (acc += v.unreadMessages), 0); } -function unreadDms(dms: RawEvent[], myPubKey: HexKey, pk: HexKey) { +function unreadDms(dms: NostrEvent[], myPubKey: HexKey, pk: HexKey) { if (pk === myPubKey) return 0; const lastRead = lastReadDm(pk); return dmsInChat(dms, pk).filter(a => a.created_at >= lastRead && a.pubkey !== myPubKey).length; } -function newestMessage(dms: readonly RawEvent[], myPubKey: HexKey, pk: HexKey) { +function newestMessage(dms: readonly NostrEvent[], myPubKey: HexKey, pk: HexKey) { if (pk === myPubKey) { return dmsInChat( dms.filter(d => isToSelf(d, myPubKey)), @@ -196,11 +196,11 @@ function newestMessage(dms: readonly RawEvent[], myPubKey: HexKey, pk: HexKey) { return dmsInChat(dms, pk).reduce((acc, v) => (acc = v.created_at > acc ? v.created_at : acc), 0); } -export function dmsForLogin(dms: readonly RawEvent[], myPubKey: HexKey) { +export function dmsForLogin(dms: readonly NostrEvent[], myPubKey: HexKey) { return dms.filter(a => a.pubkey === myPubKey || (a.pubkey !== myPubKey && dmTo(a) === myPubKey)); } -export function extractChats(dms: RawEvent[], myPubKey: HexKey) { +export function extractChats(dms: NostrEvent[], myPubKey: HexKey) { const myDms = dmsForLogin(dms, myPubKey); const keys = myDms.map(a => [a.pubkey, dmTo(a)]).flat(); const filteredKeys = dedupe(keys); diff --git a/packages/app/src/SnortUtils/index.ts b/packages/app/src/SnortUtils/index.ts index 429808667..5ff04ed45 100644 --- a/packages/app/src/SnortUtils/index.ts +++ b/packages/app/src/SnortUtils/index.ts @@ -15,7 +15,7 @@ import { NostrPrefix, decodeTLV, TLVEntryType, - RawEvent, + NostrEvent, } from "System"; import { MetadataCache } from "Cache"; import NostrLink from "Element/NostrLink"; @@ -482,7 +482,7 @@ export function chunks(arr: T[], length: number) { return result; } -export function findTag(e: RawEvent, tag: string) { +export function findTag(e: NostrEvent, tag: string) { const maybeTag = e.tags.find(evTag => { return evTag[0] === tag; }); diff --git a/packages/app/src/State/NoteCreator.ts b/packages/app/src/State/NoteCreator.ts index e4941bb63..f0bf02fdf 100644 --- a/packages/app/src/State/NoteCreator.ts +++ b/packages/app/src/State/NoteCreator.ts @@ -1,19 +1,19 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; -import { RawEvent, TaggedRawEvent } from "System"; +import { NostrEvent, TaggedRawEvent } from "System"; interface NoteCreatorStore { show: boolean; note: string; error: string; active: boolean; - preview?: RawEvent; + preview?: NostrEvent; replyTo?: TaggedRawEvent; showAdvanced: boolean; selectedCustomRelays: false | Array; zapForward: string; sensitive: string; pollOptions?: Array; - otherEvents: Array; + otherEvents: Array; } const InitState: NoteCreatorStore = { @@ -44,7 +44,7 @@ const NoteCreatorSlice = createSlice({ setActive: (state, action: PayloadAction) => { state.active = action.payload; }, - setPreview: (state, action: PayloadAction) => { + setPreview: (state, action: PayloadAction) => { state.preview = action.payload; }, setReplyTo: (state, action: PayloadAction) => { @@ -65,7 +65,7 @@ const NoteCreatorSlice = createSlice({ setPollOptions: (state, action: PayloadAction | undefined>) => { state.pollOptions = action.payload; }, - setOtherEvents: (state, action: PayloadAction>) => { + setOtherEvents: (state, action: PayloadAction>) => { state.otherEvents = action.payload; }, reset: () => InitState, diff --git a/packages/app/src/State/ReBroadcast.ts b/packages/app/src/State/ReBroadcast.ts index 97b12d577..76d0165dc 100644 --- a/packages/app/src/State/ReBroadcast.ts +++ b/packages/app/src/State/ReBroadcast.ts @@ -1,10 +1,10 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; -import { RawEvent } from "System"; +import { NostrEvent } from "System"; interface ReBroadcastStore { show: boolean; selectedCustomRelays: false | Array; - note?: RawEvent; + note?: NostrEvent; } const InitState: ReBroadcastStore = { @@ -19,7 +19,7 @@ const ReBroadcastSlice = createSlice({ setShow: (state, action: PayloadAction) => { state.show = action.payload; }, - setNote: (state, action: PayloadAction) => { + setNote: (state, action: PayloadAction) => { state.note = action.payload; }, setSelectedCustomRelays: (state, action: PayloadAction>) => { diff --git a/packages/app/src/System/Connection.ts b/packages/app/src/System/Connection.ts index 75e2832b7..109410e6a 100644 --- a/packages/app/src/System/Connection.ts +++ b/packages/app/src/System/Connection.ts @@ -2,12 +2,12 @@ import { v4 as uuid } from "uuid"; import { DefaultConnectTimeout } from "./Const"; import { ConnectionStats } from "./ConnectionStats"; -import { RawEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr"; +import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr"; import { RelayInfo } from "./RelayInfo"; import { unwrap } from "./Util"; import ExternalStore from "ExternalStore"; -export type AuthHandler = (challenge: string, relay: string) => Promise; +export type AuthHandler = (challenge: string, relay: string) => Promise; /** * Relay settings @@ -232,7 +232,7 @@ export class Connection extends ExternalStore { /** * Send event on this connection */ - SendEvent(e: RawEvent) { + SendEvent(e: NostrEvent) { if (!this.Settings.write) { return; } @@ -245,7 +245,7 @@ export class Connection extends ExternalStore { /** * Send event on this connection and wait for OK response */ - async SendAsync(e: RawEvent, timeout = 5000) { + async SendAsync(e: NostrEvent, timeout = 5000) { return new Promise(resolve => { if (!this.Settings.write) { resolve(); diff --git a/packages/app/src/System/EventBuilder.ts b/packages/app/src/System/EventBuilder.ts index ea7c1df81..73a285e19 100644 --- a/packages/app/src/System/EventBuilder.ts +++ b/packages/app/src/System/EventBuilder.ts @@ -1,4 +1,4 @@ -import { EventKind, HexKey, NostrPrefix, RawEvent } from "System"; +import { EventKind, HexKey, NostrPrefix, NostrEvent } from "System"; import { HashtagRegex } from "Const"; import { getPublicKey, parseNostrLink, unixNow } from "SnortUtils"; import { EventExt } from "./EventExt"; @@ -63,7 +63,7 @@ export class EventBuilder { kind: this.#kind, created_at: this.#createdAt ?? unixNow(), tags: this.#tags, - } as RawEvent; + } as NostrEvent; ev.id = EventExt.createId(ev); return ev; } diff --git a/packages/app/src/System/EventExt.ts b/packages/app/src/System/EventExt.ts index ca2b34ee2..ebbf0f154 100644 --- a/packages/app/src/System/EventExt.ts +++ b/packages/app/src/System/EventExt.ts @@ -1,6 +1,6 @@ import * as secp from "@noble/curves/secp256k1"; import * as utils from "@noble/curves/abstract/utils"; -import { EventKind, HexKey, RawEvent, Tag } from "System"; +import { EventKind, HexKey, NostrEvent, Tag } from "System"; import base64 from "@protobufjs/base64"; import { sha256, unixNow } from "SnortUtils"; @@ -15,7 +15,7 @@ export abstract class EventExt { /** * Get the pub key of the creator of this event NIP-26 */ - static getRootPubKey(e: RawEvent): HexKey { + static getRootPubKey(e: NostrEvent): HexKey { const delegation = e.tags.find(a => a[0] === "delegation"); if (delegation?.[1]) { return delegation[1]; @@ -26,7 +26,7 @@ export abstract class EventExt { /** * Sign this message with a private key */ - static async sign(e: RawEvent, key: HexKey) { + static async sign(e: NostrEvent, key: HexKey) { e.id = this.createId(e); const sig = await secp.schnorr.sign(e.id, key); @@ -40,13 +40,13 @@ export abstract class EventExt { * Check the signature of this message * @returns True if valid signature */ - static async verify(e: RawEvent) { + static async verify(e: NostrEvent) { const id = this.createId(e); const result = await secp.schnorr.verify(e.sig, id, e.pubkey); return result; } - static createId(e: RawEvent) { + static createId(e: NostrEvent) { const payload = [0, e.pubkey, e.created_at, e.kind, e.tags, e.content]; const hash = sha256(JSON.stringify(payload)); @@ -69,10 +69,10 @@ export abstract class EventExt { tags: [], id: "", sig: "", - } as RawEvent; + } as NostrEvent; } - static extractThread(ev: RawEvent) { + static extractThread(ev: NostrEvent) { const isThread = ev.tags.some(a => (a[0] === "e" && a[3] !== "mention") || a[0] == "a"); if (!isThread) { return undefined; diff --git a/packages/app/src/System/EventPublisher.ts b/packages/app/src/System/EventPublisher.ts index 5fecf80fb..8aa29bad3 100644 --- a/packages/app/src/System/EventPublisher.ts +++ b/packages/app/src/System/EventPublisher.ts @@ -5,7 +5,7 @@ import { FullRelaySettings, HexKey, Lists, - RawEvent, + NostrEvent, RelaySettings, SystemInterface, TaggedRawEvent, @@ -27,7 +27,7 @@ declare global { interface Window { nostr?: { getPublicKey: () => Promise; - signEvent: (event: T) => Promise; + signEvent: (event: T) => Promise; getRelays?: () => Promise>; @@ -113,7 +113,7 @@ export class EventPublisher { return await this.#sign(eb); } - broadcast(ev: RawEvent) { + broadcast(ev: NostrEvent) { console.debug(ev); this.#system.BroadcastEvent(ev); } @@ -123,7 +123,7 @@ export class EventPublisher { * If a user removes all the DefaultRelays from their relay list and saves that relay list, * When they open the site again we wont see that updated relay list and so it will appear to reset back to the previous state */ - broadcastForBootstrap(ev: RawEvent) { + broadcastForBootstrap(ev: NostrEvent) { for (const [k] of DefaultRelays) { this.#system.WriteOnceToRelay(k, ev); } @@ -132,7 +132,7 @@ export class EventPublisher { /** * Write event to all given relays. */ - broadcastAll(ev: RawEvent, relays: string[]) { + broadcastAll(ev: NostrEvent, relays: string[]) { for (const k of relays) { this.#system.WriteOnceToRelay(k, ev); } @@ -249,7 +249,7 @@ export class EventPublisher { return await this.#sign(eb); } - async react(evRef: RawEvent, content = "+") { + async react(evRef: NostrEvent, content = "+") { const eb = this.#eb(EventKind.Reaction); eb.content(content); eb.tag(["e", evRef.id]); @@ -298,14 +298,14 @@ export class EventPublisher { /** * Repost a note (NIP-18) */ - async repost(note: RawEvent) { + async repost(note: NostrEvent) { const eb = this.#eb(EventKind.Repost); eb.tag(["e", note.id, ""]); eb.tag(["p", note.pubkey]); return await this.#sign(eb); } - async decryptDm(note: RawEvent) { + async decryptDm(note: NostrEvent) { if (note.pubkey !== this.#pubKey && !note.tags.some(a => a[1] === this.#pubKey)) { throw new Error("Can't decrypt, DM does not belong to this user"); } diff --git a/packages/app/src/System/GossipModel.ts b/packages/app/src/System/GossipModel.ts index 3a6db6198..5cdd2d417 100644 --- a/packages/app/src/System/GossipModel.ts +++ b/packages/app/src/System/GossipModel.ts @@ -1,4 +1,4 @@ -import { FullRelaySettings, RawReqFilter } from "System"; +import { FullRelaySettings, ReqFilter } from "System"; import { unwrap } from "SnortUtils"; import debug from "debug"; @@ -6,19 +6,19 @@ const PickNRelays = 2; export interface RelayTaggedFilter { relay: string; - filter: RawReqFilter; + filter: ReqFilter; } export interface RelayTaggedFilters { relay: string; - filters: Array; + filters: Array; } export interface RelayCache { get(pubkey?: string): Array | undefined; } -export function splitAllByWriteRelays(cache: RelayCache, filters: Array) { +export function splitAllByWriteRelays(cache: RelayCache, filters: Array) { const allSplit = filters .map(a => splitByWriteRelays(cache, a)) .reduce((acc, v) => { @@ -31,7 +31,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array>()); + }, new Map>()); return [...allSplit.entries()].map(([k, v]) => { return { @@ -46,7 +46,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array { +export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array { if ((filter.authors?.length ?? 0) === 0) return [ { diff --git a/packages/app/src/System/Nostr.ts b/packages/app/src/System/Nostr.ts index 8e36ae3b4..86e665975 100644 --- a/packages/app/src/System/Nostr.ts +++ b/packages/app/src/System/Nostr.ts @@ -1,6 +1,6 @@ import { RelaySettings } from "./Connection"; -export type RawEvent = { +export interface NostrEvent { id: u256; pubkey: HexKey; created_at: number; @@ -8,9 +8,9 @@ export type RawEvent = { tags: Array>; content: string; sig: string; -}; +} -export interface TaggedRawEvent extends RawEvent { +export interface TaggedRawEvent extends NostrEvent { /** * A list of relays this event was seen on */ @@ -32,12 +32,12 @@ export type MaybeHexKey = HexKey | undefined; */ export type u256 = string; -export type ReqCommand = [cmd: "REQ", id: string, ...filters: Array]; +export type ReqCommand = [cmd: "REQ", id: string, ...filters: Array]; /** * Raw REQ filter object */ -export type RawReqFilter = { +export interface ReqFilter { ids?: u256[]; authors?: u256[]; kinds?: number[]; @@ -50,7 +50,7 @@ export type RawReqFilter = { since?: number; until?: number; limit?: number; -}; +} /** * Medatadata event content diff --git a/packages/app/src/System/NostrSystem.ts b/packages/app/src/System/NostrSystem.ts index 9af2e9b2c..16c9f14ba 100644 --- a/packages/app/src/System/NostrSystem.ts +++ b/packages/app/src/System/NostrSystem.ts @@ -2,7 +2,7 @@ import debug from "debug"; import { v4 as uuid } from "uuid"; import ExternalStore from "ExternalStore"; -import { RawEvent, RawReqFilter, TaggedRawEvent } from "./Nostr"; +import { NostrEvent, ReqFilter, TaggedRawEvent } from "./Nostr"; import { AuthHandler, Connection, RelaySettings, ConnectionStateSnapshot } from "./Connection"; import { Query, QueryBase } from "./Query"; import { RelayCache } from "./GossipModel"; @@ -194,7 +194,7 @@ export class NostrSystem extends ExternalStore implements System /** * Send events to writable relays */ - BroadcastEvent(ev: RawEvent) { + BroadcastEvent(ev: NostrEvent) { for (const [, s] of this.#sockets) { s.SendEvent(ev); } @@ -203,7 +203,7 @@ export class NostrSystem extends ExternalStore implements System /** * Write an event to a relay then disconnect */ - async WriteOnceToRelay(address: string, ev: RawEvent) { + async WriteOnceToRelay(address: string, ev: NostrEvent) { return new Promise((resolve, reject) => { const c = new Connection(address, { write: true, read: false }, this.HandleAuth, true); diff --git a/packages/app/src/System/Query.ts b/packages/app/src/System/Query.ts index 305ebe779..065eb7ba7 100644 --- a/packages/app/src/System/Query.ts +++ b/packages/app/src/System/Query.ts @@ -1,6 +1,6 @@ import { v4 as uuid } from "uuid"; import debug from "debug"; -import { Connection, RawReqFilter, Nips, TaggedRawEvent } from "System"; +import { Connection, ReqFilter, Nips, TaggedRawEvent } from "System"; import { unixNowMs, unwrap } from "SnortUtils"; import { NoteStore } from "./NoteCollection"; import { simpleMerge } from "./RequestMerger"; @@ -22,7 +22,7 @@ class QueryTrace { constructor( readonly relay: string, - readonly filters: Array, + readonly filters: Array, readonly connId: string, fnClose: (id: string) => void, fnProgress: () => void @@ -94,7 +94,7 @@ export interface QueryBase { /** * The query payload (REQ filters) */ - filters: Array; + filters: Array; /** * List of relays to send this query to diff --git a/packages/app/src/System/RequestBuilder.ts b/packages/app/src/System/RequestBuilder.ts index 2ee6e7efa..dfbbb8b5f 100644 --- a/packages/app/src/System/RequestBuilder.ts +++ b/packages/app/src/System/RequestBuilder.ts @@ -1,7 +1,8 @@ -import { RawReqFilter, u256, HexKey, EventKind } from "System"; +import { ReqFilter, u256, HexKey, EventKind } from "System"; import { appendDedupe, dedupe } from "SnortUtils"; import { diffFilters } from "./RequestSplitter"; import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel"; +import { mergeSimilar } from "./RequestMerger"; /** * Which strategy is used when building REQ filters @@ -28,7 +29,7 @@ export enum RequestStrategy { * A built REQ filter ready for sending to System */ export interface BuiltRawReqFilter { - filters: Array; + filters: Array; relay: string; strategy: RequestStrategy; } @@ -77,7 +78,7 @@ export class RequestBuilder { return this; } - buildRaw(): Array { + buildRaw(): Array { return this.#builders.map(f => f.filter); } @@ -91,11 +92,11 @@ export class RequestBuilder { * @param q All previous filters merged * @returns */ - buildDiff(relays: RelayCache, filters: Array): Array { + buildDiff(relays: RelayCache, filters: Array): Array { const next = this.buildRaw(); const diff = diffFilters(filters, next); if (diff.changed) { - return splitAllByWriteRelays(relays, diff.filters).map(a => { + return splitAllByWriteRelays(relays, diff.added).map(a => { return { strategy: RequestStrategy.AuthorsRelays, filters: a.filters, @@ -124,7 +125,7 @@ export class RequestBuilder { const filtersSquashed = [...relayMerged.values()].map(a => { return { - filters: a.flatMap(b => b.filters), + filters: mergeSimilar(a.flatMap(b => b.filters)), relay: a[0].relay, strategy: a[0].strategy, } as BuiltRawReqFilter; @@ -138,7 +139,7 @@ export class RequestBuilder { * Builder class for a single request filter */ export class RequestFilterBuilder { - #filter: RawReqFilter = {}; + #filter: ReqFilter = {}; #relayHints = new Map>(); get filter() { diff --git a/packages/app/src/System/RequestExpander.test.ts b/packages/app/src/System/RequestExpander.test.ts new file mode 100644 index 000000000..0ccb58d15 --- /dev/null +++ b/packages/app/src/System/RequestExpander.test.ts @@ -0,0 +1,34 @@ +import { expandFilter } from "./RequestExpander"; + +describe("RequestExpander", () => { + test("expand filter", () => { + const a = { + authors: ["a", "b", "c"], + kinds: [1, 2, 3], + ids: ["x", "y"], + "#p": ["a"], + since: 99, + limit: 10, + }; + expect(expandFilter(a)).toEqual([ + { authors: "a", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "a", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "b", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 1, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 1, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 2, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 2, ids: "y", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 3, ids: "x", "#p": "a", since: 99, limit: 10 }, + { authors: "c", kinds: 3, ids: "y", "#p": "a", since: 99, limit: 10 }, + ]); + }); +}); diff --git a/packages/app/src/System/RequestExpander.ts b/packages/app/src/System/RequestExpander.ts new file mode 100644 index 000000000..8aff42386 --- /dev/null +++ b/packages/app/src/System/RequestExpander.ts @@ -0,0 +1,48 @@ +import { u256, ReqFilter } from "./Nostr"; + +export interface FlatReqFilter { + ids?: u256; + authors?: u256; + kinds?: number; + "#e"?: u256; + "#p"?: u256; + "#t"?: string; + "#d"?: string; + "#r"?: string; + search?: string; + since?: number; + until?: number; + limit?: number; +} + +/** + * Expand a filter into its most fine grained form + */ +export function expandFilter(f: ReqFilter): Array { + const ret: Array = []; + const src = Object.entries(f); + const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]); + const props = src.filter(([, v]) => !Array.isArray(v)); + + function generateCombinations(index: number, currentCombination: FlatReqFilter) { + if (index === keys.length) { + ret.push(currentCombination); + return; + } + + const key = keys[index]; + const values = (f as Record>)[key]; + + for (let i = 0; i < values.length; i++) { + const value = values[i]; + const updatedCombination = { ...currentCombination, [key]: value }; + generateCombinations(index + 1, updatedCombination); + } + } + + generateCombinations(0, { + ...Object.fromEntries(props), + }); + + return ret; +} diff --git a/packages/app/src/System/RequestMatcher.ts b/packages/app/src/System/RequestMatcher.ts index e8ffc4504..8845702d9 100644 --- a/packages/app/src/System/RequestMatcher.ts +++ b/packages/app/src/System/RequestMatcher.ts @@ -1,6 +1,6 @@ -import { RawEvent, RawReqFilter } from "./Nostr"; +import { NostrEvent, ReqFilter } from "./Nostr"; -export function eventMatchesFilter(ev: RawEvent, filter: RawReqFilter) { +export function eventMatchesFilter(ev: NostrEvent, filter: ReqFilter) { if (!(filter.ids?.includes(ev.id) ?? false)) { return false; } diff --git a/packages/app/src/System/RequestMerger.test.ts b/packages/app/src/System/RequestMerger.test.ts index 2de15e7dd..462607897 100644 --- a/packages/app/src/System/RequestMerger.test.ts +++ b/packages/app/src/System/RequestMerger.test.ts @@ -1,17 +1,19 @@ -import { RawReqFilter } from "System"; -import { filterIncludes, mergeSimilar, simpleMerge } from "./RequestMerger"; +import { ReqFilter } from "System"; +import { filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "./RequestMerger"; +import { FlatReqFilter, expandFilter } from "./RequestExpander"; +import { distance } from "./Util"; describe("RequestMerger", () => { it("should simple merge authors", () => { const a = { authors: ["a"], - } as RawReqFilter; + } as ReqFilter; const b = { authors: ["b"], - } as RawReqFilter; + } as ReqFilter; const merged = mergeSimilar([a, b]); - expect(merged).toMatchObject([ + expect(merged).toEqual([ { authors: ["a", "b"], }, @@ -21,17 +23,17 @@ describe("RequestMerger", () => { it("should append non-mergable filters", () => { const a = { authors: ["a"], - } as RawReqFilter; + } as ReqFilter; const b = { authors: ["b"], - } as RawReqFilter; + } as ReqFilter; const c = { limit: 5, authors: ["a"], }; const merged = mergeSimilar([a, b, c]); - expect(merged).toMatchObject([ + expect(merged).toEqual([ { authors: ["a", "b"], }, @@ -46,11 +48,11 @@ describe("RequestMerger", () => { const bigger = { authors: ["a", "b", "c"], since: 99, - } as RawReqFilter; + } as ReqFilter; const smaller = { authors: ["c"], since: 100, - } as RawReqFilter; + } as ReqFilter; expect(filterIncludes(bigger, smaller)).toBe(true); }); @@ -58,14 +60,50 @@ describe("RequestMerger", () => { const a = { authors: ["a", "b", "c"], since: 99, - } as RawReqFilter; + } as ReqFilter; const b = { authors: ["c", "d", "e"], since: 100, - } as RawReqFilter; + } as ReqFilter; expect(simpleMerge([a, b])).toEqual({ authors: ["a", "b", "c", "d", "e"], since: 100, }); }); }); + +describe("flatMerge", () => { + it("should flat merge simple", () => { + const input = [ + { ids: 0, authors: "a" }, + { ids: 0, authors: "b" }, + { kinds: 1 }, + { kinds: 2 }, + { ids: 0, authors: "c" }, + { authors: "c", kinds: 1 }, + { authors: "c", limit: 100 }, + { ids: 1, authors: "c" }, + ] as Array; + const output = [ + { ids: [0], authors: ["a", "b", "c"] }, + { kinds: [1, 2] }, + { authors: ["c"], kinds: [1] }, + { authors: ["c"], limit: 100 }, + { ids: [1], authors: ["c"] }, + ] as Array; + + expect(flatMerge(input)).toEqual(output); + }); + + it("should expand and flat merge complex same", () => { + const input = [ + { kinds: [1, 6969, 6], authors: ["kieran", "snort", "c", "d", "e"], since: 1, until: 100 }, + { kinds: [4], authors: ["kieran"] }, + { kinds: [4], "#p": ["kieran"] }, + { kinds: [1000], authors: ["snort"], "#p": ["kieran"] }, + ] as Array; + + const dut = flatMerge(input.flatMap(expandFilter).sort(() => (Math.random() > 0.5 ? 1 : -1))); + expect(dut.every(a => input.some(b => distance(b, a) === 0))).toEqual(true); + }); +}); diff --git a/packages/app/src/System/RequestMerger.ts b/packages/app/src/System/RequestMerger.ts index a33f0226b..acc359ab5 100644 --- a/packages/app/src/System/RequestMerger.ts +++ b/packages/app/src/System/RequestMerger.ts @@ -1,12 +1,40 @@ -import { RawReqFilter } from "System"; +import { ReqFilter } from "System"; +import { FlatReqFilter } from "./RequestExpander"; +import { distance } from "./Util"; -export function mergeSimilar(filters: Array): Array { - const hasCriticalKeySet = (a: RawReqFilter) => { - return a.limit !== undefined || a.since !== undefined || a.until !== undefined; - }; - const canEasilyMerge = filters.filter(a => !hasCriticalKeySet(a)); - const cannotMerge = filters.filter(a => hasCriticalKeySet(a)); - return [...(canEasilyMerge.length > 0 ? [simpleMerge(canEasilyMerge)] : []), ...cannotMerge]; +/** + * Keys which can change the entire meaning of the filter outside the array types + */ +const DiscriminatorKeys = ["since", "until", "limit", "search"]; + +export function canMergeFilters(a: any, b: any): boolean { + for (const key of DiscriminatorKeys) { + if (key in a || key in b) { + if (a[key] !== b[key]) { + return false; + } + } + } + + return true; +} + +export function mergeSimilar(filters: Array): Array { + const ret = []; + + while (filters.length > 0) { + const current = filters.shift()!; + const mergeSet = [current]; + for (let i = 0; i < filters.length; i++) { + const f = filters[i]; + if (mergeSet.every(v => canMergeFilters(v, f) && distance(v, f) === 1)) { + mergeSet.push(filters.splice(i, 1)[0]); + i--; + } + } + ret.push(simpleMerge(mergeSet)); + } + return ret; } /** @@ -14,7 +42,7 @@ export function mergeSimilar(filters: Array): Array * @param filters * @returns */ -export function simpleMerge(filters: Array) { +export function simpleMerge(filters: Array) { const result: any = {}; filters.forEach(filter => { @@ -31,7 +59,7 @@ export function simpleMerge(filters: Array) { }); }); - return result as RawReqFilter; + return result as ReqFilter; } /** @@ -40,7 +68,7 @@ export function simpleMerge(filters: Array) { * @param smaller * @returns */ -export function filterIncludes(bigger: RawReqFilter, smaller: RawReqFilter) { +export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) { const outside = bigger as Record | number>; for (const [k, v] of Object.entries(smaller)) { if (outside[k] === undefined) { @@ -61,3 +89,61 @@ export function filterIncludes(bigger: RawReqFilter, smaller: RawReqFilter) { } return true; } + +/** + * Merge expanded flat filters into combined concise filters + * @param all + * @returns + */ +export function flatMerge(all: Array): Array { + let ret: Array = []; + + // to compute filters which can be merged we need to calucate the distance change between each filter + // then we can merge filters which are exactly 1 change diff from each other + + function mergeFiltersInSet(filters: Array) { + const result: any = {}; + + filters.forEach(f => { + const filter = f as Record; + Object.entries(filter).forEach(([key, value]) => { + if (!DiscriminatorKeys.includes(key)) { + if (result[key] === undefined) { + result[key] = [value]; + } else { + result[key] = [...new Set([...result[key], value])]; + } + } else { + result[key] = value; + } + }); + }); + + return result as ReqFilter; + } + + // reducer, kinda verbose + while (all.length > 0) { + const currentFilter = all.shift()!; + const mergeSet = [currentFilter]; + + for (let i = 0; i < all.length; i++) { + const f = all[i]; + + if (mergeSet.every(a => canMergeFilters(a, f) && distance(a, f) === 1)) { + mergeSet.push(all.splice(i, 1)[0]); + i--; + } + } + ret.push(mergeFiltersInSet(mergeSet)); + } + + while (true) { + const n = mergeSimilar([...ret]); + if (n.length === ret.length) { + break; + } + ret = n; + } + return ret; +} diff --git a/packages/app/src/System/RequestSplitter.test.ts b/packages/app/src/System/RequestSplitter.test.ts index 10e78a6bd..e6caa5a75 100644 --- a/packages/app/src/System/RequestSplitter.test.ts +++ b/packages/app/src/System/RequestSplitter.test.ts @@ -1,104 +1,87 @@ -import { RawReqFilter } from "System"; +import { ReqFilter } from "System"; import { describe, expect } from "@jest/globals"; -import { diffFilters, expandFilter } from "./RequestSplitter"; +import { diffFilters } from "./RequestSplitter"; describe("RequestSplitter", () => { test("single filter add value", () => { - const a: Array = [{ kinds: [0], authors: ["a"] }]; - const b: Array = [{ kinds: [0], authors: ["a", "b"] }]; + const a: Array = [{ kinds: [0], authors: ["a"] }]; + const b: Array = [{ kinds: [0], authors: ["a", "b"] }]; const diff = diffFilters(a, b); - expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["b"] }], changed: true }); + expect(diff).toEqual({ + added: [{ kinds: [0], authors: ["b"] }], + removed: [], + changed: true, + }); }); test("single filter remove value", () => { - const a: Array = [{ kinds: [0], authors: ["a"] }]; - const b: Array = [{ kinds: [0], authors: ["b"] }]; + const a: Array = [{ kinds: [0], authors: ["a"] }]; + const b: Array = [{ kinds: [0], authors: ["b"] }]; const diff = diffFilters(a, b); - expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["b"] }], changed: true }); + expect(diff).toEqual({ + added: [{ kinds: [0], authors: ["b"] }], + removed: [{ kinds: [0], authors: ["a"] }], + changed: true, + }); }); test("single filter change critical key", () => { - const a: Array = [{ kinds: [0], authors: ["a"], since: 100 }]; - const b: Array = [{ kinds: [0], authors: ["a", "b"], since: 101 }]; + const a: Array = [{ kinds: [0], authors: ["a"], since: 100 }]; + const b: Array = [{ kinds: [0], authors: ["a", "b"], since: 101 }]; const diff = diffFilters(a, b); - expect(diff).toEqual({ filters: [{ kinds: [0], authors: ["a", "b"], since: 101 }], changed: true }); + expect(diff).toEqual({ + added: [{ kinds: [0], authors: ["a", "b"], since: 101 }], + removed: [{ kinds: [0], authors: ["a"], since: 100 }], + changed: true, + }); }); test("multiple filter add value", () => { - const a: Array = [ + const a: Array = [ { kinds: [0], authors: ["a"] }, { kinds: [69], authors: ["a"] }, ]; - const b: Array = [ + const b: Array = [ { kinds: [0], authors: ["a", "b"] }, { kinds: [69], authors: ["a", "c"] }, ]; const diff = diffFilters(a, b); expect(diff).toEqual({ - filters: [ + added: [ { kinds: [0], authors: ["b"] }, { kinds: [69], authors: ["c"] }, ], + removed: [], changed: true, }); }); test("multiple filter remove value", () => { - const a: Array = [ + const a: Array = [ { kinds: [0], authors: ["a"] }, { kinds: [69], authors: ["a"] }, ]; - const b: Array = [ + const b: Array = [ { kinds: [0], authors: ["b"] }, { kinds: [69], authors: ["c"] }, ]; const diff = diffFilters(a, b); expect(diff).toEqual({ - filters: [ + added: [ { kinds: [0], authors: ["b"] }, { kinds: [69], authors: ["c"] }, ], + removed: [{ kinds: [0, 69], authors: ["a"] }], changed: true, }); }); test("add filter", () => { - const a: Array = [{ kinds: [0], authors: ["a"] }]; - const b: Array = [ + const a: Array = [{ kinds: [0], authors: ["a"] }]; + const b: Array = [ { kinds: [0], authors: ["a"] }, { kinds: [69], authors: ["c"] }, ]; const diff = diffFilters(a, b); expect(diff).toEqual({ - filters: [ - { kinds: [0], authors: ["a"] }, - { kinds: [69], authors: ["c"] }, - ], + added: [{ kinds: [69], authors: ["c"] }], + removed: [], changed: true, }); }); - test("expand filter", () => { - const a = { - authors: ["a", "b", "c"], - kinds: [1, 2, 3], - ids: ["x", "y"], - since: 99, - limit: 10, - }; - expect(expandFilter(a)).toEqual([ - { authors: ["a"], kinds: [1], ids: ["x"], since: 99, limit: 10 }, - { authors: ["a"], kinds: [1], ids: ["y"], since: 99, limit: 10 }, - { authors: ["a"], kinds: [2], ids: ["x"], since: 99, limit: 10 }, - { authors: ["a"], kinds: [2], ids: ["y"], since: 99, limit: 10 }, - { authors: ["a"], kinds: [3], ids: ["x"], since: 99, limit: 10 }, - { authors: ["a"], kinds: [3], ids: ["y"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [1], ids: ["x"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [1], ids: ["y"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [2], ids: ["x"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [2], ids: ["y"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [3], ids: ["x"], since: 99, limit: 10 }, - { authors: ["b"], kinds: [3], ids: ["y"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [1], ids: ["x"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [1], ids: ["y"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [2], ids: ["x"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [2], ids: ["y"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [3], ids: ["x"], since: 99, limit: 10 }, - { authors: ["c"], kinds: [3], ids: ["y"], since: 99, limit: 10 }, - ]); - }); }); diff --git a/packages/app/src/System/RequestSplitter.ts b/packages/app/src/System/RequestSplitter.ts index 2c942ff9e..e1a3d6532 100644 --- a/packages/app/src/System/RequestSplitter.ts +++ b/packages/app/src/System/RequestSplitter.ts @@ -1,76 +1,18 @@ -import { RawReqFilter } from "System"; +import { ReqFilter } from "System"; +import { deepEqual } from "./Util"; +import { expandFilter } from "./RequestExpander"; +import { flatMerge } from "./RequestMerger"; -// Critical keys changing means the entire filter has changed -export const CriticalKeys = ["since", "until", "limit"]; +export function diffFilters(prev: Array, next: Array) { + const prevExpanded = prev.flatMap(expandFilter); + const nextExpanded = next.flatMap(expandFilter); -export function diffFilters(a: Array, b: Array) { - const result: Array = []; - let anyChanged = false; - for (const [i, bN] of b.entries()) { - const prev: Record = a[i]; - if (!prev) { - result.push(bN); - anyChanged = true; - } else { - let anyCriticalKeyChanged = false; - for (const [k, v] of Object.entries(bN)) { - if (Array.isArray(v)) { - const prevArray = prev[k] as Array | undefined; - const thisArray = v as Array; - const added = thisArray.filter(a => !prevArray?.includes(a)); - // support adding new values to array, removing values is ignored since we only care about getting new values - result[i] = { ...result[i], [k]: added.length === 0 ? prevArray ?? [] : added }; - if (added.length > 0) { - anyChanged = true; - } - } else if (prev[k] !== v) { - result[i] = { ...result[i], [k]: v }; - if (CriticalKeys.includes(k)) { - anyCriticalKeyChanged = anyChanged = true; - break; - } - } - } - if (anyCriticalKeyChanged) { - result[i] = bN; - } - } - } + const added = flatMerge(nextExpanded.filter(a => !prevExpanded.some(b => deepEqual(a, b)))); + const removed = flatMerge(prevExpanded.filter(a => !nextExpanded.some(b => deepEqual(a, b)))); return { - filters: result, - changed: anyChanged, + added, + removed, + changed: added.length > 0 || removed.length > 0, }; } - -/** - * Expand a filter into its most fine grained form - */ -export function expandFilter(f: RawReqFilter): Array { - const ret: Array = []; - const src = Object.entries(f); - const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]); - const props = src.filter(([, v]) => !Array.isArray(v)); - - function generateCombinations(index: number, currentCombination: RawReqFilter) { - if (index === keys.length) { - ret.push(currentCombination); - return; - } - - const key = keys[index]; - const values = (f as Record>)[key]; - - for (let i = 0; i < values.length; i++) { - const value = values[i]; - const updatedCombination = { ...currentCombination, [key]: [value] }; - generateCombinations(index + 1, updatedCombination); - } - } - - generateCombinations(0, { - ...Object.fromEntries(props), - }); - - return ret; -} diff --git a/packages/app/src/System/SystemWorker.ts b/packages/app/src/System/SystemWorker.ts index eae78c983..fb80e02aa 100644 --- a/packages/app/src/System/SystemWorker.ts +++ b/packages/app/src/System/SystemWorker.ts @@ -2,7 +2,7 @@ import ExternalStore from "ExternalStore"; import { NoteStore, Query, - RawEvent, + NostrEvent, RelaySettings, RequestBuilder, SystemSnapshot, @@ -51,11 +51,11 @@ export class SystemWorker extends ExternalStore implements Syste throw new Error("Method not implemented."); } - BroadcastEvent(ev: RawEvent): void { + BroadcastEvent(ev: NostrEvent): void { throw new Error("Method not implemented."); } - WriteOnceToRelay(relay: string, ev: RawEvent): Promise { + WriteOnceToRelay(relay: string, ev: NostrEvent): Promise { throw new Error("Method not implemented."); } diff --git a/packages/app/src/System/Util.test.ts b/packages/app/src/System/Util.test.ts new file mode 100644 index 000000000..14812019d --- /dev/null +++ b/packages/app/src/System/Util.test.ts @@ -0,0 +1,72 @@ +import { distance } from "./Util"; + +describe("distance", () => { + it("should have 0 distance", () => { + const a = { + ids: "a", + }; + const b = { + ids: "a", + }; + expect(distance(a, b)).toEqual(0); + }); + it("should have 1 distance", () => { + const a = { + ids: "a", + }; + const b = { + ids: "b", + }; + expect(distance(a, b)).toEqual(1); + }); + it("should have 10 distance", () => { + const a = { + ids: "a", + }; + const b = { + ids: "a", + kinds: 1, + }; + expect(distance(a, b)).toEqual(10); + }); + it("should have 11 distance", () => { + const a = { + ids: "a", + }; + const b = { + ids: "b", + kinds: 1, + }; + expect(distance(a, b)).toEqual(11); + }); + it("should have 1 distance, arrays", () => { + const a = { + since: 1, + until: 100, + kinds: [1], + authors: ["kieran", "snort", "c", "d", "e"], + }; + const b = { + since: 1, + until: 100, + kinds: [6969], + authors: ["kieran", "snort", "c", "d", "e"], + }; + expect(distance(a, b)).toEqual(1); + }); + it("should have 1 distance, array change extra", () => { + const a = { + since: 1, + until: 100, + kinds: [1], + authors: ["f", "kieran", "snort", "c", "d"], + }; + const b = { + since: 1, + until: 100, + kinds: [1], + authors: ["kieran", "snort", "c", "d", "e"], + }; + expect(distance(a, b)).toEqual(1); + }); +}); diff --git a/packages/app/src/System/Util.ts b/packages/app/src/System/Util.ts index 0e4178148..c56987391 100644 --- a/packages/app/src/System/Util.ts +++ b/packages/app/src/System/Util.ts @@ -40,3 +40,47 @@ export function unixNow() { export function unixNowMs() { return new Date().getTime(); } + +export function deepEqual(x: any, y: any): boolean { + const ok = Object.keys, + tx = typeof x, + ty = typeof y; + + return x && y && tx === "object" && tx === ty + ? ok(x).length === ok(y).length && ok(x).every(key => deepEqual(x[key], y[key])) + : x === y; +} + +/** + * Compute the "distance" between two objects by comparing their difference in properties + * Missing/Added keys result in +10 distance + * This is not recursive + */ +export function distance(a: any, b: any): number { + const keys1 = Object.keys(a); + const keys2 = Object.keys(b); + const maxKeys = keys1.length > keys2.length ? keys1 : keys2; + + let distance = 0; + for (const key of maxKeys) { + if (key in a && key in b) { + if (Array.isArray(a[key]) && Array.isArray(b[key])) { + const aa = a[key] as Array; + const bb = b[key] as Array; + if (aa.length === bb.length) { + if (aa.some(v => !bb.includes(v))) { + distance++; + } + } else { + distance++; + } + } else if (a[key] !== b[key]) { + distance++; + } + } else { + distance += 10; + } + } + + return distance; +} diff --git a/packages/app/src/System/index.ts b/packages/app/src/System/index.ts index 1bb1380ef..46be1f899 100644 --- a/packages/app/src/System/index.ts +++ b/packages/app/src/System/index.ts @@ -9,7 +9,7 @@ import { ReplaceableNoteStore, } from "./NoteCollection"; import { Query } from "./Query"; -import { RawEvent, RawReqFilter } from "./Nostr"; +import { NostrEvent, ReqFilter } from "./Nostr"; export * from "./NostrSystem"; export { default as EventKind } from "./EventKind"; @@ -29,15 +29,15 @@ export interface SystemInterface { Query(type: { new (): T }, req: RequestBuilder | null): Query | undefined; ConnectToRelay(address: string, options: RelaySettings): Promise; DisconnectRelay(address: string): void; - BroadcastEvent(ev: RawEvent): void; - WriteOnceToRelay(relay: string, ev: RawEvent): Promise; + BroadcastEvent(ev: NostrEvent): void; + WriteOnceToRelay(relay: string, ev: NostrEvent): Promise; } export interface SystemSnapshot { queries: Array<{ id: string; - filters: Array; - subFilters: Array; + filters: Array; + subFilters: Array; closing: boolean; }>; } diff --git a/packages/app/src/Upload/index.ts b/packages/app/src/Upload/index.ts index 9ea2cbcc6..8cf575868 100644 --- a/packages/app/src/Upload/index.ts +++ b/packages/app/src/Upload/index.ts @@ -1,5 +1,5 @@ import useLogin from "Hooks/useLogin"; -import { RawEvent } from "System"; +import { NostrEvent } from "System"; import NostrBuild from "Upload/NostrBuild"; import VoidCat from "Upload/VoidCat"; @@ -14,7 +14,7 @@ export interface UploadResult { /** * NIP-94 File Header */ - header?: RawEvent; + header?: NostrEvent; } /** diff --git a/packages/app/src/Wallet/NostrWalletConnect.ts b/packages/app/src/Wallet/NostrWalletConnect.ts index b447e81d5..3b4abe53d 100644 --- a/packages/app/src/Wallet/NostrWalletConnect.ts +++ b/packages/app/src/Wallet/NostrWalletConnect.ts @@ -1,4 +1,4 @@ -import { Connection, EventKind, RawEvent } from "System"; +import { Connection, EventKind, NostrEvent } from "System"; import { EventBuilder } from "System"; import { EventExt } from "System/EventExt"; import { LNWallet, WalletError, WalletErrorCode, WalletInfo, WalletInvoice, WalletInvoiceState } from "Wallet"; @@ -123,7 +123,7 @@ export class NostrConnectWallet implements LNWallet { return Promise.resolve([]); } - async #onReply(sub: string, e: RawEvent) { + async #onReply(sub: string, e: NostrEvent) { if (sub === "info") { const pending = this.#commandQueue.get("info"); if (!pending) {