Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize

Browse Source
This commit is contained in:
Kieran
2023-06-12 14:15:45 +01:00
parent ae655dfc69
commit 7bfe2f0c91
22 changed files with 221 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/system/src/Connection.ts
View File

@ -4,7 +4,7 @@ import { DefaultConnectTimeout } from "./Const";
import { ConnectionStats } from "./ConnectionStats";
import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
import { RelayInfo } from "./RelayInfo";
import { unwrap } from "./Util";
import { unwrap } from "./Utils";
import ExternalStore from "./ExternalStore";
export type AuthHandler = (challenge: string, relay: string) => Promise<NostrEvent | undefined>;

2
packages/system/src/EventBuilder.ts
View File

@ -1,6 +1,6 @@
import { EventKind, HexKey, NostrPrefix, NostrEvent } from ".";
import { HashtagRegex } from "./Const";
import { getPublicKey, unixNow } from "./Util";
import { getPublicKey, unixNow } from "./Utils";
import { EventExt } from "./EventExt";
import { parseNostrLink } from "./NostrLink";

2
packages/system/src/EventExt.ts
View File

@ -2,7 +2,7 @@ import * as secp from "@noble/curves/secp256k1";
import * as utils from "@noble/curves/abstract/utils";
import { EventKind, HexKey, NostrEvent, Tag } from ".";
import base64 from "@protobufjs/base64";
import { sha256, unixNow } from "./Util";
import { sha256, unixNow } from "./Utils";
export interface Thread {
root?: Tag;

2
packages/system/src/EventPublisher.ts
View File

@ -13,7 +13,7 @@ import {
UserMetadata,
} from ".";
import { unwrap } from "./Util";
import { unwrap } from "./Utils";
import { EventBuilder } from "./EventBuilder";
import { EventExt } from "./EventExt";
import { barrierQueue, processWorkQueue, WorkQueueItem } from "./WorkQueue";

2
packages/system/src/GossipModel.ts
View File

@ -1,5 +1,5 @@
import { FullRelaySettings, ReqFilter } from ".";
import { unwrap } from "./Util";
import { unwrap } from "./Utils";
import debug from "debug";
const PickNRelays = 2;

2
packages/system/src/NostrLink.ts
View File

@ -1,4 +1,4 @@
import { bech32ToHex, hexToBech32 } from "./Util";
import { bech32ToHex, hexToBech32 } from "./Utils";
import { NostrPrefix, decodeTLV, TLVEntryType } from ".";
export interface NostrLink {

12
packages/system/src/NostrSystem.ts
View File

@ -7,7 +7,7 @@ import { Query } from "./Query";
import { RelayCache } from "./GossipModel";
import { NoteStore } from "./NoteCollection";
import { BuiltRawReqFilter, RequestBuilder } from "./RequestBuilder";
import { unwrap, sanitizeRelayUrl } from "./Util";
import { unwrap, sanitizeRelayUrl } from "./Utils";
import { SystemInterface, SystemSnapshot } from ".";
/**
@ -122,15 +122,13 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
const existing = this.Queries.get(req.id);
if (existing) {
const filters = !req.options?.skipDiff
? req.buildDiff(this.#relayCache, existing.filters)
? req.buildDiff(this.#relayCache, existing.flatFilters)
: req.build(this.#relayCache);
if (filters.length === 0 && !!req.options?.skipDiff) {
return existing;
} else {
for (const subQ of filters) {
this.SendQuery(existing, subQ).then(qta =>
qta.forEach(v => this.#log("New QT from diff %s %s %O from: %O", req.id, v.id, v.filters, existing.filters))
);
this.SendQuery(existing, subQ);
}
this.notifyChange();
return existing;
@ -142,9 +140,7 @@ export class NostrSystem extends ExternalStore<SystemSnapshot> implements System
const q = new Query(req.id, store, req.options?.leaveOpen);
this.Queries.set(req.id, q);
for (const subQ of filters) {
this.SendQuery(q, subQ).then(qta =>
qta.forEach(v => this.#log("New QT from diff %s %s %O", req.id, v.id, v.filters))
);
this.SendQuery(q, subQ);
}
this.notifyChange();
return q;

2
packages/system/src/NoteCollection.ts
View File

@ -1,5 +1,5 @@
import { TaggedRawEvent, u256 } from ".";
import { appendDedupe, findTag } from "./Util";
import { appendDedupe, findTag } from "./Utils";
export interface StoreSnapshot<TSnapshot> {
data: TSnapshot | undefined;

2
packages/system/src/ProfileCache.ts
View File

@ -1,7 +1,7 @@
import { EventKind, HexKey, SystemInterface, TaggedRawEvent, PubkeyReplaceableNoteStore, RequestBuilder } from ".";
import { ProfileCacheExpire } from "./Const";
import { CacheStore, mapEventToProfile, MetadataCache } from "./cache";
import { unixNowMs } from "./Util";
import { unixNowMs } from "./Utils";
import debug from "debug";
export class ProfileLoaderService {

16
packages/system/src/Query.ts
View File

@ -1,7 +1,7 @@
import { v4 as uuid } from "uuid";
import debug from "debug";
import { Connection, ReqFilter, Nips, TaggedRawEvent } from ".";
import { unixNowMs, unwrap } from "./Util";
import { unixNowMs, unwrap } from "./Utils";
import { NoteStore } from "./NoteCollection";
import { flatMerge } from "./RequestMerger";
import { BuiltRawReqFilter } from "./RequestBuilder";
@ -137,7 +137,6 @@ export class Query implements QueryBase {
#feed: NoteStore;
#log = debug("Query");
#allFilters: Array<ReqFilter> = [];
constructor(id: string, feed: NoteStore, leaveOpen?: boolean) {
this.id = id;
@ -154,7 +153,11 @@ export class Query implements QueryBase {
* Recompute the complete set of compressed filters from all query traces
*/
get filters() {
return this.#allFilters;
return flatMerge(this.flatFilters);
}
get flatFilters() {
return this.#tracing.flatMap(a => a.filters).flatMap(expandFilter);
}
get feed() {
@ -271,14 +274,7 @@ export class Query implements QueryBase {
() => this.#onProgress()
);
this.#tracing.push(qt);
this.#reComputeFilters();
c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
return qt;
}
#reComputeFilters() {
console.time("reComputeFilters");
this.#allFilters = flatMerge(this.#tracing.flatMap(a => a.filters).flatMap(expandFilter));
console.timeEnd("reComputeFilters");
}
}

16
packages/system/src/RequestBuilder.ts
View File

@ -1,8 +1,10 @@
import { ReqFilter, u256, HexKey, EventKind } from ".";
import { appendDedupe, dedupe } from "./Util";
import { appendDedupe, dedupe, unixNowMs } from "./Utils";
import { diffFilters } from "./RequestSplitter";
import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel";
import { mergeSimilar } from "./RequestMerger";
import { FlatReqFilter, expandFilter } from "./RequestExpander";
import debug from "debug";
/**
* Which strategy is used when building REQ filters
@ -92,10 +94,18 @@ export class RequestBuilder {
* @param q All previous filters merged
* @returns
*/
buildDiff(relays: RelayCache, filters: Array<ReqFilter>): Array<BuiltRawReqFilter> {
const next = this.buildRaw();
buildDiff(relays: RelayCache, filters: Array<FlatReqFilter>): Array<BuiltRawReqFilter> {
const start = unixNowMs();
const next = this.#builders.flatMap(f => expandFilter(f.filter))
const diff = diffFilters(filters, next);
const ts = (unixNowMs() - start);
const log = debug("buildDiff");
log("%s %d ms", this.id, ts);
if (ts > 200) {
console.warn(diff, filters);
}
if (diff.changed) {
log(diff);
return splitAllByWriteRelays(relays, diff.added).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,

103
packages/system/src/RequestMerger.ts
View File

@ -1,3 +1,4 @@
import { distance } from "./Utils";
import { ReqFilter } from ".";
import { FlatReqFilter } from "./RequestExpander";
@ -16,74 +17,10 @@ export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter |
}
}
}
let flag = false;
if (!equalProp(a.ids, b.ids)) {
flag = true;
}
if (!equalProp(a.authors, b.authors)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a.kinds, b.kinds)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a.limit, b.limit)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a.until, b.until)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a.since, b.since)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a.search, b.search)) {
if (flag) return false;
flag = true;
}
if (!equalProp(a["#e"], b["#e"])) {
if (flag) return false;
flag = true;
}
if (!equalProp(a["#p"], b["#p"])) {
if (flag) return false;
flag = true;
}
if (!equalProp(a["#d"], b["#d"])) {
if (flag) return false;
flag = true;
}
if (!equalProp(a["#r"], b["#r"])) {
if (flag) return false;
flag = true;
}
if (!equalProp(a["#t"], b["#t"])) {
if (flag) return false;
flag = true;
}
return true;
}
function equalProp(a: string | number | Array<string | number> | undefined, b: string | number | Array<string | number> | undefined) {
if ((a !== undefined && b === undefined) || (a === undefined && b !== undefined)) {
return false;
}
if (Array.isArray(a) && Array.isArray(b)) {
if (a.length !== b.length) {
return false;
}
if (!a.every(v => b.includes(v))) {
return false;
}
}
return a === b;
return distance(a, b) <= 1;
}
export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
console.time("mergeSimilar");
const ret = [];
const fCopy = [...filters];
@ -92,14 +29,13 @@ export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
const mergeSet = [current];
for (let i = 0; i < fCopy.length; i++) {
const f = fCopy[i];
if (mergeSet.every(v => canMergeFilters(v, f))) {
if (!mergeSet.some(v => !canMergeFilters(v, f))) {
mergeSet.push(fCopy.splice(i, 1)[0]);
i--;
}
}
ret.push(simpleMerge(mergeSet));
}
console.timeEnd("mergeSimilar");
return ret;
}
@ -117,7 +53,8 @@ export function simpleMerge(filters: Array<ReqFilter>) {
if (result[key] === undefined) {
result[key] = [...value];
} else {
result[key] = [...new Set([...result[key], ...value])];
const toAdd = value.filter(a => !result[key].includes(a));
result[key].push(...toAdd);
}
} else {
result[key] = value;
@ -162,31 +99,25 @@ export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
* @returns
*/
export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
console.time("flatMerge");
let ret: Array<ReqFilter> = [];
// to compute filters which can be merged we need to calucate the distance change between each filter
// then we can merge filters which are exactly 1 change diff from each other
function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
const result: any = {};
filters.forEach(f => {
const filter = f as Record<string, string | number>;
Object.entries(filter).forEach(([key, value]) => {
if (!DiscriminatorKeys.includes(key)) {
if (result[key] === undefined) {
result[key] = [value];
} else {
result[key] = [...new Set([...result[key], value])];
}
return filters.reduce((acc, a) => {
Object.entries(a).forEach(([k, v]) => {
if (DiscriminatorKeys.includes(k)) {
acc[k] = v;
} else {
result[key] = value;
acc[k] ??= [];
if (!acc[k].includes(v)) {
acc[k].push(v);
}
}
});
});
return result as ReqFilter;
})
return acc;
}, {} as any) as ReqFilter;
}
// reducer, kinda verbose
@ -212,7 +143,5 @@ export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
}
ret = n;
}
console.timeEnd("flatMerge");
console.debug(ret);
return ret;
}

21
packages/system/src/RequestSplitter.ts
View File

@ -1,18 +1,15 @@
import { ReqFilter } from ".";
import { flatReqFilterEq } from "./Util";
import { expandFilter } from "./RequestExpander";
import { flatFilterEq } from "./Utils";
import { FlatReqFilter } from "./RequestExpander";
import { flatMerge } from "./RequestMerger";
export function diffFilters(prev: Array<ReqFilter>, next: Array<ReqFilter>) {
const prevExpanded = prev.flatMap(expandFilter);
const nextExpanded = next.flatMap(expandFilter);
const added = flatMerge(nextExpanded.filter(a => !prevExpanded.some(b => flatReqFilterEq(a, b))));
const removed = flatMerge(prevExpanded.filter(a => !nextExpanded.some(b => flatReqFilterEq(a, b))));
export function diffFilters(prev: Array<FlatReqFilter>, next: Array<FlatReqFilter>, calcRemoved?: boolean) {
const added = next.filter(a => !prev.some(b => flatFilterEq(a, b)));
const removed = calcRemoved ? prev.filter(a => !next.some(b => flatFilterEq(a, b))) : [];
const changed = added.length > 0 || removed.length > 0;
return {
added,
removed,
changed: added.length > 0 || removed.length > 0,
added: changed ? flatMerge(added) : [],
removed: changed ? flatMerge(removed) : [],
changed,
};
}

2
packages/system/src/Tag.ts
View File

@ -1,5 +1,5 @@
import { HexKey, u256 } from "./Nostr";
import { unwrap } from "./Util";
import { unwrap } from "./Utils";
export default class Tag {
Original: string[];

68
packages/system/src/Util.ts → packages/system/src/Utils.ts
View File

@ -2,7 +2,7 @@ import * as utils from "@noble/curves/abstract/utils";
import * as secp from "@noble/curves/secp256k1";
import { sha256 as sha2 } from "@noble/hashes/sha256";
import { bech32 } from "bech32";
import { NostrEvent, u256 } from "./Nostr";
import { NostrEvent, ReqFilter, u256 } from "./Nostr";
import { FlatReqFilter } from "RequestExpander";
export function unwrap<T>(v: T | undefined | null): T {
@ -55,7 +55,22 @@ export function deepEqual(x: any, y: any): boolean {
: x === y;
}
export function flatReqFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
export function reqFilterEq(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
return equalProp(a.ids, b.ids)
&& equalProp(a.kinds, b.kinds)
&& equalProp(a.authors, b.authors)
&& equalProp(a.limit, b.limit)
&& equalProp(a.since, b.since)
&& equalProp(a.until, b.until)
&& equalProp(a.search, b.search)
&& equalProp(a["#e"], b["#e"])
&& equalProp(a["#p"], b["#p"])
&& equalProp(a["#t"], b["#t"])
&& equalProp(a["#d"], b["#d"])
&& equalProp(a["#r"], b["#r"]);
}
export function flatFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
return a.ids === b.ids
&& a.kinds === b.kinds
&& a.authors === b.authors
@ -70,6 +85,55 @@ export function flatReqFilterEq(a: FlatReqFilter, b: FlatReqFilter): boolean {
&& a["#r"] === b["#r"];
}
export function equalProp(a: string | number | Array<string | number> | undefined, b: string | number | Array<string | number> | undefined) {
if ((a !== undefined && b === undefined) || (a === undefined && b !== undefined)) {
return false;
}
if (Array.isArray(a) && Array.isArray(b)) {
if (a.length !== b.length) {
return false;
}
if (!a.every(v => b.includes(v))) {
return false;
}
}
return a === b;
}
/**
* Compute the "distance" between two objects by comparing their difference in properties
* Missing/Added keys result in +10 distance
* This is not recursive
*/
export function distance(a: any, b: any): number {
const keys1 = Object.keys(a);
const keys2 = Object.keys(b);
const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
let distance = 0;
for (const key of maxKeys) {
if (key in a && key in b) {
if (Array.isArray(a[key]) && Array.isArray(b[key])) {
const aa = a[key] as Array<string | number>;
const bb = b[key] as Array<string | number>;
if (aa.length === bb.length) {
if (aa.some(v => !bb.includes(v))) {
distance++;
}
} else {
distance++;
}
} else if (a[key] !== b[key]) {
distance++;
}
} else {
distance += 10;
}
}
return distance;
}
export function dedupe<T>(v: Array<T>) {
return [...new Set(v)];
}

12
packages/system/src/cache/index.ts vendored
View File

@ -1,5 +1,5 @@
import { HexKey, NostrEvent, UserMetadata } from "..";
import { hexToBech32, unixNowMs } from "../Util";
import { hexToBech32, unixNowMs } from "../Utils";
export interface MetadataCache extends UserMetadata {
/**
@ -36,13 +36,21 @@ export interface MetadataCache extends UserMetadata {
export function mapEventToProfile(ev: NostrEvent) {
try {
const data: UserMetadata = JSON.parse(ev.content);
return {
let ret = {
...data,
pubkey: ev.pubkey,
npub: hexToBech32("npub", ev.pubkey),
created: ev.created_at,
loaded: unixNowMs(),
} as MetadataCache;
// sanitize non-string/number
for (const [k, v] of Object.entries(ret)) {
if (typeof v !== "number" && typeof v !== "string") {
(ret as any)[k] = undefined;
}
}
return ret;
} catch (e) {
console.error("Failed to parse JSON", ev, e);
}