Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move to pkg

Browse Source
This commit is contained in:
Kieran
2023-06-08 12:45:23 +02:00
parent 2b80109e3b
commit 81ccb95d82
256 changed files with 4856 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

415
packages/system/src/Connection.ts Normal file
View File

@ -0,0 +1,415 @@
import { v4 as uuid } from "uuid";
import { DefaultConnectTimeout } from "./Const";
import { ConnectionStats } from "./ConnectionStats";
import { NostrEvent, ReqCommand, TaggedRawEvent, u256 } from "./Nostr";
import { RelayInfo } from "./RelayInfo";
import { unwrap } from "./Util";
import ExternalStore from "./ExternalStore";
export type AuthHandler = (challenge: string, relay: string) => Promise<NostrEvent | undefined>;
/**
* Relay settings
*/
export interface RelaySettings {
read: boolean;
write: boolean;
}
/**
* Snapshot of connection stats
*/
export interface ConnectionStateSnapshot {
connected: boolean;
disconnects: number;
avgLatency: number;
events: {
received: number;
send: number;
};
settings?: RelaySettings;
info?: RelayInfo;
pendingRequests: Array<string>;
activeRequests: Array<string>;
id: string;
ephemeral: boolean;
address: string;
}
export class Connection extends ExternalStore<ConnectionStateSnapshot> {
Id: string;
Address: string;
Socket: WebSocket | null = null;
PendingRaw: Array<object> = [];
PendingRequests: Array<{
cmd: ReqCommand;
cb: () => void;
}> = [];
ActiveRequests = new Set<string>();
Settings: RelaySettings;
Info?: RelayInfo;
ConnectTimeout: number = DefaultConnectTimeout;
Stats: ConnectionStats = new ConnectionStats();
HasStateChange: boolean = true;
IsClosed: boolean;
ReconnectTimer: ReturnType<typeof setTimeout> | null;
EventsCallback: Map<u256, (msg: boolean[]) => void>;
OnConnected?: () => void;
OnEvent?: (sub: string, e: TaggedRawEvent) => void;
OnEose?: (sub: string) => void;
OnDisconnect?: (id: string) => void;
Auth?: AuthHandler;
AwaitingAuth: Map<string, boolean>;
Authed = false;
Ephemeral: boolean;
EphemeralTimeout: ReturnType<typeof setTimeout> | undefined;
Down = true;
constructor(addr: string, options: RelaySettings, auth?: AuthHandler, ephemeral: boolean = false) {
super();
this.Id = uuid();
this.Address = addr;
this.Settings = options;
this.IsClosed = false;
this.ReconnectTimer = null;
this.EventsCallback = new Map();
this.AwaitingAuth = new Map();
this.Auth = auth;
this.Ephemeral = ephemeral;
}
ResetEphemeralTimeout() {
if (this.EphemeralTimeout) {
clearTimeout(this.EphemeralTimeout);
}
if (this.Ephemeral) {
this.EphemeralTimeout = setTimeout(() => {
this.Close();
}, 30_000);
}
}
async Connect() {
try {
if (this.Info === undefined) {
const u = new URL(this.Address);
const rsp = await fetch(`${u.protocol === "wss:" ? "https:" : "http:"}//${u.host}`, {
headers: {
accept: "application/nostr+json",
},
});
if (rsp.ok) {
const data = await rsp.json();
for (const [k, v] of Object.entries(data)) {
if (v === "unset" || v === "" || v === "~") {
data[k] = undefined;
}
}
this.Info = data;
}
}
} catch (e) {
console.warn("Could not load relay information", e);
}
if (this.Socket) {
this.Id = uuid();
this.Socket.onopen = null;
this.Socket.onmessage = null;
this.Socket.onerror = null;
this.Socket.onclose = null;
}
this.IsClosed = false;
this.Socket = new WebSocket(this.Address);
this.Socket.onopen = () => this.OnOpen();
this.Socket.onmessage = e => this.OnMessage(e);
this.Socket.onerror = e => this.OnError(e);
this.Socket.onclose = e => this.OnClose(e);
}
Close() {
this.IsClosed = true;
if (this.ReconnectTimer !== null) {
clearTimeout(this.ReconnectTimer);
this.ReconnectTimer = null;
}
this.Socket?.close();
this.notifyChange();
}
OnOpen() {
this.ConnectTimeout = DefaultConnectTimeout;
console.log(`[${this.Address}] Open!`);
this.Down = false;
if (this.Ephemeral) {
this.ResetEphemeralTimeout();
}
this.OnConnected?.();
this.#sendPendingRaw();
}
OnClose(e: CloseEvent) {
if (!this.IsClosed) {
this.ConnectTimeout = this.ConnectTimeout * 2;
console.log(
`[${this.Address}] Closed (${e.reason}), trying again in ${(this.ConnectTimeout / 1000)
.toFixed(0)
.toLocaleString()} sec`
);
this.ReconnectTimer = setTimeout(() => {
this.Connect();
}, this.ConnectTimeout);
this.Stats.Disconnects++;
} else {
console.log(`[${this.Address}] Closed!`);
this.ReconnectTimer = null;
}
this.OnDisconnect?.(this.Id);
this.#ResetQueues();
// reset connection Id on disconnect, for query-tracking
this.Id = uuid();
this.notifyChange();
}
OnMessage(e: MessageEvent) {
if (e.data.length > 0) {
const msg = JSON.parse(e.data);
const tag = msg[0];
switch (tag) {
case "AUTH": {
this._OnAuthAsync(msg[1])
.then(() => this.#sendPendingRaw())
.catch(console.error);
this.Stats.EventsReceived++;
this.notifyChange();
break;
}
case "EVENT": {
this.OnEvent?.(msg[1], {
...msg[2],
relays: [this.Address],
});
this.Stats.EventsReceived++;
this.notifyChange();
break;
}
case "EOSE": {
this.OnEose?.(msg[1]);
break;
}
case "OK": {
// feedback to broadcast call
console.debug(`${this.Address} OK: `, msg);
const id = msg[1];
if (this.EventsCallback.has(id)) {
const cb = unwrap(this.EventsCallback.get(id));
this.EventsCallback.delete(id);
cb(msg);
}
break;
}
case "NOTICE": {
console.warn(`[${this.Address}] NOTICE: ${msg[1]}`);
break;
}
default: {
console.warn(`Unknown tag: ${tag}`);
break;
}
}
}
}
OnError(e: Event) {
console.error(e);
this.notifyChange();
}
/**
* Send event on this connection
*/
SendEvent(e: NostrEvent) {
if (!this.Settings.write) {
return;
}
const req = ["EVENT", e];
this.#SendJson(req);
this.Stats.EventsSent++;
this.notifyChange();
}
/**
* Send event on this connection and wait for OK response
*/
async SendAsync(e: NostrEvent, timeout = 5000) {
return new Promise<void>(resolve => {
if (!this.Settings.write) {
resolve();
return;
}
const t = setTimeout(() => {
resolve();
}, timeout);
this.EventsCallback.set(e.id, () => {
clearTimeout(t);
resolve();
});
const req = ["EVENT", e];
this.#SendJson(req);
this.Stats.EventsSent++;
this.notifyChange();
});
}
/**
* Using relay document to determine if this relay supports a feature
*/
SupportsNip(n: number) {
return this.Info?.supported_nips?.some(a => a === n) ?? false;
}
/**
* Queue or send command to the relay
* @param cmd The REQ to send to the server
*/
QueueReq(cmd: ReqCommand, cbSent: () => void) {
if (this.ActiveRequests.size >= this.#maxSubscriptions) {
this.PendingRequests.push({
cmd,
cb: cbSent,
});
console.debug("Queuing:", this.Address, cmd);
} else {
this.ActiveRequests.add(cmd[1]);
this.#SendJson(cmd);
cbSent();
}
this.notifyChange();
}
CloseReq(id: string) {
if (this.ActiveRequests.delete(id)) {
this.#SendJson(["CLOSE", id]);
this.OnEose?.(id);
this.#SendQueuedRequests();
}
this.notifyChange();
}
takeSnapshot(): ConnectionStateSnapshot {
return {
connected: this.Socket?.readyState === WebSocket.OPEN,
events: {
received: this.Stats.EventsReceived,
send: this.Stats.EventsSent,
},
avgLatency:
this.Stats.Latency.length > 0
? this.Stats.Latency.reduce((acc, v) => acc + v, 0) / this.Stats.Latency.length
: 0,
disconnects: this.Stats.Disconnects,
info: this.Info,
id: this.Id,
pendingRequests: [...this.PendingRequests.map(a => a.cmd[1])],
activeRequests: [...this.ActiveRequests],
ephemeral: this.Ephemeral,
address: this.Address,
};
}
#SendQueuedRequests() {
const canSend = this.#maxSubscriptions - this.ActiveRequests.size;
if (canSend > 0) {
for (let x = 0; x < canSend; x++) {
const p = this.PendingRequests.shift();
if (p) {
this.ActiveRequests.add(p.cmd[1]);
this.#SendJson(p.cmd);
p.cb();
console.debug("Sent pending REQ", this.Address, p.cmd);
}
}
}
}
#ResetQueues() {
this.ActiveRequests.clear();
this.PendingRequests = [];
this.PendingRaw = [];
this.notifyChange();
}
#SendJson(obj: object) {
const authPending = !this.Authed && (this.AwaitingAuth.size > 0 || this.Info?.limitation?.auth_required === true);
if (this.Socket?.readyState !== WebSocket.OPEN || authPending) {
this.PendingRaw.push(obj);
if (this.Socket?.readyState === WebSocket.CLOSED && this.Ephemeral && this.IsClosed) {
this.Connect();
}
return false;
}
this.#sendPendingRaw();
this.#sendOnWire(obj);
}
#sendPendingRaw() {
while (this.PendingRaw.length > 0) {
const next = this.PendingRaw.shift();
if (next) {
this.#sendOnWire(next);
}
}
}
#sendOnWire(obj: unknown) {
if (this.Socket?.readyState !== WebSocket.OPEN) {
throw new Error(`Socket is not open, state is ${this.Socket?.readyState}`);
}
const json = JSON.stringify(obj);
this.Socket.send(json);
return true;
}
async _OnAuthAsync(challenge: string): Promise<void> {
const authCleanup = () => {
this.AwaitingAuth.delete(challenge);
};
if (!this.Auth) {
throw new Error("Auth hook not registered");
}
this.AwaitingAuth.set(challenge, true);
const authEvent = await this.Auth(challenge, this.Address);
return new Promise(resolve => {
if (!authEvent) {
authCleanup();
return Promise.reject("no event");
}
const t = setTimeout(() => {
authCleanup();
resolve();
}, 10_000);
this.EventsCallback.set(authEvent.id, (msg: boolean[]) => {
clearTimeout(t);
authCleanup();
if (msg.length > 3 && msg[2] === true) {
this.Authed = true;
}
resolve();
});
this.#sendOnWire(["AUTH", authEvent]);
});
}
get #maxSubscriptions() {
return this.Info?.limitation?.max_subscriptions ?? 25;
}
}

34
packages/system/src/ConnectionStats.ts Normal file
View File

@ -0,0 +1,34 @@
/**
* Stats class for tracking metrics per connection
*/
export class ConnectionStats {
/**
* Last n records of how long between REQ->EOSE
*/
Latency: number[] = [];
/**
* Total number of REQ's sent on this connection
*/
Subs: number = 0;
/**
* Count of REQ which took too long and where abandoned
*/
SubsTimeout: number = 0;
/**
* Total number of EVENT messages received
*/
EventsReceived: number = 0;
/**
* Total number of EVENT messages sent
*/
EventsSent: number = 0;
/**
* Total number of times this connection was lost
*/
Disconnects: number = 0;
}

16
packages/system/src/Const.ts Normal file
View File

@ -0,0 +1,16 @@
/**
* Websocket re-connect timeout
*/
export const DefaultConnectTimeout = 2000;
/**
* Hashtag regex
*/
// eslint-disable-next-line no-useless-escape
export const HashtagRegex = /(#[^\s!@#$%^&*()=+.\/,\[{\]};:'"?><]+)/g;
/**
* How long profile cache should be considered valid for
*/
export const ProfileCacheExpire = 1_000 * 60 * 60 * 6;

108
packages/system/src/EventBuilder.ts Normal file
View File

@ -0,0 +1,108 @@
import { EventKind, HexKey, NostrPrefix, NostrEvent } from ".";
import { HashtagRegex } from "./Const";
import { getPublicKey, unixNow } from "./Util";
import { EventExt } from "./EventExt";
import { parseNostrLink } from "./NostrLink";
export class EventBuilder {
#kind?: EventKind;
#content?: string;
#createdAt?: number;
#pubkey?: string;
#tags: Array<Array<string>> = [];
kind(k: EventKind) {
this.#kind = k;
return this;
}
content(c: string) {
this.#content = c;
return this;
}
createdAt(n: number) {
this.#createdAt = n;
return this;
}
pubKey(k: string) {
this.#pubkey = k;
return this;
}
tag(t: Array<string>): EventBuilder {
const duplicate = this.#tags.some(a => a.length === t.length && a.every((b, i) => b !== a[i]));
if (duplicate) return this;
this.#tags.push(t);
return this;
}
/**
* Extract mentions
*/
processContent() {
if (this.#content) {
this.#content = this.#content.replace(/@n(pub|profile|event|ote|addr|)1[acdefghjklmnpqrstuvwxyz023456789]+/g, m =>
this.#replaceMention(m)
);
const hashTags = [...this.#content.matchAll(HashtagRegex)];
hashTags.map(hashTag => {
this.#addHashtag(hashTag[0]);
});
}
return this;
}
build() {
this.#validate();
const ev = {
id: "",
pubkey: this.#pubkey ?? "",
content: this.#content ?? "",
kind: this.#kind,
created_at: this.#createdAt ?? unixNow(),
tags: this.#tags,
} as NostrEvent;
ev.id = EventExt.createId(ev);
return ev;
}
/**
* Build and sign event
* @param pk Private key to sign event with
*/
async buildAndSign(pk: HexKey) {
const ev = this.pubKey(getPublicKey(pk)).build();
await EventExt.sign(ev, pk);
return ev;
}
#validate() {
if (this.#kind === undefined) {
throw new Error("Kind must be set");
}
if (this.#pubkey === undefined) {
throw new Error("Pubkey must be set");
}
}
#replaceMention(match: string) {
const npub = match.slice(1);
const link = parseNostrLink(npub);
if (link) {
if (link.type === NostrPrefix.Profile || link.type === NostrPrefix.PublicKey) {
this.tag(["p", link.id]);
}
return `nostr:${link.encode()}`;
} else {
return match;
}
}
#addHashtag(match: string) {
const tag = match.slice(1);
this.tag(["t", tag.toLowerCase()]);
}
}

167
packages/system/src/EventExt.ts Normal file
View File

@ -0,0 +1,167 @@
import * as secp from "@noble/curves/secp256k1";
import * as utils from "@noble/curves/abstract/utils";
import { EventKind, HexKey, NostrEvent, Tag } from ".";
import base64 from "@protobufjs/base64";
import { sha256, unixNow } from "./Util";
export interface Thread {
root?: Tag;
replyTo?: Tag;
mentions: Array<Tag>;
pubKeys: Array<HexKey>;
}
export abstract class EventExt {
/**
* Get the pub key of the creator of this event NIP-26
*/
static getRootPubKey(e: NostrEvent): HexKey {
const delegation = e.tags.find(a => a[0] === "delegation");
if (delegation?.[1]) {
// todo: verify sig
return delegation[1];
}
return e.pubkey;
}
/**
* Sign this message with a private key
*/
static sign(e: NostrEvent, key: HexKey) {
e.id = this.createId(e);
const sig = secp.schnorr.sign(e.id, key);
e.sig = utils.bytesToHex(sig);
if (!(secp.schnorr.verify(e.sig, e.id, e.pubkey))) {
throw new Error("Signing failed");
}
}
/**
* Check the signature of this message
* @returns True if valid signature
*/
static verify(e: NostrEvent) {
const id = this.createId(e);
const result = secp.schnorr.verify(e.sig, id, e.pubkey);
return result;
}
static createId(e: NostrEvent) {
const payload = [0, e.pubkey, e.created_at, e.kind, e.tags, e.content];
const hash = sha256(JSON.stringify(payload));
if (e.id !== "" && hash !== e.id) {
console.debug(payload);
throw new Error("ID doesnt match!");
}
return hash;
}
/**
* Create a new event for a specific pubkey
*/
static forPubKey(pk: HexKey, kind: EventKind) {
return {
pubkey: pk,
kind: kind,
created_at: unixNow(),
content: "",
tags: [],
id: "",
sig: "",
} as NostrEvent;
}
static extractThread(ev: NostrEvent) {
const isThread = ev.tags.some(a => (a[0] === "e" && a[3] !== "mention") || a[0] == "a");
if (!isThread) {
return undefined;
}
const shouldWriteMarkers = ev.kind === EventKind.TextNote;
const ret = {
mentions: [],
pubKeys: [],
} as Thread;
const eTags = ev.tags.filter(a => a[0] === "e" || a[0] === "a").map((v, i) => new Tag(v, i));
const marked = eTags.some(a => a.Marker !== undefined);
if (!marked) {
ret.root = eTags[0];
ret.root.Marker = shouldWriteMarkers ? "root" : undefined;
if (eTags.length > 1) {
ret.replyTo = eTags[1];
ret.replyTo.Marker = shouldWriteMarkers ? "reply" : undefined;
}
if (eTags.length > 2) {
ret.mentions = eTags.slice(2);
if (shouldWriteMarkers) {
ret.mentions.forEach(a => (a.Marker = "mention"));
}
}
} else {
const root = eTags.find(a => a.Marker === "root");
const reply = eTags.find(a => a.Marker === "reply");
ret.root = root;
ret.replyTo = reply;
ret.mentions = eTags.filter(a => a.Marker === "mention");
}
ret.pubKeys = Array.from(new Set(ev.tags.filter(a => a[0] === "p").map(a => a[1])));
return ret;
}
/**
* Encrypt the given message content
*/
static async encryptData(content: string, pubkey: HexKey, privkey: HexKey) {
const key = await this.#getDmSharedKey(pubkey, privkey);
const iv = window.crypto.getRandomValues(new Uint8Array(16));
const data = new TextEncoder().encode(content);
const result = await window.crypto.subtle.encrypt(
{
name: "AES-CBC",
iv: iv,
},
key,
data
);
const uData = new Uint8Array(result);
return `${base64.encode(uData, 0, result.byteLength)}?iv=${base64.encode(iv, 0, 16)}`;
}
/**
* Decrypt the content of the message
*/
static async decryptData(cyphertext: string, privkey: HexKey, pubkey: HexKey) {
const key = await this.#getDmSharedKey(pubkey, privkey);
const cSplit = cyphertext.split("?iv=");
const data = new Uint8Array(base64.length(cSplit[0]));
base64.decode(cSplit[0], data, 0);
const iv = new Uint8Array(base64.length(cSplit[1]));
base64.decode(cSplit[1], iv, 0);
const result = await window.crypto.subtle.decrypt(
{
name: "AES-CBC",
iv: iv,
},
key,
data
);
return new TextDecoder().decode(result);
}
/**
* Decrypt the content of this message in place
*/
static async decryptDm(content: string, privkey: HexKey, pubkey: HexKey) {
return await this.decryptData(content, privkey, pubkey);
}
static async #getDmSharedKey(pubkey: HexKey, privkey: HexKey) {
const sharedPoint = secp.secp256k1.getSharedSecret(privkey, "02" + pubkey);
const sharedX = sharedPoint.slice(1, 33);
return await window.crypto.subtle.importKey("raw", sharedX, { name: "AES-CBC" }, false, ["encrypt", "decrypt"]);
}
}

29
packages/system/src/EventKind.ts Normal file
View File

@ -0,0 +1,29 @@
enum EventKind {
Unknown = -1,
SetMetadata = 0,
TextNote = 1,
RecommendServer = 2,
ContactList = 3, // NIP-02
DirectMessage = 4, // NIP-04
Deletion = 5, // NIP-09
Repost = 6, // NIP-18
Reaction = 7, // NIP-25
BadgeAward = 8, // NIP-58
SnortSubscriptions = 1000, // NIP-XX
Polls = 6969, // NIP-69
FileHeader = 1063, // NIP-94
Relays = 10002, // NIP-65
Ephemeral = 20_000,
Auth = 22242, // NIP-42
PubkeyLists = 30000, // NIP-51a
NoteLists = 30001, // NIP-51b
TagLists = 30002, // NIP-51c
Badge = 30009, // NIP-58
ProfileBadges = 30008, // NIP-58
ZapstrTrack = 31337,
ZapRequest = 9734, // NIP 57
ZapReceipt = 9735, // NIP 57
HttpAuthentication = 27235, // NIP XX - HTTP Authentication
}
export default EventKind;

317
packages/system/src/EventPublisher.ts Normal file
View File

@ -0,0 +1,317 @@
import * as secp from "@noble/curves/secp256k1";
import * as utils from "@noble/curves/abstract/utils";
import {
EventKind,
FullRelaySettings,
HexKey,
Lists,
NostrEvent,
RelaySettings,
SystemInterface,
TaggedRawEvent,
u256,
UserMetadata,
} from ".";
import { unwrap } from "./Util";
import { EventBuilder } from "./EventBuilder";
import { EventExt } from "./EventExt";
import { barrierQueue, processWorkQueue, WorkQueueItem } from "./WorkQueue";
const Nip7Queue: Array<WorkQueueItem> = [];
processWorkQueue(Nip7Queue);
export type EventBuilderHook = (ev: EventBuilder) => EventBuilder;
declare global {
interface Window {
nostr?: {
getPublicKey: () => Promise<HexKey>;
signEvent: <T extends NostrEvent>(event: T) => Promise<T>;
getRelays?: () => Promise<Record<string, { read: boolean; write: boolean }>>;
nip04?: {
encrypt?: (pubkey: HexKey, plaintext: string) => Promise<string>;
decrypt?: (pubkey: HexKey, ciphertext: string) => Promise<string>;
};
};
}
}
export class EventPublisher {
#system: SystemInterface;
#pubKey: string;
#privateKey?: string;
constructor(system: SystemInterface, pubKey: string, privKey?: string) {
this.#system = system;
if (privKey) {
this.#privateKey = privKey;
this.#pubKey = utils.bytesToHex(secp.schnorr.getPublicKey(privKey));
} else {
this.#pubKey = pubKey;
}
}
get #hasNip07() {
return "nostr" in window;
}
#eb(k: EventKind) {
const eb = new EventBuilder();
return eb.pubKey(this.#pubKey).kind(k);
}
async #sign(eb: EventBuilder) {
if (this.#hasNip07 && !this.#privateKey) {
const nip7PubKey = await barrierQueue(Nip7Queue, () => unwrap(window.nostr).getPublicKey());
if (nip7PubKey !== this.#pubKey) {
throw new Error("Can't sign event, NIP-07 pubkey does not match");
}
const ev = eb.build();
return await barrierQueue(Nip7Queue, () => unwrap(window.nostr).signEvent(ev));
} else if (this.#privateKey) {
return await eb.buildAndSign(this.#privateKey);
} else {
throw new Error("Can't sign event, no private keys available");
}
}
async nip4Encrypt(content: string, key: HexKey) {
if (this.#hasNip07 && !this.#privateKey) {
const nip7PubKey = await barrierQueue(Nip7Queue, () => unwrap(window.nostr).getPublicKey());
if (nip7PubKey !== this.#pubKey) {
throw new Error("Can't encrypt content, NIP-07 pubkey does not match");
}
return await barrierQueue(Nip7Queue, () =>
unwrap(window.nostr?.nip04?.encrypt).call(window.nostr?.nip04, key, content)
);
} else if (this.#privateKey) {
return await EventExt.encryptData(content, key, this.#privateKey);
} else {
throw new Error("Can't encrypt content, no private keys available");
}
}
async nip4Decrypt(content: string, otherKey: HexKey) {
if (this.#hasNip07 && !this.#privateKey && window.nostr?.nip04?.decrypt) {
return await barrierQueue(Nip7Queue, () =>
unwrap(window.nostr?.nip04?.decrypt).call(window.nostr?.nip04, otherKey, content)
);
} else if (this.#privateKey) {
return await EventExt.decryptDm(content, this.#privateKey, otherKey);
} else {
throw new Error("Can't decrypt content, no private keys available");
}
}
async nip42Auth(challenge: string, relay: string) {
const eb = this.#eb(EventKind.Auth);
eb.tag(["relay", relay]);
eb.tag(["challenge", challenge]);
return await this.#sign(eb);
}
broadcast(ev: NostrEvent) {
console.debug(ev);
this.#system.BroadcastEvent(ev);
}
/**
* Write event to all given relays.
*/
broadcastAll(ev: NostrEvent, relays: string[]) {
for (const k of relays) {
this.#system.WriteOnceToRelay(k, ev);
}
}
async muted(keys: HexKey[], priv: HexKey[]) {
const eb = this.#eb(EventKind.PubkeyLists);
eb.tag(["d", Lists.Muted]);
keys.forEach(p => {
eb.tag(["p", p]);
});
if (priv.length > 0) {
const ps = priv.map(p => ["p", p]);
const plaintext = JSON.stringify(ps);
eb.content(await this.nip4Encrypt(plaintext, this.#pubKey));
}
return await this.#sign(eb);
}
async noteList(notes: u256[], list: Lists) {
const eb = this.#eb(EventKind.NoteLists);
eb.tag(["d", list]);
notes.forEach(n => {
eb.tag(["e", n]);
});
return await this.#sign(eb);
}
async tags(tags: string[]) {
const eb = this.#eb(EventKind.TagLists);
eb.tag(["d", Lists.Followed]);
tags.forEach(t => {
eb.tag(["t", t]);
});
return await this.#sign(eb);
}
async metadata(obj: UserMetadata) {
const eb = this.#eb(EventKind.SetMetadata);
eb.content(JSON.stringify(obj));
return await this.#sign(eb);
}
/**
* Create a basic text note
*/
async note(msg: string, fnExtra?: EventBuilderHook) {
const eb = this.#eb(EventKind.TextNote);
eb.content(msg);
eb.processContent();
fnExtra?.(eb);
return await this.#sign(eb);
}
/**
* Create a zap request event for a given target event/profile
* @param amount Millisats amout!
* @param author Author pubkey to tag in the zap
* @param note Note Id to tag in the zap
* @param msg Custom message to be included in the zap
*/
async zap(
amount: number,
author: HexKey,
relays: Array<string>,
note?: HexKey,
msg?: string,
fnExtra?: EventBuilderHook
) {
const eb = this.#eb(EventKind.ZapRequest);
eb.content(msg ?? "");
if (note) {
eb.tag(["e", note]);
}
eb.tag(["p", author]);
eb.tag(["relays", ...relays.map(a => a.trim())]);
eb.tag(["amount", amount.toString()]);
eb.processContent();
fnExtra?.(eb);
return await this.#sign(eb);
}
/**
* Reply to a note
*/
async reply(replyTo: TaggedRawEvent, msg: string, fnExtra?: EventBuilderHook) {
const eb = this.#eb(EventKind.TextNote);
eb.content(msg);
const thread = EventExt.extractThread(replyTo);
if (thread) {
if (thread.root || thread.replyTo) {
eb.tag(["e", thread.root?.Event ?? thread.replyTo?.Event ?? "", "", "root"]);
}
eb.tag(["e", replyTo.id, replyTo.relays?.[0] ?? "", "reply"]);
eb.tag(["p", replyTo.pubkey]);
for (const pk of thread.pubKeys) {
if (pk === this.#pubKey) {
continue;
}
eb.tag(["p", pk]);
}
} else {
eb.tag(["e", replyTo.id, "", "reply"]);
// dont tag self in replies
if (replyTo.pubkey !== this.#pubKey) {
eb.tag(["p", replyTo.pubkey]);
}
}
eb.processContent();
fnExtra?.(eb);
return await this.#sign(eb);
}
async react(evRef: NostrEvent, content = "+") {
const eb = this.#eb(EventKind.Reaction);
eb.content(content);
eb.tag(["e", evRef.id]);
eb.tag(["p", evRef.pubkey]);
return await this.#sign(eb);
}
async relayList(relays: Array<FullRelaySettings> | Record<string, RelaySettings>) {
if (!Array.isArray(relays)) {
relays = Object.entries(relays).map(([k, v]) => ({
url: k,
settings: v,
}));
}
const eb = this.#eb(EventKind.Relays);
for (const rx of relays) {
const rTag = ["r", rx.url];
if (rx.settings.read && !rx.settings.write) {
rTag.push("read");
}
if (rx.settings.write && !rx.settings.read) {
rTag.push("write");
}
eb.tag(rTag);
}
return await this.#sign(eb);
}
async contactList(follows: Array<HexKey>, relays: Record<string, RelaySettings>) {
const eb = this.#eb(EventKind.ContactList);
eb.content(JSON.stringify(relays));
const temp = new Set(follows.filter(a => a.length === 64).map(a => a.toLowerCase()));
temp.forEach(a => eb.tag(["p", a]));
return await this.#sign(eb);
}
/**
* Delete an event (NIP-09)
*/
async delete(id: u256) {
const eb = this.#eb(EventKind.Deletion);
eb.tag(["e", id]);
return await this.#sign(eb);
}
/**
* Repost a note (NIP-18)
*/
async repost(note: NostrEvent) {
const eb = this.#eb(EventKind.Repost);
eb.tag(["e", note.id, ""]);
eb.tag(["p", note.pubkey]);
return await this.#sign(eb);
}
async decryptDm(note: NostrEvent) {
if (note.pubkey !== this.#pubKey && !note.tags.some(a => a[1] === this.#pubKey)) {
throw new Error("Can't decrypt, DM does not belong to this user");
}
const otherPubKey = note.pubkey === this.#pubKey ? unwrap(note.tags.find(a => a[0] === "p")?.[1]) : note.pubkey;
return await this.nip4Decrypt(note.content, otherPubKey);
}
async sendDm(content: string, to: HexKey) {
const eb = this.#eb(EventKind.DirectMessage);
eb.content(await this.nip4Encrypt(content, to));
eb.tag(["p", to]);
return await this.#sign(eb);
}
async generic(fnHook: EventBuilderHook) {
const eb = new EventBuilder();
eb.pubKey(this.#pubKey);
fnHook(eb);
return await this.#sign(eb);
}
}

41
packages/system/src/ExternalStore.ts Normal file
View File

@ -0,0 +1,41 @@
type HookFn<TSnapshot> = (e?: TSnapshot) => void;
interface HookFilter<TSnapshot> {
fn: HookFn<TSnapshot>;
}
/**
* Simple React hookable store with manual change notifications
*/
export default abstract class ExternalStore<TSnapshot> {
#hooks: Array<HookFilter<TSnapshot>> = [];
#snapshot: Readonly<TSnapshot> = {} as Readonly<TSnapshot>;
#changed = true;
hook(fn: HookFn<TSnapshot>) {
this.#hooks.push({
fn,
});
return () => {
const idx = this.#hooks.findIndex(a => a.fn === fn);
if (idx >= 0) {
this.#hooks.splice(idx, 1);
}
};
}
snapshot() {
if (this.#changed) {
this.#snapshot = this.takeSnapshot();
this.#changed = false;
}
return this.#snapshot;
}
protected notifyChange(sn?: TSnapshot) {
this.#changed = true;
this.#hooks.forEach(h => h.fn(sn));
}
abstract takeSnapshot(): TSnapshot;
}

117
packages/system/src/GossipModel.ts Normal file
View File

@ -0,0 +1,117 @@
import { FullRelaySettings, ReqFilter } from ".";
import { unwrap } from "./Util";
import debug from "debug";
const PickNRelays = 2;
export interface RelayTaggedFilter {
relay: string;
filter: ReqFilter;
}
export interface RelayTaggedFilters {
relay: string;
filters: Array<ReqFilter>;
}
export interface RelayCache {
get(pubkey?: string): Array<FullRelaySettings> | undefined;
}
export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilter>) {
const allSplit = filters
.map(a => splitByWriteRelays(cache, a))
.reduce((acc, v) => {
for (const vn of v) {
const existing = acc.get(vn.relay);
if (existing) {
existing.push(vn.filter);
} else {
acc.set(vn.relay, [vn.filter]);
}
}
return acc;
}, new Map<string, Array<ReqFilter>>());
return [...allSplit.entries()].map(([k, v]) => {
return {
relay: k,
filters: v,
} as RelayTaggedFilters;
});
}
/**
* Split filters by authors
* @param filter
* @returns
*/
export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<RelayTaggedFilter> {
if ((filter.authors?.length ?? 0) === 0)
return [
{
relay: "",
filter,
},
];
const allRelays = unwrap(filter.authors).map(a => {
return {
key: a,
relays: cache.get(a)?.filter(a => a.settings.write),
};
});
const missing = allRelays.filter(a => a.relays === undefined);
const hasRelays = allRelays.filter(a => a.relays !== undefined);
const relayUserMap = hasRelays.reduce((acc, v) => {
for (const r of unwrap(v.relays)) {
if (!acc.has(r.url)) {
acc.set(r.url, new Set([v.key]));
} else {
unwrap(acc.get(r.url)).add(v.key);
}
}
return acc;
}, new Map<string, Set<string>>());
// selection algo will just pick relays with the most users
const topRelays = [...relayUserMap.entries()].sort(([, v], [, v1]) => v1.size - v.size);
// <relay, key[]> - count keys per relay
// <key, relay[]> - pick n top relays
// <relay, key[]> - map keys per relay (for subscription filter)
const userPickedRelays = unwrap(filter.authors).map(k => {
// pick top 3 relays for this key
const relaysForKey = topRelays
.filter(([, v]) => v.has(k))
.slice(0, PickNRelays)
.map(([k]) => k);
return { k, relaysForKey };
});
const pickedRelays = new Set(userPickedRelays.map(a => a.relaysForKey).flat());
const picked = [...pickedRelays].map(a => {
const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k));
return {
relay: a,
filter: {
...filter,
authors: [...keysOnPickedRelay],
},
} as RelayTaggedFilter;
});
if (missing.length > 0) {
picked.push({
relay: "",
filter: {
...filter,
authors: missing.map(a => a.key),
},
});
}
debug("GOSSIP")("Picked %o", picked);
return picked;
}

88
packages/system/src/Links.ts Normal file
View File

@ -0,0 +1,88 @@
import * as utils from "@noble/curves/abstract/utils";
import { bech32 } from "bech32";
import { HexKey } from "./Nostr";
export enum NostrPrefix {
PublicKey = "npub",
PrivateKey = "nsec",
Note = "note",
// TLV prefixes
Profile = "nprofile",
Event = "nevent",
Relay = "nrelay",
Address = "naddr",
}
export enum TLVEntryType {
Special = 0,
Relay = 1,
Author = 2,
Kind = 3,
}
export interface TLVEntry {
type: TLVEntryType;
length: number;
value: string | HexKey | number;
}
export function encodeTLV(prefix: NostrPrefix, id: string, relays?: string[], kind?: number, author?: string) {
const enc = new TextEncoder();
const buf = prefix === NostrPrefix.Address ? enc.encode(id) : utils.hexToBytes(id);
const tl0 = [0, buf.length, ...buf];
const tl1 =
relays
?.map(a => {
const data = enc.encode(a);
return [1, data.length, ...data];
})
.flat() ?? [];
const tl2 = author ? [2, 32, ...utils.hexToBytes(author)] : [];
const tl3 = kind ? [3, 4, ...new Uint8Array(new Uint32Array([kind]).buffer).reverse()] : [];
return bech32.encode(prefix, bech32.toWords([...tl0, ...tl1, ...tl2, ...tl3]), 1_000);
}
export function decodeTLV(str: string) {
const decoded = bech32.decode(str, 1_000);
const data = bech32.fromWords(decoded.words);
const entries: TLVEntry[] = [];
let x = 0;
while (x < data.length) {
const t = data[x];
const l = data[x + 1];
const v = data.slice(x + 2, x + 2 + l);
entries.push({
type: t,
length: l,
value: decodeTLVEntry(t, decoded.prefix, new Uint8Array(v)),
});
x += 2 + l;
}
return entries;
}
function decodeTLVEntry(type: TLVEntryType, prefix: string, data: Uint8Array) {
switch (type) {
case TLVEntryType.Special: {
if (prefix === NostrPrefix.Address) {
return new TextDecoder("ASCII").decode(data);
} else {
return utils.bytesToHex(data);
}
}
case TLVEntryType.Author: {
return utils.bytesToHex(data);
}
case TLVEntryType.Kind: {
return new Uint32Array(new Uint8Array(data.reverse()).buffer)[0];
}
case TLVEntryType.Relay: {
return new TextDecoder("ASCII").decode(data);
}
}
}

3
packages/system/src/Nips.ts Normal file
View File

@ -0,0 +1,3 @@
export enum Nips {
Search = 50,
}

84
packages/system/src/Nostr.ts Normal file
View File

@ -0,0 +1,84 @@
import { RelaySettings } from "./Connection";
export interface NostrEvent {
id: u256;
pubkey: HexKey;
created_at: number;
kind: number;
tags: Array<Array<string>>;
content: string;
sig: string;
}
export interface TaggedRawEvent extends NostrEvent {
/**
* A list of relays this event was seen on
*/
relays: string[];
}
/**
* Basic raw key as hex
*/
export type HexKey = string;
/**
* Optional HexKey
*/
export type MaybeHexKey = HexKey | undefined;
/**
* A 256bit hex id
*/
export type u256 = string;
export type ReqCommand = [cmd: "REQ", id: string, ...filters: Array<ReqFilter>];
/**
* Raw REQ filter object
*/
export interface ReqFilter {
ids?: u256[];
authors?: u256[];
kinds?: number[];
"#e"?: u256[];
"#p"?: u256[];
"#t"?: string[];
"#d"?: string[];
"#r"?: string[];
search?: string;
since?: number;
until?: number;
limit?: number;
}
/**
* Medatadata event content
*/
export type UserMetadata = {
name?: string;
display_name?: string;
about?: string;
picture?: string;
website?: string;
banner?: string;
nip05?: string;
lud06?: string;
lud16?: string;
};
/**
* NIP-51 list types
*/
export enum Lists {
Muted = "mute",
Pinned = "pin",
Bookmarked = "bookmark",
Followed = "follow",
Badges = "profile_badges",
}
export interface FullRelaySettings {
url: string;
settings: RelaySettings;
}

110
packages/system/src/NostrLink.ts Normal file
View File

@ -0,0 +1,110 @@
import { bech32ToHex, hexToBech32 } from "./Util";
import { NostrPrefix, decodeTLV, TLVEntryType } from ".";
export interface NostrLink {
type: NostrPrefix;
id: string;
kind?: number;
author?: string;
relays?: Array<string>;
encode(): string;
}
export function validateNostrLink(link: string): boolean {
try {
const parsedLink = parseNostrLink(link);
if (!parsedLink) {
return false;
}
if (parsedLink.type === NostrPrefix.PublicKey || parsedLink.type === NostrPrefix.Note) {
return parsedLink.id.length === 64;
}
return true;
} catch {
return false;
}
}
export function tryParseNostrLink(link: string, prefixHint?: NostrPrefix): NostrLink | undefined {
try {
return parseNostrLink(link, prefixHint);
} catch {
return undefined;
}
}
export function parseNostrLink(link: string, prefixHint?: NostrPrefix): NostrLink {
const entity = link.startsWith("web+nostr:") || link.startsWith("nostr:") ? link.split(":")[1] : link;
const isPrefix = (prefix: NostrPrefix) => {
return entity.startsWith(prefix);
};
if (isPrefix(NostrPrefix.PublicKey)) {
const id = bech32ToHex(entity);
if (id.length !== 64) throw new Error("Invalid nostr link, must contain 32 byte id");
return {
type: NostrPrefix.PublicKey,
id: id,
encode: () => hexToBech32(NostrPrefix.PublicKey, id),
};
} else if (isPrefix(NostrPrefix.Note)) {
const id = bech32ToHex(entity);
if (id.length !== 64) throw new Error("Invalid nostr link, must contain 32 byte id");
return {
type: NostrPrefix.Note,
id: id,
encode: () => hexToBech32(NostrPrefix.Note, id),
};
} else if (isPrefix(NostrPrefix.Profile) || isPrefix(NostrPrefix.Event) || isPrefix(NostrPrefix.Address)) {
const decoded = decodeTLV(entity);
const id = decoded.find(a => a.type === TLVEntryType.Special)?.value as string;
const relays = decoded.filter(a => a.type === TLVEntryType.Relay).map(a => a.value as string);
const author = decoded.find(a => a.type === TLVEntryType.Author)?.value as string;
const kind = decoded.find(a => a.type === TLVEntryType.Kind)?.value as number;
const encode = () => {
return entity; // return original
};
if (isPrefix(NostrPrefix.Profile)) {
if (id.length !== 64) throw new Error("Invalid nostr link, must contain 32 byte id");
return {
type: NostrPrefix.Profile,
id,
relays,
kind,
author,
encode,
};
} else if (isPrefix(NostrPrefix.Event)) {
if (id.length !== 64) throw new Error("Invalid nostr link, must contain 32 byte id");
return {
type: NostrPrefix.Event,
id,
relays,
kind,
author,
encode,
};
} else if (isPrefix(NostrPrefix.Address)) {
return {
type: NostrPrefix.Address,
id,
relays,
kind,
author,
encode,
};
}
} else if (prefixHint) {
return {
type: prefixHint,
id: link,
encode: () => hexToBech32(prefixHint, link),
};
}
throw new Error("Invalid nostr link");
}

243
packages/system/src/NostrSystem.ts Normal file
View File

@ -0,0 +1,243 @@
import debug from "debug";
import ExternalStore from "./ExternalStore";
import { NostrEvent, TaggedRawEvent } from "./Nostr";
import { AuthHandler, Connection, RelaySettings, ConnectionStateSnapshot } from "./Connection";
import { Query } from "./Query";
import { RelayCache } from "./GossipModel";
import { NoteStore } from "./NoteCollection";
import { BuiltRawReqFilter, RequestBuilder } from "./RequestBuilder";
import { unwrap, sanitizeRelayUrl } from "./Util";
import { SystemInterface, SystemSnapshot } from ".";
/**
* Manages nostr content retrieval system
*/
export class NostrSystem extends ExternalStore<SystemSnapshot> implements SystemInterface {
/**
* All currently connected websockets
*/
#sockets = new Map<string, Connection>();
/**
* All active queries
*/
Queries: Map<string, Query> = new Map();
/**
* Handler function for NIP-42
*/
HandleAuth?: AuthHandler;
#log = debug("System");
#relayCache: RelayCache;
constructor(relayCache: RelayCache) {
super();
this.#relayCache = relayCache;
this.#cleanup();
}
get Sockets(): ConnectionStateSnapshot[] {
return [...this.#sockets.values()].map(a => a.snapshot());
}
/**
* Connect to a NOSTR relay if not already connected
*/
async ConnectToRelay(address: string, options: RelaySettings) {
try {
const addr = unwrap(sanitizeRelayUrl(address));
if (!this.#sockets.has(addr)) {
const c = new Connection(addr, options, this.HandleAuth?.bind(this));
this.#sockets.set(addr, c);
c.OnEvent = (s, e) => this.OnEvent(s, e);
c.OnEose = s => this.OnEndOfStoredEvents(c, s);
c.OnDisconnect = id => this.OnRelayDisconnect(id);
await c.Connect();
} else {
// update settings if already connected
unwrap(this.#sockets.get(addr)).Settings = options;
}
} catch (e) {
console.error(e);
}
}
OnRelayDisconnect(id: string) {
for (const [, q] of this.Queries) {
q.connectionLost(id);
}
}
OnEndOfStoredEvents(c: Readonly<Connection>, sub: string) {
for (const [, v] of this.Queries) {
v.eose(sub, c);
}
}
OnEvent(sub: string, ev: TaggedRawEvent) {
for (const [, v] of this.Queries) {
v.onEvent(sub, ev);
}
}
/**
*
* @param address Relay address URL
*/
async ConnectEphemeralRelay(address: string): Promise<Connection | undefined> {
try {
const addr = unwrap(sanitizeRelayUrl(address));
if (!this.#sockets.has(addr)) {
const c = new Connection(addr, { read: true, write: false }, this.HandleAuth?.bind(this), true);
this.#sockets.set(addr, c);
c.OnEvent = (s, e) => this.OnEvent(s, e);
c.OnEose = s => this.OnEndOfStoredEvents(c, s);
c.OnDisconnect = id => this.OnRelayDisconnect(id);
await c.Connect();
return c;
}
} catch (e) {
console.error(e);
}
}
/**
* Disconnect from a relay
*/
DisconnectRelay(address: string) {
const c = this.#sockets.get(address);
if (c) {
this.#sockets.delete(address);
c.Close();
}
}
GetQuery(id: string): Query | undefined {
return this.Queries.get(id);
}
Query<T extends NoteStore>(type: { new (): T }, req: RequestBuilder): Query {
const existing = this.Queries.get(req.id);
if (existing) {
const filters = !req.options?.skipDiff
? req.buildDiff(this.#relayCache, existing.filters)
: req.build(this.#relayCache);
if (filters.length === 0 && !!req.options?.skipDiff) {
return existing;
} else {
for (const subQ of filters) {
this.SendQuery(existing, subQ).then(qta =>
qta.forEach(v => this.#log("New QT from diff %s %s %O from: %O", req.id, v.id, v.filters, existing.filters))
);
}
this.notifyChange();
return existing;
}
} else {
const store = new type();
const filters = req.build(this.#relayCache);
const q = new Query(req.id, store, req.options?.leaveOpen);
this.Queries.set(req.id, q);
for (const subQ of filters) {
this.SendQuery(q, subQ).then(qta =>
qta.forEach(v => this.#log("New QT from diff %s %s %O", req.id, v.id, v.filters))
);
}
this.notifyChange();
return q;
}
}
async SendQuery(q: Query, qSend: BuiltRawReqFilter) {
if (qSend.relay) {
this.#log("Sending query to %s %O", qSend.relay, qSend);
const s = this.#sockets.get(qSend.relay);
if (s) {
const qt = q.sendToRelay(s, qSend);
if (qt) {
return [qt];
}
} else {
const nc = await this.ConnectEphemeralRelay(qSend.relay);
if (nc) {
const qt = q.sendToRelay(nc, qSend);
if (qt) {
return [qt];
}
} else {
console.warn("Failed to connect to new relay for:", qSend.relay, q);
}
}
} else {
const ret = [];
for (const [, s] of this.#sockets) {
if (!s.Ephemeral) {
const qt = q.sendToRelay(s, qSend);
if (qt) {
ret.push(qt);
}
}
}
return ret;
}
return [];
}
/**
* Send events to writable relays
*/
BroadcastEvent(ev: NostrEvent) {
for (const [, s] of this.#sockets) {
s.SendEvent(ev);
}
}
/**
* Write an event to a relay then disconnect
*/
async WriteOnceToRelay(address: string, ev: NostrEvent) {
return new Promise<void>((resolve, reject) => {
const c = new Connection(address, { write: true, read: false }, this.HandleAuth, true);
const t = setTimeout(reject, 5_000);
c.OnConnected = async () => {
clearTimeout(t);
await c.SendAsync(ev);
c.Close();
resolve();
};
c.Connect();
});
}
takeSnapshot(): SystemSnapshot {
return {
queries: [...this.Queries.values()].map(a => {
return {
id: a.id,
filters: a.filters,
subFilters: [],
};
}),
};
}
#cleanup() {
let changed = false;
for (const [k, v] of this.Queries) {
if (v.canRemove()) {
v.sendClose();
this.Queries.delete(k);
this.#log("Deleted query %s", k);
changed = true;
}
}
if (changed) {
this.notifyChange();
}
setTimeout(() => this.#cleanup(), 1_000);
}
}

265
packages/system/src/NoteCollection.ts Normal file
View File

@ -0,0 +1,265 @@
import { TaggedRawEvent, u256 } from ".";
import { appendDedupe, findTag } from "./Util";
export interface StoreSnapshot<TSnapshot> {
data: TSnapshot | undefined;
clear: () => void;
loading: () => boolean;
add: (ev: Readonly<TaggedRawEvent> | Readonly<Array<TaggedRawEvent>>) => void;
}
export const EmptySnapshot = {
data: undefined,
clear: () => {
// empty
},
loading: () => true,
add: () => {
// empty
},
} as StoreSnapshot<FlatNoteStore>;
export type NoteStoreSnapshotData = Readonly<Array<TaggedRawEvent>> | Readonly<TaggedRawEvent>;
export type NoteStoreHook = () => void;
export type NoteStoreHookRelease = () => void;
export type OnEventCallback = (e: Readonly<Array<TaggedRawEvent>>) => void;
export type OnEventCallbackRelease = () => void;
export type OnEoseCallback = (c: string) => void;
export type OnEoseCallbackRelease = () => void;
/**
* Generic note store interface
*/
export abstract class NoteStore {
abstract add(ev: Readonly<TaggedRawEvent> | Readonly<Array<TaggedRawEvent>>): void;
abstract clear(): void;
// react hooks
abstract hook(cb: NoteStoreHook): NoteStoreHookRelease;
abstract getSnapshotData(): NoteStoreSnapshotData | undefined;
// events
abstract onEvent(cb: OnEventCallback): OnEventCallbackRelease;
abstract get snapshot(): StoreSnapshot<NoteStoreSnapshotData>;
abstract get loading(): boolean;
abstract set loading(v: boolean);
}
export abstract class HookedNoteStore<TSnapshot extends NoteStoreSnapshotData> implements NoteStore {
#hooks: Array<NoteStoreHook> = [];
#eventHooks: Array<OnEventCallback> = [];
#loading = true;
#storeSnapshot: StoreSnapshot<TSnapshot> = {
clear: () => this.clear(),
loading: () => this.loading,
add: ev => this.add(ev),
data: undefined,
};
#needsSnapshot = true;
#nextNotifyTimer?: ReturnType<typeof setTimeout>;
get snapshot() {
this.#updateSnapshot();
return this.#storeSnapshot;
}
get loading() {
return this.#loading;
}
set loading(v: boolean) {
this.#loading = v;
this.onChange([]);
}
abstract add(ev: Readonly<TaggedRawEvent> | Readonly<Array<TaggedRawEvent>>): void;
abstract clear(): void;
hook(cb: NoteStoreHook): NoteStoreHookRelease {
this.#hooks.push(cb);
return () => {
const idx = this.#hooks.findIndex(a => a === cb);
this.#hooks.splice(idx, 1);
};
}
getSnapshotData() {
this.#updateSnapshot();
return this.#storeSnapshot.data;
}
onEvent(cb: OnEventCallback): OnEventCallbackRelease {
const existing = this.#eventHooks.find(a => a === cb);
if (!existing) {
this.#eventHooks.push(cb);
return () => {
const idx = this.#eventHooks.findIndex(a => a === cb);
this.#eventHooks.splice(idx, 1);
};
}
return () => {
//noop
};
}
protected abstract takeSnapshot(): TSnapshot | undefined;
protected onChange(changes: Readonly<Array<TaggedRawEvent>>): void {
this.#needsSnapshot = true;
if (!this.#nextNotifyTimer) {
this.#nextNotifyTimer = setTimeout(() => {
this.#nextNotifyTimer = undefined;
for (const hk of this.#hooks) {
hk();
}
}, 500);
}
if (changes.length > 0) {
for (const hkE of this.#eventHooks) {
hkE(changes);
}
}
}
#updateSnapshot() {
if (this.#needsSnapshot) {
this.#storeSnapshot = {
...this.#storeSnapshot,
data: this.takeSnapshot(),
};
this.#needsSnapshot = false;
}
}
}
/**
* A simple flat container of events with no duplicates
*/
export class FlatNoteStore extends HookedNoteStore<Readonly<Array<TaggedRawEvent>>> {
#events: Array<TaggedRawEvent> = [];
#ids: Set<u256> = new Set();
add(ev: TaggedRawEvent | Array<TaggedRawEvent>) {
ev = Array.isArray(ev) ? ev : [ev];
const changes: Array<TaggedRawEvent> = [];
ev.forEach(a => {
if (!this.#ids.has(a.id)) {
this.#events.push(a);
this.#ids.add(a.id);
changes.push(a);
} else {
const existing = this.#events.find(b => b.id === a.id);
if (existing) {
existing.relays = appendDedupe(existing.relays, a.relays);
}
}
});
if (changes.length > 0) {
this.onChange(changes);
}
}
clear() {
this.#events = [];
this.#ids.clear();
this.onChange([]);
}
takeSnapshot() {
return [...this.#events];
}
}
/**
* A note store that holds a single replaceable event for a given user defined key generator function
*/
export class KeyedReplaceableNoteStore extends HookedNoteStore<Readonly<Array<TaggedRawEvent>>> {
#keyFn: (ev: TaggedRawEvent) => string;
#events: Map<string, TaggedRawEvent> = new Map();
constructor(fn: (ev: TaggedRawEvent) => string) {
super();
this.#keyFn = fn;
}
add(ev: TaggedRawEvent | Array<TaggedRawEvent>) {
ev = Array.isArray(ev) ? ev : [ev];
const changes: Array<TaggedRawEvent> = [];
ev.forEach(a => {
const keyOnEvent = this.#keyFn(a);
const existingCreated = this.#events.get(keyOnEvent)?.created_at ?? 0;
if (a.created_at > existingCreated) {
this.#events.set(keyOnEvent, a);
changes.push(a);
}
});
if (changes.length > 0) {
this.onChange(changes);
}
}
clear() {
this.#events.clear();
this.onChange([]);
}
takeSnapshot() {
return [...this.#events.values()];
}
}
/**
* A note store that holds a single replaceable event
*/
export class ReplaceableNoteStore extends HookedNoteStore<Readonly<TaggedRawEvent>> {
#event?: TaggedRawEvent;
add(ev: TaggedRawEvent | Array<TaggedRawEvent>) {
ev = Array.isArray(ev) ? ev : [ev];
const changes: Array<TaggedRawEvent> = [];
ev.forEach(a => {
const existingCreated = this.#event?.created_at ?? 0;
if (a.created_at > existingCreated) {
this.#event = a;
changes.push(a);
}
});
if (changes.length > 0) {
this.onChange(changes);
}
}
clear() {
this.#event = undefined;
this.onChange([]);
}
takeSnapshot() {
if (this.#event) {
return Object.freeze({ ...this.#event });
}
}
}
/**
* A note store that holds a single replaceable event per pubkey
*/
export class PubkeyReplaceableNoteStore extends KeyedReplaceableNoteStore {
constructor() {
super(e => e.pubkey);
}
}
/**
* A note store that holds a single replaceable event per "pubkey-dtag"
*/
export class ParameterizedReplaceableNoteStore extends KeyedReplaceableNoteStore {
constructor() {
super(ev => {
const dTag = findTag(ev, "d");
return `${ev.pubkey}-${dTag}`;
});
}
}

124
packages/system/src/ProfileCache.ts Normal file
View File

@ -0,0 +1,124 @@
import { EventKind, HexKey, SystemInterface, TaggedRawEvent, PubkeyReplaceableNoteStore, RequestBuilder } from ".";
import { ProfileCacheExpire } from "./Const";
import { CacheStore, mapEventToProfile, MetadataCache } from "./cache";
import { unixNowMs } from "./Util";
import debug from "debug";
export class ProfileLoaderService {
#system: SystemInterface;
#cache: CacheStore<MetadataCache>;
/**
* List of pubkeys to fetch metadata for
*/
WantsMetadata: Set<HexKey> = new Set();
readonly #log = debug("ProfileCache");
constructor(system: SystemInterface, cache: CacheStore<MetadataCache>) {
this.#system = system;
this.#cache = cache;
this.#FetchMetadata();
}
/**
* Request profile metadata for a set of pubkeys
*/
TrackMetadata(pk: HexKey | Array<HexKey>) {
const bufferNow = [];
for (const p of Array.isArray(pk) ? pk : [pk]) {
if (p.length > 0 && this.WantsMetadata.add(p)) {
bufferNow.push(p);
}
}
this.#cache.buffer(bufferNow);
}
/**
* Stop tracking metadata for a set of pubkeys
*/
UntrackMetadata(pk: HexKey | Array<HexKey>) {
for (const p of Array.isArray(pk) ? pk : [pk]) {
if (p.length > 0) {
this.WantsMetadata.delete(p);
}
}
}
async onProfileEvent(e: Readonly<TaggedRawEvent>) {
const profile = mapEventToProfile(e);
if (profile) {
await this.#cache.update(profile);
}
}
async #FetchMetadata() {
const missingFromCache = await this.#cache.buffer([...this.WantsMetadata]);
const expire = unixNowMs() - ProfileCacheExpire;
const expired = [...this.WantsMetadata]
.filter(a => !missingFromCache.includes(a))
.filter(a => (this.#cache.getFromCache(a)?.loaded ?? 0) < expire);
const missing = new Set([...missingFromCache, ...expired]);
if (missing.size > 0) {
this.#log("Wants profiles: %d missing, %d expired", missingFromCache.length, expired.length);
const sub = new RequestBuilder("profiles");
sub
.withOptions({
skipDiff: true,
})
.withFilter()
.kinds([EventKind.SetMetadata])
.authors([...missing]);
const newProfiles = new Set<string>();
const q = this.#system.Query<PubkeyReplaceableNoteStore>(PubkeyReplaceableNoteStore, sub);
const feed = (q?.feed as PubkeyReplaceableNoteStore) ?? new PubkeyReplaceableNoteStore();
// never release this callback, it will stop firing anyway after eose
const releaseOnEvent = feed.onEvent(async e => {
for (const pe of e) {
newProfiles.add(pe.id);
await this.onProfileEvent(pe);
}
});
const results = await new Promise<Readonly<Array<TaggedRawEvent>>>(resolve => {
let timeout: ReturnType<typeof setTimeout> | undefined = undefined;
const release = feed.hook(() => {
if (!feed.loading) {
clearTimeout(timeout);
resolve(feed.getSnapshotData() ?? []);
this.#log("Profiles finished: %s", sub.id);
release();
}
});
timeout = setTimeout(() => {
release();
resolve(feed.getSnapshotData() ?? []);
this.#log("Profiles timeout: %s", sub.id);
}, 5_000);
});
releaseOnEvent();
const couldNotFetch = [...missing].filter(a => !results.some(b => b.pubkey === a));
if (couldNotFetch.length > 0) {
this.#log("No profiles: %o", couldNotFetch);
const empty = couldNotFetch.map(a =>
this.#cache.update({
pubkey: a,
loaded: unixNowMs() - ProfileCacheExpire + 5_000, // expire in 5s
created: 69,
} as MetadataCache)
);
await Promise.all(empty);
}
// When we fetch an expired profile and its the same as what we already have
// onEvent is not fired and the loaded timestamp never gets updated
const expiredSame = results.filter(a => !newProfiles.has(a.id) && expired.includes(a.pubkey));
await Promise.all(expiredSame.map(v => this.onProfileEvent(v)));
}
setTimeout(() => this.#FetchMetadata(), 500);
}
}

284
packages/system/src/Query.ts Normal file
View File

@ -0,0 +1,284 @@
import { v4 as uuid } from "uuid";
import debug from "debug";
import { Connection, ReqFilter, Nips, TaggedRawEvent } from ".";
import { unixNowMs, unwrap } from "./Util";
import { NoteStore } from "./NoteCollection";
import { flatMerge } from "./RequestMerger";
import { BuiltRawReqFilter } from "./RequestBuilder";
import { expandFilter } from "./RequestExpander";
/**
* Tracing for relay query status
*/
class QueryTrace {
readonly id: string;
readonly start: number;
sent?: number;
eose?: number;
close?: number;
#wasForceClosed = false;
readonly #fnClose: (id: string) => void;
readonly #fnProgress: () => void;
constructor(
readonly relay: string,
readonly filters: Array<ReqFilter>,
readonly connId: string,
fnClose: (id: string) => void,
fnProgress: () => void
) {
this.id = uuid();
this.start = unixNowMs();
this.#fnClose = fnClose;
this.#fnProgress = fnProgress;
}
sentToRelay() {
this.sent = unixNowMs();
this.#fnProgress();
}
gotEose() {
this.eose = unixNowMs();
this.#fnProgress();
}
forceEose() {
this.eose = unixNowMs();
this.#wasForceClosed = true;
this.#fnProgress();
this.sendClose();
}
sendClose() {
this.close = unixNowMs();
this.#fnClose(this.id);
this.#fnProgress();
}
/**
* Time spent in queue
*/
get queued() {
return (this.sent === undefined ? unixNowMs() : this.#wasForceClosed ? unwrap(this.eose) : this.sent) - this.start;
}
/**
* Total query runtime
*/
get runtime() {
return (this.eose === undefined ? unixNowMs() : this.eose) - this.start;
}
/**
* Total time spent waiting for relay to respond
*/
get responseTime() {
return this.finished ? unwrap(this.eose) - unwrap(this.sent) : 0;
}
/**
* If tracing is finished, we got EOSE or timeout
*/
get finished() {
return this.eose !== undefined;
}
}
export interface QueryBase {
/**
* Uniquie ID of this query
*/
id: string;
/**
* The query payload (REQ filters)
*/
filters: Array<ReqFilter>;
/**
* List of relays to send this query to
*/
relays?: Array<string>;
}
/**
* Active or queued query on the system
*/
export class Query implements QueryBase {
/**
* Uniquie ID of this query
*/
id: string;
/**
* Which relays this query has already been executed on
*/
#tracing: Array<QueryTrace> = [];
/**
* Leave the query open until its removed
*/
#leaveOpen = false;
/**
* Time when this query can be removed
*/
#cancelAt?: number;
/**
* Timer used to track tracing status
*/
#checkTrace?: ReturnType<typeof setInterval>;
/**
* Feed object which collects events
*/
#feed: NoteStore;
#log = debug("Query");
#allFilters: Array<ReqFilter> = [];
constructor(id: string, feed: NoteStore, leaveOpen?: boolean) {
this.id = id;
this.#feed = feed;
this.#leaveOpen = leaveOpen ?? false;
this.#checkTraces();
}
canRemove() {
return this.#cancelAt !== undefined && this.#cancelAt < unixNowMs();
}
/**
* Recompute the complete set of compressed filters from all query traces
*/
get filters() {
return this.#allFilters;
}
get feed() {
return this.#feed;
}
onEvent(sub: string, e: TaggedRawEvent) {
for (const t of this.#tracing) {
if (t.id === sub) {
this.feed.add(e);
break;
}
}
}
/**
* This function should be called when this Query object and FeedStore is no longer needed
*/
cancel() {
this.#cancelAt = unixNowMs() + 5_000;
}
uncancel() {
this.#cancelAt = undefined;
}
cleanup() {
this.#stopCheckTraces();
}
sendToRelay(c: Connection, subq: BuiltRawReqFilter) {
if (!this.#canSendQuery(c, subq)) {
return;
}
return this.#sendQueryInternal(c, subq);
}
connectionLost(id: string) {
this.#tracing.filter(a => a.connId == id).forEach(a => a.forceEose());
}
sendClose() {
for (const qt of this.#tracing) {
qt.sendClose();
}
this.cleanup();
}
eose(sub: string, conn: Readonly<Connection>) {
const qt = this.#tracing.find(a => a.id === sub && a.connId === conn.Id);
qt?.gotEose();
if (!this.#leaveOpen) {
qt?.sendClose();
}
}
/**
* Get the progress to EOSE, can be used to determine when we should load more content
*/
get progress() {
const thisProgress = this.#tracing.reduce((acc, v) => (acc += v.finished ? 1 : 0), 0) / this.#tracing.length;
if (isNaN(thisProgress)) {
return 0;
}
return thisProgress;
}
#onProgress() {
const isFinished = this.progress === 1;
if (this.feed.loading !== isFinished) {
this.#log("%s loading=%s, progress=%d", this.id, this.feed.loading, this.progress);
this.feed.loading = isFinished;
}
}
#stopCheckTraces() {
if (this.#checkTrace) {
clearInterval(this.#checkTrace);
}
}
#checkTraces() {
this.#stopCheckTraces();
this.#checkTrace = setInterval(() => {
for (const v of this.#tracing) {
if (v.runtime > 5_000 && !v.finished) {
v.forceEose();
}
}
}, 500);
}
#canSendQuery(c: Connection, q: BuiltRawReqFilter) {
if (q.relay && q.relay !== c.Address) {
return false;
}
if (!q.relay && c.Ephemeral) {
this.#log("Cant send non-specific REQ to ephemeral connection %O %O %O", q, q.relay, c);
return false;
}
if (q.filters.some(a => a.search) && !c.SupportsNip(Nips.Search)) {
this.#log("Cant send REQ to non-search relay", c.Address);
return false;
}
return true;
}
#sendQueryInternal(c: Connection, q: BuiltRawReqFilter) {
const qt = new QueryTrace(
c.Address,
q.filters,
c.Id,
x => c.CloseReq(x),
() => this.#onProgress()
);
this.#tracing.push(qt);
this.#reComputeFilters();
c.QueueReq(["REQ", qt.id, ...q.filters], () => qt.sentToRelay());
return qt;
}
#reComputeFilters() {
console.time("reComputeFilters");
this.#allFilters = flatMerge(this.#tracing.flatMap(a => a.filters).flatMap(expandFilter));
console.timeEnd("reComputeFilters");
}
}

16
packages/system/src/RelayInfo.ts Normal file
View File

@ -0,0 +1,16 @@
export interface RelayInfo {
name?: string;
description?: string;
pubkey?: string;
contact?: string;
supported_nips?: number[];
software?: string;
version?: string;
limitation?: {
payment_required: boolean;
max_subscriptions: number;
max_filters: number;
max_event_tags: number;
auth_required: boolean;
};
}

249
packages/system/src/RequestBuilder.ts Normal file
View File

@ -0,0 +1,249 @@
import { ReqFilter, u256, HexKey, EventKind } from ".";
import { appendDedupe, dedupe } from "./Util";
import { diffFilters } from "./RequestSplitter";
import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./GossipModel";
import { mergeSimilar } from "./RequestMerger";
/**
* Which strategy is used when building REQ filters
*/
export enum RequestStrategy {
/**
* Use the users default relays to fetch events,
* this is the fallback option when there is no better way to query a given filter set
*/
DefaultRelays = 1,
/**
* Using a cached copy of the authors relay lists NIP-65, split a given set of request filters by pubkey
*/
AuthorsRelays = 2,
/**
* Relay hints are usually provided when using replies
*/
RelayHintedEventIds = 3,
}
/**
* A built REQ filter ready for sending to System
*/
export interface BuiltRawReqFilter {
filters: Array<ReqFilter>;
relay: string;
strategy: RequestStrategy;
}
export interface RequestBuilderOptions {
leaveOpen?: boolean;
relays?: Array<string>;
/**
* Do not apply diff logic and always use full filters for query
*/
skipDiff?: boolean;
}
/**
* Nostr REQ builder
*/
export class RequestBuilder {
id: string;
#builders: Array<RequestFilterBuilder>;
#options?: RequestBuilderOptions;
constructor(id: string) {
this.id = id;
this.#builders = [];
}
get numFilters() {
return this.#builders.length;
}
get options() {
return this.#options;
}
withFilter() {
const ret = new RequestFilterBuilder();
this.#builders.push(ret);
return ret;
}
withOptions(opt: RequestBuilderOptions) {
this.#options = {
...this.#options,
...opt,
};
return this;
}
buildRaw(): Array<ReqFilter> {
return this.#builders.map(f => f.filter);
}
build(relays: RelayCache): Array<BuiltRawReqFilter> {
const expanded = this.#builders.flatMap(a => a.build(relays, this.id));
return this.#groupByRelay(expanded);
}
/**
* Detects a change in request from a previous set of filters
* @param q All previous filters merged
* @returns
*/
buildDiff(relays: RelayCache, filters: Array<ReqFilter>): Array<BuiltRawReqFilter> {
const next = this.buildRaw();
const diff = diffFilters(filters, next);
if (diff.changed) {
return splitAllByWriteRelays(relays, diff.added).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
filters: a.filters,
relay: a.relay,
};
});
}
return [];
}
/**
* Merge a set of expanded filters into the smallest number of subscriptions by merging similar requests
* @param expanded
* @returns
*/
#groupByRelay(expanded: Array<BuiltRawReqFilter>) {
const relayMerged = expanded.reduce((acc, v) => {
const existing = acc.get(v.relay);
if (existing) {
existing.push(v);
} else {
acc.set(v.relay, [v]);
}
return acc;
}, new Map<string, Array<BuiltRawReqFilter>>());
const filtersSquashed = [...relayMerged.values()].map(a => {
return {
filters: mergeSimilar(a.flatMap(b => b.filters)),
relay: a[0].relay,
strategy: a[0].strategy,
} as BuiltRawReqFilter;
});
return filtersSquashed;
}
}
/**
* Builder class for a single request filter
*/
export class RequestFilterBuilder {
#filter: ReqFilter = {};
#relayHints = new Map<u256, Array<string>>();
get filter() {
return { ...this.#filter };
}
get relayHints() {
return new Map(this.#relayHints);
}
ids(ids: Array<u256>) {
this.#filter.ids = appendDedupe(this.#filter.ids, ids);
return this;
}
id(id: u256, relay?: string) {
if (relay) {
this.#relayHints.set(id, appendDedupe(this.#relayHints.get(id), [relay]));
}
return this.ids([id]);
}
authors(authors?: Array<HexKey>) {
if (!authors) return this;
this.#filter.authors = appendDedupe(this.#filter.authors, authors);
return this;
}
kinds(kinds?: Array<EventKind>) {
if (!kinds) return this;
this.#filter.kinds = appendDedupe(this.#filter.kinds, kinds);
return this;
}
since(since?: number) {
if (!since) return this;
this.#filter.since = since;
return this;
}
until(until?: number) {
if (!until) return this;
this.#filter.until = until;
return this;
}
limit(limit?: number) {
if (!limit) return this;
this.#filter.limit = limit;
return this;
}
tag(key: "e" | "p" | "d" | "t" | "r", value?: Array<string>) {
if (!value) return this;
this.#filter[`#${key}`] = value;
return this;
}
search(keyword?: string) {
if (!keyword) return this;
this.#filter.search = keyword;
return this;
}
/**
* Build/expand this filter into a set of relay specific queries
*/
build(relays: RelayCache, id: string): Array<BuiltRawReqFilter> {
// when querying for specific event ids with relay hints
// take the first approach which is to split the filter by relay
if (this.#filter.ids && this.#relayHints.size > 0) {
const relays = dedupe([...this.#relayHints.values()].flat());
return relays.map(r => {
return {
filters: [
{
...this.#filter,
ids: [...this.#relayHints.entries()].filter(([, v]) => v.includes(r)).map(([k]) => k),
},
],
relay: r,
strategy: RequestStrategy.RelayHintedEventIds,
};
});
}
// If any authors are set use the gossip model to fetch data for each author
if (this.#filter.authors) {
const split = splitByWriteRelays(relays, this.#filter);
return split.map(a => {
return {
filters: [a.filter],
relay: a.relay,
strategy: RequestStrategy.AuthorsRelays,
};
});
}
return [
{
filters: [this.filter],
relay: "",
strategy: RequestStrategy.DefaultRelays,
},
];
}
}

48
packages/system/src/RequestExpander.ts Normal file
View File

@ -0,0 +1,48 @@
import { u256, ReqFilter } from "./Nostr";
export interface FlatReqFilter {
ids?: u256;
authors?: u256;
kinds?: number;
"#e"?: u256;
"#p"?: u256;
"#t"?: string;
"#d"?: string;
"#r"?: string;
search?: string;
since?: number;
until?: number;
limit?: number;
}
/**
* Expand a filter into its most fine grained form
*/
export function expandFilter(f: ReqFilter): Array<FlatReqFilter> {
const ret: Array<FlatReqFilter> = [];
const src = Object.entries(f);
const keys = src.filter(([, v]) => Array.isArray(v)).map(a => a[0]);
const props = src.filter(([, v]) => !Array.isArray(v));
function generateCombinations(index: number, currentCombination: FlatReqFilter) {
if (index === keys.length) {
ret.push(currentCombination);
return;
}
const key = keys[index];
const values = (f as Record<string, Array<string | number>>)[key];
for (let i = 0; i < values.length; i++) {
const value = values[i];
const updatedCombination = { ...currentCombination, [key]: value };
generateCombinations(index + 1, updatedCombination);
}
}
generateCombinations(0, {
...Object.fromEntries(props),
});
return ret;
}

20
packages/system/src/RequestMatcher.ts Normal file
View File

@ -0,0 +1,20 @@
import { NostrEvent, ReqFilter } from "./Nostr";
export function eventMatchesFilter(ev: NostrEvent, filter: ReqFilter) {
if (!(filter.ids?.includes(ev.id) ?? false)) {
return false;
}
if (!(filter.authors?.includes(ev.pubkey) ?? false)) {
return false;
}
if (!(filter.kinds?.includes(ev.kind) ?? false)) {
return false;
}
if (filter.since && ev.created_at < filter.since) {
return false;
}
if (filter.until && ev.created_at > filter.until) {
return false;
}
return true;
}

156
packages/system/src/RequestMerger.ts Normal file
View File

@ -0,0 +1,156 @@
import { ReqFilter } from ".";
import { FlatReqFilter } from "./RequestExpander";
import { distance } from "./Util";
/**
* Keys which can change the entire meaning of the filter outside the array types
*/
const DiscriminatorKeys = ["since", "until", "limit", "search"];
export function canMergeFilters(a: FlatReqFilter | ReqFilter, b: FlatReqFilter | ReqFilter): boolean {
const aObj = a as Record<string, string | number | undefined>;
const bObj = b as Record<string, string | number | undefined>;
for (const key of DiscriminatorKeys) {
if (key in aObj || key in bObj) {
if (aObj[key] !== bObj[key]) {
return false;
}
}
}
return distance(aObj, bObj) <= 1;
}
export function mergeSimilar(filters: Array<ReqFilter>): Array<ReqFilter> {
console.time("mergeSimilar");
const ret = [];
const fCopy = [...filters];
while (fCopy.length > 0) {
const current = fCopy.shift()!;
const mergeSet = [current];
for (let i = 0; i < fCopy.length; i++) {
const f = fCopy[i];
if (mergeSet.every(v => canMergeFilters(v, f))) {
mergeSet.push(fCopy.splice(i, 1)[0]);
i--;
}
}
ret.push(simpleMerge(mergeSet));
}
console.timeEnd("mergeSimilar");
return ret;
}
/**
* Simply flatten all filters into one
* @param filters
* @returns
*/
export function simpleMerge(filters: Array<ReqFilter>) {
const result: any = {};
filters.forEach(filter => {
Object.entries(filter).forEach(([key, value]) => {
if (Array.isArray(value)) {
if (result[key] === undefined) {
result[key] = [...value];
} else {
result[key] = [...new Set([...result[key], ...value])];
}
} else {
result[key] = value;
}
});
});
return result as ReqFilter;
}
/**
* Check if a filter includes another filter, as in the bigger filter will include the same results as the samller filter
* @param bigger
* @param smaller
* @returns
*/
export function filterIncludes(bigger: ReqFilter, smaller: ReqFilter) {
const outside = bigger as Record<string, Array<string | number> | number>;
for (const [k, v] of Object.entries(smaller)) {
if (outside[k] === undefined) {
return false;
}
if (Array.isArray(v) && v.some(a => !(outside[k] as Array<string | number>).includes(a))) {
return false;
}
if (typeof v === "number") {
if (k === "since" && (outside[k] as number) > v) {
return false;
}
if (k === "until" && (outside[k] as number) < v) {
return false;
}
// limit cannot be checked and is ignored
}
}
return true;
}
/**
* Merge expanded flat filters into combined concise filters
* @param all
* @returns
*/
export function flatMerge(all: Array<FlatReqFilter>): Array<ReqFilter> {
console.time("flatMerge");
let ret: Array<ReqFilter> = [];
// to compute filters which can be merged we need to calucate the distance change between each filter
// then we can merge filters which are exactly 1 change diff from each other
function mergeFiltersInSet(filters: Array<FlatReqFilter>) {
const result: any = {};
filters.forEach(f => {
const filter = f as Record<string, string | number>;
Object.entries(filter).forEach(([key, value]) => {
if (!DiscriminatorKeys.includes(key)) {
if (result[key] === undefined) {
result[key] = [value];
} else {
result[key] = [...new Set([...result[key], value])];
}
} else {
result[key] = value;
}
});
});
return result as ReqFilter;
}
// reducer, kinda verbose
while (all.length > 0) {
const currentFilter = all.shift()!;
const mergeSet = [currentFilter];
for (let i = 0; i < all.length; i++) {
const f = all[i];
if (mergeSet.every(a => canMergeFilters(a, f))) {
mergeSet.push(all.splice(i, 1)[0]);
i--;
}
}
ret.push(mergeFiltersInSet(mergeSet));
}
while (true) {
const n = mergeSimilar([...ret]);
if (n.length === ret.length) {
break;
}
ret = n;
}
console.timeEnd("flatMerge");
console.debug(ret);
return ret;
}

18
packages/system/src/RequestSplitter.ts Normal file
View File

@ -0,0 +1,18 @@
import { ReqFilter } from ".";
import { deepEqual } from "./Util";
import { expandFilter } from "./RequestExpander";
import { flatMerge } from "./RequestMerger";
export function diffFilters(prev: Array<ReqFilter>, next: Array<ReqFilter>) {
const prevExpanded = prev.flatMap(expandFilter);
const nextExpanded = next.flatMap(expandFilter);
const added = flatMerge(nextExpanded.filter(a => !prevExpanded.some(b => deepEqual(a, b))));
const removed = flatMerge(prevExpanded.filter(a => !nextExpanded.some(b => deepEqual(a, b))));
return {
added,
removed,
changed: added.length > 0 || removed.length > 0,
};
}

64
packages/system/src/SystemWorker.ts Normal file
View File

@ -0,0 +1,64 @@
import { SystemSnapshot, SystemInterface } from ".";
import { AuthHandler, ConnectionStateSnapshot, RelaySettings } from "./Connection";
import ExternalStore from "./ExternalStore";
import { NostrEvent } from "./Nostr";
import { NoteStore } from "./NoteCollection";
import { Query } from "./Query";
import { RequestBuilder } from "./RequestBuilder";
export class SystemWorker extends ExternalStore<SystemSnapshot> implements SystemInterface {
#port: MessagePort;
constructor() {
super();
if ("SharedWorker" in window) {
const worker = new SharedWorker("/system.js");
this.#port = worker.port;
this.#port.onmessage = m => this.#onMessage(m);
} else {
throw new Error("SharedWorker is not supported");
}
}
HandleAuth?: AuthHandler;
get Sockets(): ConnectionStateSnapshot[] {
throw new Error("Method not implemented.");
}
Query<T extends NoteStore>(type: new () => T, req: RequestBuilder | null): Query | undefined {
throw new Error("Method not implemented.");
}
CancelQuery(sub: string): void {
throw new Error("Method not implemented.");
}
GetQuery(sub: string): Query | undefined {
throw new Error("Method not implemented.");
}
ConnectToRelay(address: string, options: RelaySettings): Promise<void> {
throw new Error("Method not implemented.");
}
DisconnectRelay(address: string): void {
throw new Error("Method not implemented.");
}
BroadcastEvent(ev: NostrEvent): void {
throw new Error("Method not implemented.");
}
WriteOnceToRelay(relay: string, ev: NostrEvent): Promise<void> {
throw new Error("Method not implemented.");
}
takeSnapshot(): SystemSnapshot {
throw new Error("Method not implemented.");
}
#onMessage(e: MessageEvent<any>) {
console.debug(e);
}
}

88
packages/system/src/Tag.ts Normal file
View File

@ -0,0 +1,88 @@
import { HexKey, u256 } from "./Nostr";
import { unwrap } from "./Util";
export default class Tag {
Original: string[];
Key: string;
Event?: u256;
PubKey?: HexKey;
Relay?: string;
Marker?: string;
Hashtag?: string;
DTag?: string;
ATag?: string;
Index: number;
Invalid: boolean;
LNURL?: string;
constructor(tag: string[], index: number) {
this.Original = tag;
this.Key = tag[0];
this.Index = index;
this.Invalid = false;
switch (this.Key) {
case "e": {
// ["e", <event-id>, <relay-url>, <marker>]
this.Event = tag[1];
this.Relay = tag.length > 2 ? tag[2] : undefined;
this.Marker = tag.length > 3 ? tag[3] : undefined;
if (!this.Event) {
this.Invalid = true;
}
break;
}
case "p": {
// ["p", <pubkey>]
this.PubKey = tag[1];
if (!this.PubKey) {
this.Invalid = true;
}
break;
}
case "d": {
this.DTag = tag[1];
break;
}
case "a": {
this.ATag = tag[1];
break;
}
case "t": {
this.Hashtag = tag[1];
break;
}
case "delegation": {
this.PubKey = tag[1];
break;
}
case "zap": {
this.LNURL = tag[1];
break;
}
}
}
ToObject(): string[] | null {
switch (this.Key) {
case "e": {
let ret = ["e", this.Event, this.Relay, this.Marker];
const trimEnd = ret.reverse().findIndex(a => a !== undefined);
ret = ret.reverse().slice(0, ret.length - trimEnd);
return <string[]>ret;
}
case "p": {
return this.PubKey ? ["p", this.PubKey] : null;
}
case "t": {
return ["t", unwrap(this.Hashtag)];
}
case "d": {
return ["d", unwrap(this.DTag)];
}
default: {
return this.Original;
}
}
}
}

122
packages/system/src/Util.ts Normal file
View File

@ -0,0 +1,122 @@
import * as utils from "@noble/curves/abstract/utils";
import * as secp from "@noble/curves/secp256k1";
import { sha256 as sha2 } from "@noble/hashes/sha256";
import { bech32 } from "bech32";
import { NostrEvent, u256 } from "./Nostr";
export function unwrap<T>(v: T | undefined | null): T {
if (v === undefined || v === null) {
throw new Error("missing value");
}
return v;
}
/**
* Convert hex to bech32
*/
export function hexToBech32(hrp: string, hex?: string) {
if (typeof hex !== "string" || hex.length === 0 || hex.length % 2 !== 0) {
return "";
}
try {
const buf = utils.hexToBytes(hex);
return bech32.encode(hrp, bech32.toWords(buf));
} catch (e) {
console.warn("Invalid hex", hex, e);
return "";
}
}
export function sanitizeRelayUrl(url: string) {
try {
return new URL(url).toString();
} catch {
// ignore
}
}
export function unixNow() {
return Math.floor(unixNowMs() / 1000);
}
export function unixNowMs() {
return new Date().getTime();
}
export function deepEqual(x: any, y: any): boolean {
const ok = Object.keys,
tx = typeof x,
ty = typeof y;
return x && y && tx === "object" && tx === ty
? ok(x).length === ok(y).length && ok(x).every(key => deepEqual(x[key], y[key]))
: x === y;
}
/**
* Compute the "distance" between two objects by comparing their difference in properties
* Missing/Added keys result in +10 distance
* This is not recursive
*/
export function distance(a: any, b: any): number {
const keys1 = Object.keys(a);
const keys2 = Object.keys(b);
const maxKeys = keys1.length > keys2.length ? keys1 : keys2;
let distance = 0;
for (const key of maxKeys) {
if (key in a && key in b) {
if (Array.isArray(a[key]) && Array.isArray(b[key])) {
const aa = a[key] as Array<string | number>;
const bb = b[key] as Array<string | number>;
if (aa.length === bb.length) {
if (aa.some(v => !bb.includes(v))) {
distance++;
}
} else {
distance++;
}
} else if (a[key] !== b[key]) {
distance++;
}
} else {
distance += 10;
}
}
return distance;
}
export function dedupe<T>(v: Array<T>) {
return [...new Set(v)];
}
export function appendDedupe<T>(a?: Array<T>, b?: Array<T>) {
return dedupe([...(a ?? []), ...(b ?? [])]);
}
export function findTag(e: NostrEvent, tag: string) {
const maybeTag = e.tags.find(evTag => {
return evTag[0] === tag;
});
return maybeTag && maybeTag[1];
}
export const sha256 = (str: string | Uint8Array): u256 => {
return utils.bytesToHex(sha2(str));
}
export function getPublicKey(privKey: string) {
return utils.bytesToHex(secp.schnorr.getPublicKey(privKey));
}
export function bech32ToHex(str: string) {
try {
const nKey = bech32.decode(str, 1_000);
const buff = bech32.fromWords(nKey.words);
return utils.bytesToHex(Uint8Array.from(buff));
} catch (e) {
return str;
}
}

30
packages/system/src/WorkQueue.ts Normal file
View File

@ -0,0 +1,30 @@
export interface WorkQueueItem {
next: () => Promise<unknown>;
resolve(v: unknown): void;
reject(e: unknown): void;
}
export async function processWorkQueue(queue?: Array<WorkQueueItem>, queueDelay = 200) {
while (queue && queue.length > 0) {
const v = queue.shift();
if (v) {
try {
const ret = await v.next();
v.resolve(ret);
} catch (e) {
v.reject(e);
}
}
}
setTimeout(() => processWorkQueue(queue, queueDelay), queueDelay);
}
export const barrierQueue = async <T>(queue: Array<WorkQueueItem>, then: () => Promise<T>): Promise<T> => {
return new Promise<T>((resolve, reject) => {
queue.push({
next: then,
resolve,
reject,
});
});
};

68
packages/system/src/cache/index.ts vendored Normal file
View File

@ -0,0 +1,68 @@
import { HexKey, NostrEvent, UserMetadata } from "..";
import { hexToBech32, unixNowMs } from "../Util";
export interface MetadataCache extends UserMetadata {
/**
* When the object was saved in cache
*/
loaded: number;
/**
* When the source metadata event was created
*/
created: number;
/**
* The pubkey of the owner of this metadata
*/
pubkey: HexKey;
/**
* The bech32 encoded pubkey
*/
npub: string;
/**
* Pubkey of zapper service
*/
zapService?: HexKey;
/**
* If the nip05 is valid for this user
*/
isNostrAddressValid: boolean;
}
export function mapEventToProfile(ev: NostrEvent) {
try {
const data: UserMetadata = JSON.parse(ev.content);
return {
...data,
pubkey: ev.pubkey,
npub: hexToBech32("npub", ev.pubkey),
created: ev.created_at,
loaded: unixNowMs(),
} as MetadataCache;
} catch (e) {
console.error("Failed to parse JSON", ev, e);
}
}
export interface CacheStore<T> {
preload(): Promise<void>;
getFromCache(key?: string): T | undefined;
get(key?: string): Promise<T | undefined>;
bulkGet(keys: Array<string>): Promise<Array<T>>;
set(obj: T): Promise<void>;
bulkSet(obj: Array<T>): Promise<void>;
update<TCachedWithCreated extends T & { created: number; loaded: number }>(m: TCachedWithCreated): Promise<"new" | "updated" | "refresh" | "no_change">
/**
* Loads a list of rows from disk cache
* @param keys List of ids to load
* @returns Keys that do not exist on disk cache
*/
buffer(keys: Array<string>): Promise<Array<string>>;
clear(): Promise<void>;
}

44
packages/system/src/index.ts Normal file
View File

@ -0,0 +1,44 @@
import { AuthHandler, RelaySettings, ConnectionStateSnapshot } from "./Connection";
import { RequestBuilder } from "./RequestBuilder";
import { NoteStore } from "./NoteCollection";
import { Query } from "./Query";
import { NostrEvent, ReqFilter } from "./Nostr";
export * from "./NostrSystem";
export { default as EventKind } from "./EventKind";
export * from "./Nostr";
export * from "./Links";
export { default as Tag } from "./Tag";
export * from "./Nips";
export * from "./RelayInfo";
export * from "./EventExt";
export * from "./Connection";
export * from "./NoteCollection";
export * from "./RequestBuilder";
export * from "./EventPublisher";
export * from "./EventBuilder";
export * from "./NostrLink";
export * from "./cache";
export * from "./ProfileCache";
export interface SystemInterface {
/**
* Handler function for NIP-42
*/
HandleAuth?: AuthHandler;
get Sockets(): Array<ConnectionStateSnapshot>;
GetQuery(id: string): Query | undefined;
Query<T extends NoteStore>(type: { new(): T }, req: RequestBuilder | null): Query | undefined;
ConnectToRelay(address: string, options: RelaySettings): Promise<void>;
DisconnectRelay(address: string): void;
BroadcastEvent(ev: NostrEvent): void;
WriteOnceToRelay(relay: string, ev: NostrEvent): Promise<void>;
}
export interface SystemSnapshot {
queries: Array<{
id: string;
filters: Array<ReqFilter>;
subFilters: Array<ReqFilter>;
}>;
}