Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: @snort/system CacheRelay

Browse Source
This commit is contained in:
Kieran
2024-01-23 15:35:28 +00:00
parent d6c578fafc
commit 5cea096067
29 changed files with 296 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
packages/system/src/cache-relay.ts Normal file
View File

@ -0,0 +1,17 @@
import { NostrEvent, OkResponse, ReqCommand } from "./nostr";
/**
* A cache relay is an always available local (local network / browser worker) relay
* Which should contain all of the content we're looking for and respond quickly.
*/
export interface CacheRelay {
/**
* Write event to cache relay
*/
event(ev: NostrEvent): Promise<OkResponse>;
/**
* Read event from cache relay
*/
query(req: ReqCommand): Promise<Array<NostrEvent>>;
}

4
packages/system/src/connection-pool.ts
View File

@ -2,8 +2,8 @@ import { removeUndefined, sanitizeRelayUrl, unwrap } from "@snort/shared";
import debug from "debug";
import EventEmitter from "eventemitter3";
import { Connection, ConnectionStateSnapshot, OkResponse, RelaySettings } from "./connection";
import { NostrEvent, TaggedNostrEvent } from "./nostr";
import { Connection, ConnectionStateSnapshot, RelaySettings } from "./connection";
import { NostrEvent, OkResponse, TaggedNostrEvent } from "./nostr";
import { pickRelaysForReply } from "./outbox-model";
import { SystemInterface } from ".";

10
packages/system/src/connection.ts
View File

@ -6,7 +6,7 @@ import EventEmitter from "eventemitter3";
import { DefaultConnectTimeout } from "./const";
import { ConnectionStats } from "./connection-stats";
import { NostrEvent, ReqCommand, ReqFilter, TaggedNostrEvent, u256 } from "./nostr";
import { NostrEvent, OkResponse, ReqCommand, ReqFilter, TaggedNostrEvent, u256 } from "./nostr";
import { RelayInfo } from "./relay-info";
import EventKind from "./event-kind";
import { EventExt } from "./event-ext";
@ -19,14 +19,6 @@ export interface RelaySettings {
write: boolean;
}
export interface OkResponse {
ok: boolean;
id: string;
relay: string;
message?: string;
event: NostrEvent;
}
/**
* Snapshot of connection stats
*/

14
packages/system/src/index.ts
View File

@ -1,14 +1,15 @@
import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
import { RelaySettings, ConnectionStateSnapshot } from "./connection";
import { RequestBuilder } from "./request-builder";
import { NostrEvent, ReqFilter, TaggedNostrEvent } from "./nostr";
import { NostrEvent, OkResponse, ReqFilter, TaggedNostrEvent } from "./nostr";
import { ProfileLoaderService } from "./profile-cache";
import { RelayCache, RelayMetadataLoader } from "./outbox-model";
import { AuthorsRelaysCache, RelayMetadataLoader } from "./outbox-model";
import { Optimizer } from "./query-optimizer";
import { base64 } from "@scure/base";
import { CachedTable } from "@snort/shared";
import { ConnectionPool } from "./connection-pool";
import EventEmitter from "eventemitter3";
import { QueryEvents } from "./query";
import { CacheRelay } from "./cache-relay";
export { NostrSystem } from "./nostr-system";
export { default as EventKind } from "./event-kind";
@ -133,7 +134,7 @@ export interface SystemInterface {
/**
* Relay cache for "Gossip" model
*/
get relayCache(): RelayCache;
get relayCache(): AuthorsRelaysCache;
/**
* Query optimizer
@ -154,6 +155,11 @@ export interface SystemInterface {
* Main connection pool
*/
get pool(): ConnectionPool;
/**
* Local relay cache service
*/
get cacheRelay(): CacheRelay | undefined;
}
export interface SystemSnapshot {

8
packages/system/src/nostr-system.ts
View File

@ -2,8 +2,8 @@ import debug from "debug";
import EventEmitter from "eventemitter3";
import { CachedTable } from "@snort/shared";
import { NostrEvent, TaggedNostrEvent } from "./nostr";
import { RelaySettings, ConnectionStateSnapshot, OkResponse } from "./connection";
import { NostrEvent, TaggedNostrEvent, OkResponse } from "./nostr";
import { RelaySettings, ConnectionStateSnapshot } from "./connection";
import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
import { RelayMetricHandler } from "./relay-metric-handler";
import {
@ -24,6 +24,7 @@ import { RelayMetadataLoader } from "./outbox-model";
import { Optimizer, DefaultOptimizer } from "./query-optimizer";
import { ConnectionPool, DefaultConnectionPool } from "./connection-pool";
import { QueryManager } from "./query-manager";
import { CacheRelay } from "./cache-relay";
export interface NostrSystemEvents {
change: (state: SystemSnapshot) => void;
@ -37,6 +38,7 @@ export interface NostrsystemProps {
profileCache?: CachedTable<CachedMetadata>;
relayMetrics?: CachedTable<RelayMetrics>;
eventsCache?: CachedTable<NostrEvent>;
cacheRelay?: CacheRelay;
optimizer?: Optimizer;
db?: SnortSystemDb;
checkSigs?: boolean;
@ -82,6 +84,7 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
readonly pool: ConnectionPool;
readonly eventsCache: CachedTable<NostrEvent>;
readonly relayLoader: RelayMetadataLoader;
readonly cacheRelay: CacheRelay | undefined;
/**
* Check event signatures (reccomended)
@ -95,6 +98,7 @@ export class NostrSystem extends EventEmitter<NostrSystemEvents> implements Syst
this.relayMetricsCache = props.relayMetrics ?? new RelayMetricCache(props.db?.relayMetrics);
this.eventsCache = props.eventsCache ?? new EventsCache(props.db?.events);
this.optimizer = props.optimizer ?? DefaultOptimizer;
this.cacheRelay = props.cacheRelay;
this.profileLoader = new ProfileLoaderService(this, this.profileCache);
this.relayMetricsHandler = new RelayMetricHandler(this.relayMetricsCache);

12
packages/system/src/nostr.ts
View File

@ -57,8 +57,8 @@ export interface ReqFilter {
since?: number;
until?: number;
limit?: number;
not?: ReqFilter;
[key: string]: Array<string> | Array<number> | string | number | undefined | ReqFilter;
ids_only?: boolean;
[key: string]: Array<string> | Array<number> | string | number | undefined | boolean;
}
/**
@ -92,3 +92,11 @@ export interface IMeta {
alt?: string;
fallback?: Array<string>;
}
export interface OkResponse {
ok: boolean;
id: string;
relay: string;
message?: string;
event: NostrEvent;
}

10
packages/system/src/outbox-model.ts
View File

@ -33,14 +33,14 @@ export interface RelayTaggedFilters {
const logger = debug("OutboxModel");
export interface RelayCache {
export interface AuthorsRelaysCache {
getFromCache(pubkey?: string): UsersRelays | undefined;
update(obj: UsersRelays): Promise<"new" | "updated" | "refresh" | "no_change">;
buffer(keys: Array<string>): Promise<Array<string>>;
bulkSet(objs: Array<UsersRelays>): Promise<void>;
}
export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilter>) {
export function splitAllByWriteRelays(cache: AuthorsRelaysCache, filters: Array<ReqFilter>) {
const allSplit = filters
.map(a => splitByWriteRelays(cache, a))
.reduce((acc, v) => {
@ -66,7 +66,7 @@ export function splitAllByWriteRelays(cache: RelayCache, filters: Array<ReqFilte
/**
* Split filters by authors
*/
export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter> {
export function splitByWriteRelays(cache: AuthorsRelaysCache, filter: ReqFilter, pickN?: number): Array<RelayTaggedFilter> {
const authors = filter.authors;
if ((authors?.length ?? 0) === 0) {
return [
@ -108,7 +108,7 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter, pickN?:
* Split filters by author
*/
export function splitFlatByWriteRelays(
cache: RelayCache,
cache: AuthorsRelaysCache,
input: Array<FlatReqFilter>,
pickN?: number,
): Array<RelayTaggedFlatFilters> {
@ -146,7 +146,7 @@ export function splitFlatByWriteRelays(
/**
* Pick most popular relays for each authors
*/
export function pickTopRelays(cache: RelayCache, authors: Array<string>, n: number, type: "write" | "read") {
export function pickTopRelays(cache: AuthorsRelaysCache, authors: Array<string>, n: number, type: "write" | "read") {
// map of pubkey -> [write relays]
const allRelays = authors.map(a => {
return {

17
packages/system/src/query-manager.ts
View File

@ -1,6 +1,6 @@
import debug from "debug";
import EventEmitter from "eventemitter3";
import { BuiltRawReqFilter, RequestBuilder, SystemInterface, TaggedNostrEvent } from ".";
import { BuiltRawReqFilter, RequestBuilder, RequestStrategy, SystemInterface, TaggedNostrEvent } from ".";
import { Query, TraceReport } from "./query";
import { FilterCacheLayer, IdsFilterCacheLayer } from "./filter-cache-layer";
import { trimFilters } from "./request-trim";
@ -105,9 +105,17 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
}
async #send(q: Query, qSend: BuiltRawReqFilter) {
if (qSend.strategy === RequestStrategy.CacheRelay && this.#system.cacheRelay) {
const qt = q.insertCompletedTrace(qSend, []);
const res = await this.#system.cacheRelay.query(["REQ", qt.id, ...qSend.filters]);
q.feed.add(res?.map(a => ({ ...a, relays: [] }) as TaggedNostrEvent));
return;
}
for (const qfl of this.#queryCacheLayers) {
qSend = await qfl.processFilter(q, qSend);
}
// automated outbox model, load relays for queried authors
for (const f of qSend.filters) {
if (f.authors) {
this.#system.relayLoader.TrackKeys(f.authors);
@ -156,6 +164,13 @@ export class QueryManager extends EventEmitter<QueryManagerEvents> {
}
}
/**
* Split request into 2 branches.
* 1. Request cache for results
* 2. Send query to relays
*/
#splitSyncRequest(req: BuiltRawReqFilter) {}
#cleanup() {
let changed = false;
for (const [k, v] of this.#queries) {

6
packages/system/src/query.ts
View File

@ -323,15 +323,15 @@ export class Query extends EventEmitter<QueryEvents> {
}
}
#emitFilters() {
async #emitFilters() {
this.#log("Starting emit of %s", this.id);
const existing = this.filters;
if (!(this.request.options?.skipDiff ?? false) && existing.length > 0) {
const filters = this.request.buildDiff(this.#system, existing);
const filters = await this.request.buildDiff(this.#system, existing);
this.#log("Build %s %O", this.id, filters);
filters.forEach(f => this.emit("request", this.id, f));
} else {
const filters = this.request.build(this.#system);
const filters = await this.request.build(this.#system);
this.#log("Build %s %O", this.id, filters);
filters.forEach(f => this.emit("request", this.id, f));
}

73
packages/system/src/request-builder.ts
View File

@ -5,27 +5,33 @@ import { appendDedupe, dedupe, sanitizeRelayUrl, unixNowMs, unwrap } from "@snor
import EventKind from "./event-kind";
import { NostrLink, NostrPrefix, SystemInterface } from ".";
import { ReqFilter, u256, HexKey } from "./nostr";
import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./outbox-model";
import { AuthorsRelaysCache, splitByWriteRelays, splitFlatByWriteRelays } from "./outbox-model";
import { CacheRelay } from "cache-relay";
/**
* Which strategy is used when building REQ filters
*/
export enum RequestStrategy {
export const enum RequestStrategy {
/**
* Use the users default relays to fetch events,
* this is the fallback option when there is no better way to query a given filter set
*/
DefaultRelays = 1,
DefaultRelays = "default",
/**
* Using a cached copy of the authors relay lists NIP-65, split a given set of request filters by pubkey
*/
AuthorsRelays = 2,
AuthorsRelays = "authors-relays",
/**
* Use pre-determined relays for query
*/
ExplicitRelays = 3,
ExplicitRelays = "explicit-relays",
/**
* Query the cache relay
*/
CacheRelay = "cache-relay",
}
/**
@ -121,21 +127,24 @@ export class RequestBuilder {
return this.#builders.map(f => f.filter);
}
build(system: SystemInterface): Array<BuiltRawReqFilter> {
const expanded = this.#builders.flatMap(a => a.build(system.relayCache, this.#options));
async build(system: SystemInterface): Promise<Array<BuiltRawReqFilter>> {
const expanded = (
await Promise.all(this.#builders.map(a => a.build(system.relayCache, system.cacheRelay, this.#options)))
).flat();
return this.#groupByRelay(system, expanded);
}
/**
* Detects a change in request from a previous set of filters
*/
buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Array<BuiltRawReqFilter> {
async buildDiff(system: SystemInterface, prev: Array<ReqFilter>): Promise<Array<BuiltRawReqFilter>> {
const start = unixNowMs();
const diff = system.optimizer.getDiff(prev, this.buildRaw());
const ts = unixNowMs() - start;
this.#log("buildDiff %s %d ms +%d", this.id, ts, diff.length);
if (diff.length > 0) {
// todo: fix
return splitFlatByWriteRelays(system.relayCache, diff).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
@ -143,8 +152,6 @@ export class RequestBuilder {
relay: a.relay,
};
});
} else {
this.#log(`Wasted ${ts} ms detecting no changes!`);
}
return [];
}
@ -284,12 +291,48 @@ export class RequestFilterBuilder {
/**
* Build/expand this filter into a set of relay specific queries
*/
build(relays: RelayCache, options?: RequestBuilderOptions): Array<BuiltRawReqFilter> {
async build(
relays: AuthorsRelaysCache,
cacheRelay?: CacheRelay,
options?: RequestBuilderOptions,
): Promise<Array<BuiltRawReqFilter>> {
// if since/until are set ignore sync split, cache relay wont be used
if (cacheRelay && this.#filter.since === undefined && this.#filter.until === undefined) {
const latest = await cacheRelay.query([
"REQ",
uuid(),
{
...this.#filter,
since: undefined,
until: undefined,
limit: 1,
},
]);
if (latest.length === 1) {
return [
...this.#buildFromFilter(relays, {
...this.#filter,
since: latest[0].created_at,
until: undefined,
limit: undefined,
}),
{
filters: [this.#filter],
relay: "==CACHE==",
strategy: RequestStrategy.CacheRelay,
},
];
}
}
return this.#buildFromFilter(relays, this.#filter, options);
}
#buildFromFilter(relays: AuthorsRelaysCache, f: ReqFilter, options?: RequestBuilderOptions) {
// use the explicit relay list first
if (this.#relays.size > 0) {
return [...this.#relays].map(r => {
return {
filters: [this.#filter],
filters: [f],
relay: r,
strategy: RequestStrategy.ExplicitRelays,
};
@ -297,8 +340,8 @@ export class RequestFilterBuilder {
}
// If any authors are set use the gossip model to fetch data for each author
if (this.#filter.authors) {
const split = splitByWriteRelays(relays, this.#filter, options?.outboxPickN);
if (f.authors) {
const split = splitByWriteRelays(relays, f, options?.outboxPickN);
return split.map(a => {
return {
filters: [a.filter],
@ -310,7 +353,7 @@ export class RequestFilterBuilder {
return [
{
filters: [this.#filter],
filters: [f],
relay: "",
strategy: RequestStrategy.DefaultRelays,
},

2
packages/system/src/worker/system-worker.ts
View File

@ -26,6 +26,7 @@ import { EventsCache } from "../cache/events";
import { RelayMetricHandler } from "../relay-metric-handler";
import debug from "debug";
import { ConnectionPool } from "connection-pool";
import { CacheRelay } from "cache-relay";
export class SystemWorker extends EventEmitter<NostrSystemEvents> implements SystemInterface {
#log = debug("SystemWorker");
@ -38,6 +39,7 @@ export class SystemWorker extends EventEmitter<NostrSystemEvents> implements Sys
readonly relayMetricsHandler: RelayMetricHandler;
readonly eventsCache: CachedTable<NostrEvent>;
readonly relayLoader: RelayMetadataLoader;
readonly cacheRelay: CacheRelay | undefined;
get checkSigs() {
return true;