Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use worker relay for events cache

Browse Source
This commit is contained in:
Kieran
2024-01-18 21:11:48 +00:00
parent c2f78dad1e
commit 32a6d56cf5
16 changed files with 172 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
packages/app/src/Cache/EventCacheWorker.ts Normal file
View File

@ -0,0 +1,96 @@
import { CachedTable, CacheEvents } from "@snort/shared";
import { NostrEvent } from "@snort/system";
import { WorkerRelayInterface } from "@snort/worker-relay";
import EventEmitter from "eventemitter3";
export class EventCacheWorker extends EventEmitter<CacheEvents> implements CachedTable<NostrEvent> {
#relay: WorkerRelayInterface;
#keys = new Set<string>();
#cache = new Map<string, NostrEvent>();
constructor(relay: WorkerRelayInterface) {
super();
this.#relay = relay;
}
async preload() {
const ids = await this.#relay.sql("select id from events", []);
this.#keys = new Set<string>(ids.map(a => a[0] as string));
return Promise.resolve();
}
keysOnTable(): string[] {
return [...this.#keys];
}
getFromCache(key?: string | undefined): NostrEvent | undefined {
if (key) {
return this.#cache.get(key);
}
}
discover(ev: NostrEvent) {
this.#keys.add(this.key(ev));
}
async get(key?: string | undefined): Promise<NostrEvent | undefined> {
if (key) {
const res = await this.bulkGet([key]);
if (res.length > 0) {
return res[0];
}
}
}
async bulkGet(keys: string[]): Promise<NostrEvent[]> {
const results = await this.#relay.req({
id: "EventCacheWorker.bulkGet",
filters: [
{
ids: keys,
},
],
});
for (const ev of results.result) {
this.#cache.set(ev.id, ev);
}
return results.result;
}
async set(obj: NostrEvent): Promise<void> {
await this.#relay.event(obj);
this.#keys.add(obj.id);
}
async bulkSet(obj: NostrEvent[] | readonly NostrEvent[]): Promise<void> {
await Promise.all(
obj.map(async a => {
await this.#relay.event(a);
this.#keys.add(a.id);
}),
);
}
async update<TWithCreated extends NostrEvent & { created: number; loaded: number }>(
m: TWithCreated,
): Promise<"new" | "refresh" | "updated" | "no_change"> {
if (await this.#relay.event(m)) {
return "updated";
}
return "no_change";
}
async buffer(keys: string[]): Promise<string[]> {
const missing = keys.filter(a => !this.#keys.has(a));
const res = await this.bulkGet(missing);
return missing.filter(a => !res.some(b => this.key(b) === a));
}
key(of: NostrEvent): string {
return of.id;
}
snapshot(): NostrEvent[] {
return [...this.#cache.values()];
}
}

41
packages/app/src/Cache/EventInteractionCache.ts
View File

@ -1,41 +0,0 @@
import { FeedCache } from "@snort/shared";
import { db, EventInteraction } from "@/Db";
import { LoginStore } from "@/Utils/Login";
export class EventInteractionCache extends FeedCache<EventInteraction> {
constructor() {
super("EventInteraction", db.eventInteraction);
}
key(of: EventInteraction): string {
return `${of.event}:${of.by}`;
}
override async preload(): Promise<void> {
await super.preload();
const data = window.localStorage.getItem("zap-cache");
if (data) {
const toImport = [...new Set<string>(JSON.parse(data) as Array<string>)].map(a => {
const ret = {
event: a,
by: LoginStore.takeSnapshot().publicKey,
zapped: true,
reacted: false,
reposted: false,
} as EventInteraction;
ret.id = this.key(ret);
return ret;
});
await this.bulkSet(toImport);
window.localStorage.removeItem("zap-cache");
}
await this.buffer([...this.onTable]);
}
takeSnapshot(): EventInteraction[] {
return [...this.cache.values()];
}
}

49
packages/app/src/Cache/FollowListCache.ts
View File

@ -1,49 +0,0 @@
import { unixNowMs } from "@snort/shared";
import { EventKind, RequestBuilder, socialGraphInstance, TaggedNostrEvent } from "@snort/system";
import { db } from "@/Db";
import { LoginSession } from "@/Utils/Login";
import { RefreshFeedCache } from "./RefreshFeedCache";
export class FollowListCache extends RefreshFeedCache<TaggedNostrEvent> {
constructor() {
super("FollowListCache", db.followLists);
}
buildSub(session: LoginSession, rb: RequestBuilder): void {
const since = this.newest();
rb.withFilter()
.kinds([EventKind.ContactList])
.authors(session.follows.item)
.since(since === 0 ? undefined : since);
}
async onEvent(evs: readonly TaggedNostrEvent[]) {
await Promise.all(
evs.map(async e => {
const update = await super.update({
...e,
created: e.created_at,
loaded: unixNowMs(),
});
if (update !== "no_change") {
socialGraphInstance.handleEvent(e);
}
}),
);
}
key(of: TaggedNostrEvent): string {
return of.pubkey;
}
takeSnapshot() {
return [...this.cache.values()];
}
override async preload() {
await super.preload();
this.cache.forEach(e => socialGraphInstance.handleEvent(e));
}
}

136
packages/app/src/Cache/FollowsFeed.ts
View File

@ -1,136 +0,0 @@
import { unixNow, unixNowMs } from "@snort/shared";
import { EventKind, RequestBuilder, SystemInterface, TaggedNostrEvent } from "@snort/system";
import { db } from "@/Db";
import { Day, Hour } from "@/Utils/Const";
import { LoginSession } from "@/Utils/Login";
import { RefreshFeedCache, TWithCreated } from "./RefreshFeedCache";
const WindowSize = Hour * 6;
const MaxCacheWindow = Day * 7;
export class FollowsFeedCache extends RefreshFeedCache<TaggedNostrEvent> {
#kinds = [EventKind.TextNote, EventKind.Repost, EventKind.Polls];
#oldest?: number;
constructor() {
super("FollowsFeedCache", db.followsFeed);
}
key(of: TWithCreated<TaggedNostrEvent>): string {
return of.id;
}
takeSnapshot(): TWithCreated<TaggedNostrEvent>[] {
return [...this.cache.values()];
}
buildSub(session: LoginSession, rb: RequestBuilder): void {
const authors = [...session.follows.item];
if (session.publicKey) {
authors.push(session.publicKey);
}
const since = this.newest();
rb.withFilter()
.kinds(this.#kinds)
.authors(authors)
.since(since === 0 ? unixNow() - WindowSize : since);
}
async onEvent(evs: readonly TaggedNostrEvent[]): Promise<void> {
const filtered = evs.filter(a => this.#kinds.includes(a.kind));
if (filtered.length > 0) {
await this.bulkSet(filtered);
this.emit(
"change",
filtered.map(a => this.key(a)),
);
}
}
override async preload() {
const start = unixNowMs();
const keys = (await this.table?.toCollection().primaryKeys()) ?? [];
this.onTable = new Set<string>(keys.map(a => a as string));
// load only latest 50 posts, rest can be loaded on-demand
const latest = await this.table?.orderBy("created_at").reverse().limit(50).toArray();
latest?.forEach(v => this.cache.set(this.key(v), v));
// cleanup older than 7 days
await this.table
?.where("created_at")
.below(unixNow() - MaxCacheWindow)
.delete();
const oldest = await this.table?.orderBy("created_at").first();
this.#oldest = oldest?.created_at;
this.emit("change", latest?.map(a => this.key(a)) ?? []);
this.log(`Loaded %d/%d in %d ms`, latest?.length ?? 0, keys.length, (unixNowMs() - start).toLocaleString());
}
async loadMore(system: SystemInterface, session: LoginSession, before: number) {
if (this.#oldest && before <= this.#oldest) {
const rb = new RequestBuilder(`${this.name}-loadmore`);
const authors = [...session.follows.item];
if (session.publicKey) {
authors.push(session.publicKey);
}
rb.withFilter()
.kinds(this.#kinds)
.authors(authors)
.until(before)
.since(before - WindowSize);
await system.Fetch(rb, async evs => {
await this.bulkSet(evs);
});
} else {
const latest = await this.table
?.where("created_at")
.between(before - WindowSize, before)
.reverse()
.sortBy("created_at");
latest?.forEach(v => {
const k = this.key(v);
this.cache.set(k, v);
this.onTable.add(k);
});
this.emit("change", latest?.map(a => this.key(a)) ?? []);
}
}
/**
* Backfill cache with new follows
*/
async backFill(system: SystemInterface, keys: Array<string>) {
if (keys.length === 0) return;
const rb = new RequestBuilder(`${this.name}-backfill`);
rb.withFilter()
.kinds(this.#kinds)
.authors(keys)
.until(unixNow())
.since(this.#oldest ?? unixNow() - MaxCacheWindow);
await system.Fetch(rb, async evs => {
await this.bulkSet(evs);
});
}
/**
* Backfill cache based on follows list
*/
async backFillIfMissing(system: SystemInterface, keys: Array<string>) {
if (!this.#oldest) return;
const start = unixNowMs();
const everything = await this.table?.toArray();
if ((everything?.length ?? 0) > 0) {
const allKeys = new Set(everything?.map(a => a.pubkey));
const missingKeys = keys.filter(a => !allKeys.has(a));
await this.backFill(system, missingKeys);
this.log(`Backfilled %d keys in %d ms`, missingKeys.length, (unixNowMs() - start).toLocaleString());
}
}
}

50
packages/app/src/Cache/Notifications.ts
View File

@ -1,50 +0,0 @@
import { unixNow } from "@snort/shared";
import { EventKind, NostrEvent, RequestBuilder, TaggedNostrEvent } from "@snort/system";
import { db, NostrEventForSession } from "@/Db";
import { Day } from "@/Utils/Const";
import { LoginSession } from "@/Utils/Login";
import { RefreshFeedCache, TWithCreated } from "./RefreshFeedCache";
export class NotificationsCache extends RefreshFeedCache<NostrEventForSession> {
#kinds = [EventKind.TextNote, EventKind.Reaction, EventKind.Repost, EventKind.ZapReceipt];
constructor() {
super("notifications", db.notifications);
}
buildSub(session: LoginSession, rb: RequestBuilder) {
if (session.publicKey) {
const newest = this.newest(v => v.tags.some(a => a[0] === "p" && a[1] === session.publicKey));
rb.withFilter()
.kinds(this.#kinds)
.tag("p", [session.publicKey])
.since(newest === 0 ? unixNow() - Day * 30 : newest);
}
}
async onEvent(evs: readonly TaggedNostrEvent[], pubKey: string) {
const filtered = evs.filter(a => this.#kinds.includes(a.kind) && a.tags.some(b => b[0] === "p"));
if (filtered.length > 0) {
await this.bulkSet(
filtered.map(v => ({
...v,
forSession: pubKey,
})),
);
this.emit(
"change",
filtered.map(v => this.key(v)),
);
}
}
key(of: TWithCreated<NostrEvent>): string {
return of.id;
}
takeSnapshot() {
return [...this.cache.values()];
}
}

18
packages/app/src/Cache/index.ts
View File

@ -1,13 +1,30 @@
import { RelayMetricCache, UserProfileCache, UserRelaysCache } from "@snort/system";
import { SnortSystemDb } from "@snort/system-web";
import { WorkerRelayInterface } from "@snort/worker-relay";
import WorkerRelayPath from "@snort/worker-relay/dist/worker?worker&url";
import { ChatCache } from "./ChatCache";
import { EventCacheWorker } from "./EventCacheWorker";
import { GiftWrapCache } from "./GiftWrapCache";
export const Relay = new WorkerRelayInterface(WorkerRelayPath);
export async function initRelayWorker() {
try {
if (await Relay.init()) {
if (await Relay.open()) {
await Relay.migrate();
}
}
} catch (e) {
console.error(e);
}
}
export const SystemDb = new SnortSystemDb();
export const UserCache = new UserProfileCache(SystemDb.users);
export const UserRelays = new UserRelaysCache(SystemDb.userRelays);
export const RelayMetrics = new RelayMetricCache(SystemDb.relayMetrics);
export const EventsCache = new EventCacheWorker(Relay);
export const Chats = new ChatCache();
export const GiftsCache = new GiftWrapCache();
@ -19,6 +36,7 @@ export async function preload(follows?: Array<string>) {
RelayMetrics.preload(),
GiftsCache.preload(),
UserRelays.preload(follows),
EventsCache.preload(),
];
await Promise.all(preloads);
}