Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: @snort/system CacheRelay

Browse Source
This commit is contained in:
Kieran
2024-01-23 15:35:28 +00:00
parent d6c578fafc
commit 5cea096067
29 changed files with 296 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/app/src/Cache/EventCacheWorker.ts
View File

@ -14,9 +14,14 @@ export class EventCacheWorker extends EventEmitter<CacheEvents> implements Cache
}
async preload() {
const ids = await this.#relay.sql("select id from events", []);
this.#keys = new Set<string>(ids.map(a => a[0] as string));
return Promise.resolve();
const ids = await this.#relay.query([
"REQ",
"preload-event-cache",
{
ids_only: true,
},
]);
this.#keys = new Set<string>(ids as unknown as Array<string>);
}
keysOnTable(): string[] {
@ -43,18 +48,17 @@ export class EventCacheWorker extends EventEmitter<CacheEvents> implements Cache
}
async bulkGet(keys: string[]): Promise<NostrEvent[]> {
const results = await this.#relay.req({
id: "EventCacheWorker.bulkGet",
filters: [
{
ids: keys,
},
],
});
for (const ev of results.result) {
const results = await this.#relay.query([
"REQ",
"EventCacheWorker.bulkGet",
{
ids: keys,
},
]);
for (const ev of results) {
this.#cache.set(ev.id, ev);
}
return results.result;
return results;
}
async set(obj: NostrEvent): Promise<void> {

36
packages/app/src/Cache/ProfileWorkeCache.ts
View File

@ -1,12 +1,14 @@
import { CachedTable, CacheEvents, removeUndefined } from "@snort/shared";
import { CachedTable, CacheEvents, removeUndefined, unixNowMs, unwrap } from "@snort/shared";
import { CachedMetadata, mapEventToProfile, NostrEvent } from "@snort/system";
import { WorkerRelayInterface } from "@snort/worker-relay";
import debug from "debug";
import EventEmitter from "eventemitter3";
export class ProfileCacheRelayWorker extends EventEmitter<CacheEvents> implements CachedTable<CachedMetadata> {
#relay: WorkerRelayInterface;
#keys = new Set<string>();
#cache = new Map<string, CachedMetadata>();
#log = debug("ProfileCacheRelayWorker");
constructor(relay: WorkerRelayInterface) {
super();
@ -14,8 +16,17 @@ export class ProfileCacheRelayWorker extends EventEmitter<CacheEvents> implement
}
async preload() {
const ids = await this.#relay.sql("select distinct(pubkey) from events where kind = ?", [0]);
this.#keys = new Set<string>(ids.map(a => a[0] as string));
const start = unixNowMs();
const profiles = await this.#relay.query([
"REQ",
"profiles-preload",
{
kinds: [0],
},
]);
this.#cache = new Map<string, CachedMetadata>(profiles.map(a => [a.pubkey, unwrap(mapEventToProfile(a))]));
this.#keys = new Set<string>(this.#cache.keys());
this.#log(`Loaded %d/%d in %d ms`, this.#cache.size, this.#keys.size, (unixNowMs() - start).toLocaleString());
}
keysOnTable(): string[] {
@ -50,16 +61,15 @@ export class ProfileCacheRelayWorker extends EventEmitter<CacheEvents> implement
async bulkGet(keys: string[]) {
if (keys.length === 0) return [];
const results = await this.#relay.req({
id: "ProfileCacheRelayWorker.bulkGet",
filters: [
{
authors: keys,
kinds: [0],
},
],
});
const mapped = removeUndefined(results.result.map(a => mapEventToProfile(a)));
const results = await this.#relay.query([
"REQ",
"ProfileCacheRelayWorker.bulkGet",
{
authors: keys,
kinds: [0],
},
]);
const mapped = removeUndefined(results.map(a => mapEventToProfile(a)));
for (const pf of mapped) {
this.#cache.set(this.key(pf), pf);
}

5
packages/app/src/Components/Event/Note/NoteFooter/NoteFooter.tsx
View File

@ -1,5 +1,5 @@
import { NostrLink, TaggedNostrEvent } from "@snort/system";
import { useEventReactions } from "@snort/system-react";
import { useEventReactions, useReactions } from "@snort/system-react";
import React, { useMemo, useState } from "react";
import { FooterZapButton } from "@/Components/Event/Note/NoteFooter/FooterZapButton";
@ -8,7 +8,6 @@ import { PowIcon } from "@/Components/Event/Note/NoteFooter/PowIcon";
import { ReplyButton } from "@/Components/Event/Note/NoteFooter/ReplyButton";
import { RepostButton } from "@/Components/Event/Note/NoteFooter/RepostButton";
import ReactionsModal from "@/Components/Event/Note/ReactionsModal";
import { useReactionsView } from "@/Feed/WorkerRelayView";
import useLogin from "@/Hooks/useLogin";
export interface NoteFooterProps {
@ -22,7 +21,7 @@ export default function NoteFooter(props: NoteFooterProps) {
const ids = useMemo(() => [link], [link]);
const [showReactions, setShowReactions] = useState(false);
const related = useReactionsView(ids, false);
const related = useReactions("reactions", ids, undefined, false);
const { reactions, zaps, reposts } = useEventReactions(link, related);
const { positive } = reactions;

21
packages/app/src/Components/Feed/LocalSearch.tsx
View File

@ -21,19 +21,18 @@ export function LocalSearch({ term, kind }: { term: string; kind: EventKind }) {
useEffect(() => {
setFrag(undefined);
if (term) {
Relay.req({
id: "local-search",
filters: [
{
kinds: [kind],
limit: 100,
search: term,
},
],
}).then(res => {
Relay.query([
"REQ",
"local-search",
{
kinds: [kind],
limit: 100,
search: term,
},
]).then(res => {
setFrag({
refTime: 0,
events: res.result as Array<TaggedNostrEvent>,
events: res as Array<TaggedNostrEvent>,
});
});
}

22
packages/app/src/Db/FuzzySearch.ts
View File

@ -43,19 +43,17 @@ export const addEventToFuzzySearch = (ev: NostrEvent) => {
};
export const addCachedMetadataToFuzzySearch = (profile: CachedMetadata) => {
queueMicrotask(() => {
const existing = profileTimestamps.get(profile.pubkey);
if (existing) {
if (existing > profile.created) {
return;
}
fuzzySearch.remove(doc => doc.pubkey === profile.pubkey);
const existing = profileTimestamps.get(profile.pubkey);
if (existing) {
if (existing > profile.created) {
return;
}
profileTimestamps.set(profile.pubkey, profile.created);
if (profile.pubkey && (profile.name || profile.display_name || profile.nip05)) {
fuzzySearch.add(profile);
}
});
fuzzySearch.remove(doc => doc.pubkey === profile.pubkey);
}
profileTimestamps.set(profile.pubkey, profile.created);
if (profile.pubkey && (profile.name || profile.display_name || profile.nip05)) {
fuzzySearch.add(profile);
}
};
export default fuzzySearch;

4
packages/app/src/Feed/LoginFeed.ts
View File

@ -23,9 +23,6 @@ import {
SnortAppData,
} from "@/Utils/Login";
import { SubscriptionEvent } from "@/Utils/Subscription";
import { useFollowsContactListView } from "./WorkerRelayView";
/**
* Managed loading data for the current logged in user
*/
@ -34,7 +31,6 @@ export default function useLoginFeed() {
const { publicKey: pubKey, follows } = login;
const { publisher, system } = useEventPublisher();
useFollowsContactListView();
useEffect(() => {
system.checkSigs = login.appData.item.preferences.checkSigs;
}, [login]);

7
packages/app/src/Feed/TimelineFeed.ts
View File

@ -86,9 +86,10 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
const sub = useMemo(() => {
const rb = createBuilder();
console.debug(rb?.builder.id, options);
if (rb) {
if (options.method === "LIMIT_UNTIL") {
rb.filter.until(until).limit(100);
rb.filter.until(until).limit(50);
} else {
rb.filter.since(since).until(until);
if (since === undefined) {
@ -112,8 +113,8 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
.limit(1)
.since(now);
}
return rb.builder;
}
return rb?.builder ?? null;
}, [until, since, options.method, pref, createBuilder]);
const mainQuery = useRequestBuilderAdvanced(sub);
@ -135,8 +136,8 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
});
rb.builder.id = `${rb.builder.id}:latest`;
rb.filter.limit(1).since(now);
return rb.builder;
}
return rb?.builder ?? null;
}, [pref.autoShowLatest, createBuilder]);
const latestQuery = useRequestBuilderAdvanced(subRealtime);

205
packages/app/src/Feed/WorkerRelayView.ts
View File

@ -1,200 +1,39 @@
import { unixNow } from "@snort/shared";
import { EventKind, NostrEvent, NostrLink, ReqFilter, RequestBuilder, TaggedNostrEvent } from "@snort/system";
import { SnortContext, useRequestBuilder } from "@snort/system-react";
import { useCallback, useContext, useEffect, useMemo, useState } from "react";
import { LRUCache } from "typescript-lru-cache";
import { EventKind, RequestBuilder } from "@snort/system";
import { useRequestBuilder } from "@snort/system-react";
import { useMemo } from "react";
import { Relay } from "@/Cache";
//import { LRUCache } from "typescript-lru-cache";
import useLogin from "@/Hooks/useLogin";
import { Day } from "@/Utils/Const";
const cache = new LRUCache<string, NostrEvent[]>({ maxSize: 100 });
export function useWorkerRelayView(id: string, filters: Array<ReqFilter>, leaveOpen?: boolean, maxWindow?: number) {
const cacheKey = useMemo(() => JSON.stringify(filters), [filters]);
const [events, setEvents] = useState<Array<NostrEvent>>(cache.get(cacheKey) ?? []);
const [rb, setRb] = useState<RequestBuilder>();
const system = useContext(SnortContext);
const cacheAndSetEvents = useCallback(
(evs: Array<NostrEvent>) => {
cache.set(cacheKey, evs);
setEvents(evs);
},
[cacheKey],
);
useEffect(() => {
if (rb) {
const q = system.Query(rb);
q.uncancel();
return () => q.cancel();
}
}, [rb, system]);
useEffect(() => {
setRb(undefined);
Relay.req({
id: `${id}+latest`,
filters: filters.map(f => ({
...f,
until: undefined,
since: undefined,
limit: 1,
})),
}).then(latest => {
const rb = new RequestBuilder(id);
rb.withOptions({ fillStore: false });
filters
.map((f, i) => {
const since = latest.result?.at(i)?.created_at;
return {
...f,
limit: undefined,
until: undefined,
since: since ? since + 1 : maxWindow ? unixNow() - maxWindow : f.since,
};
})
.forEach(f => rb.withBareFilter(f));
setRb(rb);
});
Relay.req({ id, filters, leaveOpen }).then(res => {
cacheAndSetEvents(res.result);
if (res.port) {
res.port.addEventListener("message", ev => {
const evs = ev.data as Array<NostrEvent>;
if (evs.length > 0) {
cacheAndSetEvents([...events, ...evs]);
}
});
res.port.start();
}
});
return () => {
Relay.close(id);
};
}, [id, filters, maxWindow]);
return events as Array<TaggedNostrEvent>;
}
export function useWorkerRelayViewCount(id: string, filters: Array<ReqFilter>, maxWindow?: number) {
const [count, setCount] = useState(0);
const [rb, setRb] = useState<RequestBuilder>();
useRequestBuilder(rb);
useEffect(() => {
Relay.req({
id: `${id}+latest`,
filters: filters.map(f => ({
...f,
until: undefined,
since: undefined,
limit: 1,
})),
}).then(latest => {
const rb = new RequestBuilder(id);
filters
.map((f, i) => ({
...f,
limit: undefined,
until: undefined,
since: latest.result?.at(i)?.created_at ?? (maxWindow ? unixNow() - maxWindow : undefined),
}))
.forEach(f => rb.withBareFilter(f));
setRb(rb);
});
Relay.count({ id, filters }).then(setCount);
}, [id, filters, maxWindow]);
return count;
}
//const cache = new LRUCache<string, NostrEvent[]>({ maxSize: 100 });
export function useFollowsTimelineView(limit = 20) {
const follows = useLogin(s => s.follows.item);
const kinds = [EventKind.TextNote, EventKind.Repost, EventKind.Polls];
const filter = useMemo(() => {
return [
{
authors: follows,
kinds,
limit,
},
];
const req = useMemo(() => {
const rb = new RequestBuilder("follows-timeline");
rb.withOptions({
leaveOpen: true,
});
rb.withFilter().kinds(kinds).authors(follows).limit(limit);
return rb;
}, [follows, limit]);
return useWorkerRelayView("follows-timeline", filter, true, Day * 7);
return useRequestBuilder(req);
}
export function useNotificationsView() {
const publicKey = useLogin(s => s.publicKey);
const kinds = [EventKind.TextNote, EventKind.Reaction, EventKind.Repost, EventKind.ZapReceipt];
const req = useMemo(() => {
return [
{
"#p": [publicKey ?? ""],
kinds,
since: unixNow() - Day * 7,
},
];
if (publicKey) {
const rb = new RequestBuilder("notifications");
rb.withOptions({
leaveOpen: true,
});
rb.withFilter().kinds(kinds).tag("p", [publicKey]).limit(1000);
return rb;
}
}, [publicKey]);
return useWorkerRelayView("notifications", req, true, Day * 30);
}
export function useReactionsView(ids: Array<NostrLink>, leaveOpen = true) {
const req = useMemo(() => {
const rb = new RequestBuilder("reactions");
rb.withOptions({ leaveOpen });
const grouped = ids.reduce(
(acc, v) => {
acc[v.type] ??= [];
acc[v.type].push(v);
return acc;
},
{} as Record<string, Array<NostrLink>>,
);
for (const [, v] of Object.entries(grouped)) {
rb.withFilter().kinds([EventKind.Reaction, EventKind.Repost, EventKind.ZapReceipt]).replyToLink(v);
}
return rb.buildRaw();
}, [ids]);
return useWorkerRelayView("reactions", req, leaveOpen, undefined);
}
export function useReactionsViewCount(ids: Array<NostrLink>, leaveOpen = true) {
const req = useMemo(() => {
const rb = new RequestBuilder("reactions");
rb.withOptions({ leaveOpen });
const grouped = ids.reduce(
(acc, v) => {
acc[v.type] ??= [];
acc[v.type].push(v);
return acc;
},
{} as Record<string, Array<NostrLink>>,
);
for (const [, v] of Object.entries(grouped)) {
rb.withFilter().kinds([EventKind.Reaction, EventKind.Repost, EventKind.ZapReceipt]).replyToLink(v);
}
return rb.buildRaw();
}, [ids]);
return useWorkerRelayViewCount("reactions", req, undefined);
}
export function useFollowsContactListView() {
const follows = useLogin(s => s.follows.item);
const kinds = [EventKind.ContactList, EventKind.Relays];
const filter = useMemo(() => {
return [
{
authors: follows,
kinds,
},
];
}, [follows]);
return useWorkerRelayView("follows-contacts-relays", filter, undefined, undefined);
return useRequestBuilder(req);
}

4
packages/app/src/Hooks/useLists.tsx
View File

@ -9,7 +9,9 @@ export function useLinkList(id: string, fn: (rb: RequestBuilder) => void) {
const sub = useMemo(() => {
const rb = new RequestBuilder(id);
fn(rb);
return rb;
if (rb.numFilters > 0) {
return rb;
}
}, [id, fn]);
const listStore = useRequestBuilder(sub);

2
packages/app/src/Pages/Profile/ProfilePage.tsx
View File

@ -163,7 +163,7 @@ export default function ProfilePage({ id: propId, state }: ProfilePageProps) {
)}
<div className="profile-wrapper w-max">
<AvatarSection id={id} loginPubKey={loginPubKey} user={user} readonly={readonly} lnurl={lnurl} />
<ProfileDetails user={user} loginPubKey={loginPubKey} id={id} aboutText={aboutText} lnurl={lnurl} />
<ProfileDetails user={user} loginPubKey={loginPubKey} id={id} aboutText={aboutText} lnurl={lnurl} showLnQr={true} />
</div>
</div>
<div className="main-content">

6
packages/app/src/Pages/Profile/ProfileTabComponents.tsx
View File

@ -11,6 +11,7 @@ import FollowsList from "@/Components/User/FollowListBase";
import useFollowersFeed from "@/Feed/FollowersFeed";
import useFollowsFeed from "@/Feed/FollowsFeed";
import useRelaysFeed from "@/Feed/RelaysFeed";
import { TimelineSubject } from "@/Feed/TimelineFeed";
import useZapsFeed from "@/Feed/ZapsFeed";
import { useBookmarkList, usePinList } from "@/Hooks/useLists";
import messages from "@/Pages/messages";
@ -52,7 +53,6 @@ export function BookMarksTab({ id }: { id: HexKey }) {
}
export function ProfileNotesTab({ id, relays, isMe }: { id: HexKey; relays?: Array<string>; isMe: boolean }) {
console.count("ProfileNotesTab");
const pinned = usePinList(id);
const options = useMemo(() => ({ showTime: false, showPinned: true, canUnpin: isMe }), [isMe]);
const subject = useMemo(
@ -61,7 +61,7 @@ export function ProfileNotesTab({ id, relays, isMe }: { id: HexKey; relays?: Arr
items: [id],
discriminator: id.slice(0, 12),
relay: relays,
}),
} as TimelineSubject),
[id, relays],
);
return (
@ -76,7 +76,7 @@ export function ProfileNotesTab({ id, relays, isMe }: { id: HexKey; relays?: Arr
subject={subject}
postsOnly={false}
method={"LIMIT_UNTIL"}
loadMore={false}
loadMore={true}
ignoreModeration={true}
window={60 * 60 * 6}
/>

31
packages/app/src/index.tsx
View File

@ -8,11 +8,11 @@ import { StrictMode } from "react";
import * as ReactDOM from "react-dom/client";
import { createBrowserRouter, RouteObject, RouterProvider } from "react-router-dom";
import { initRelayWorker, preload, Relay } from "@/Cache";
import { initRelayWorker, preload, Relay, UserCache } from "@/Cache";
import { ThreadRoute } from "@/Components/Event/Thread";
import { IntlProvider } from "@/Components/IntlProvider/IntlProvider";
import { db } from "@/Db";
import { addEventToFuzzySearch } from "@/Db/FuzzySearch";
import { addCachedMetadataToFuzzySearch } from "@/Db/FuzzySearch";
import { updateRelayConnections } from "@/Hooks/useLoginRelays";
import { AboutPage } from "@/Pages/About";
import { SnortDeckLayout } from "@/Pages/DeckLayout";
@ -54,39 +54,34 @@ async function initSite() {
const login = LoginStore.takeSnapshot();
db.ready = await db.isAvailable();
if (db.ready) {
await preload(login.follows.item);
preload(login.follows.item);
}
updateRelayConnections(System, login.relays.item).catch(console.error);
try {
if ("registerProtocolHandler" in window.navigator) {
window.navigator.registerProtocolHandler("web+nostr", `${window.location.protocol}//${window.location.host}/%s`);
console.info("Registered protocol handler for 'web+nostr'");
}
} catch (e) {
console.error("Failed to register protocol handler", e);
}
setupWebLNWalletConfig(Wallets);
Relay.sql("select json from events where kind = ?", [3]).then(res => {
for (const [json] of res) {
Relay.query(["REQ", "preload-social-graph", {
kinds: [3]
}]).then(res => {
for (const ev of res) {
try {
socialGraphInstance.handleEvent(JSON.parse(json as string));
socialGraphInstance.handleEvent(ev);
} catch (e) {
console.error("Failed to handle contact list event from sql db", e);
}
}
});
Relay.sql("select json from events where kind = ?", [0]).then(res => {
for (const [json] of res) {
queueMicrotask(() => {
for (const ev of UserCache.snapshot()) {
try {
addEventToFuzzySearch(JSON.parse(json as string));
addCachedMetadataToFuzzySearch(ev);
} catch (e) {
console.error("Failed to handle metadata event from sql db", e);
}
}
});
return null;
}

10
packages/app/src/system.ts
View File

@ -1,5 +1,5 @@
import { removeUndefined, throwIfOffline } from "@snort/shared";
import { mapEventToProfile, NostrEvent, NostrSystem, ProfileLoaderService, socialGraphInstance } from "@snort/system";
import { mapEventToProfile, NostrEvent, NostrSystem, socialGraphInstance } from "@snort/system";
import inMemoryDB from "@snort/system/src/InMemoryDB";
import { EventsCache, Relay, RelayMetrics, SystemDb, UserCache, UserRelays } from "@/Cache";
@ -15,6 +15,7 @@ export const System = new NostrSystem({
eventsCache: EventsCache,
profileCache: UserCache,
relayMetrics: RelayMetrics,
cacheRelay: Relay,
optimizer: hasWasm ? WasmOptimizer : undefined,
db: SystemDb,
});
@ -58,9 +59,4 @@ export async function fetchProfile(key: string) {
} catch (e) {
console.error(e);
}
}
/**
* Singleton user profile loader
*/
export const ProfileLoader = new ProfileLoaderService(System, UserCache);
}