Kieran
/
snort
5
14
Fork 14
Code Issues 52 Pull Requests 1 Packages Projects Releases 24 Wiki Activity

move for you feed creation to worker

This commit is contained in:
Martti Malmi 2024-02-05 10:20:03 +02:00
parent 351a249a32
commit 5bc3c10d36
5 changed files with 52 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
packages/app/src/Pages/Root/ForYouTab.tsx
View File

@ -1,16 +1,15 @@
import { EventKind, TaggedNostrEvent } from "@snort/system";
import {EventKind, NostrEvent} from "@snort/system";
import { memo, useEffect, useMemo, useState } from "react";
import { FormattedMessage } from "react-intl";
import { Link } from "react-router-dom";
import {Relay} from "@/Cache";
import { DisplayAs, DisplayAsSelector } from "@/Components/Feed/DisplayAsSelector";
import { TimelineRenderer } from "@/Components/Feed/TimelineRenderer";
import { TaskList } from "@/Components/Tasks/TaskList";
import { getForYouFeed } from "@/Db/getForYouFeed";
import useTimelineFeed, { TimelineFeedOptions, TimelineSubject } from "@/Feed/TimelineFeed";
import useLogin from "@/Hooks/useLogin";
import messages from "@/Pages/messages";
import { System } from "@/system";
const FollowsHint = () => {
const { publicKey: pubKey, follows } = useLogin();
@ -32,14 +31,14 @@ const FollowsHint = () => {
};
let forYouFeed = {
events: [] as TaggedNostrEvent[],
events: [] as NostrEvent[],
created_at: 0,
};
let getForYouFeedPromise: Promise<TaggedNostrEvent[]> | null = null;
let getForYouFeedPromise: Promise<NostrEvent[]> | null = null;
export const ForYouTab = memo(function ForYouTab() {
const [notes, setNotes] = useState<TaggedNostrEvent[]>(forYouFeed.events);
const [notes, setNotes] = useState<NostrEvent[]>(forYouFeed.events);
const { feedDisplayAs } = useLogin();
const displayAsInitial = feedDisplayAs ?? "list";
const [displayAs, setDisplayAs] = useState<DisplayAs>(displayAsInitial);
@ -64,7 +63,7 @@ export const ForYouTab = memo(function ForYouTab() {
const latestFeed = useTimelineFeed(subject, { method: "TIME_RANGE" } as TimelineFeedOptions);
const filteredLatestFeed = useMemo(() => {
// no replies
return latestFeed.main?.filter((ev: TaggedNostrEvent) => !ev.tags.some((tag: string[]) => tag[0] === "e")) ?? [];
return latestFeed.main?.filter((ev: NostrEvent) => !ev.tags.some((tag: string[]) => tag[0] === "e")) ?? [];
}, [latestFeed.main]);
const getFeed = () => {
@ -72,13 +71,13 @@ export const ForYouTab = memo(function ForYouTab() {
return [];
}
if (!getForYouFeedPromise) {
getForYouFeedPromise = getForYouFeed(publicKey);
getForYouFeedPromise = Relay.forYouFeed(publicKey);
}
getForYouFeedPromise!.then(notes => {
getForYouFeedPromise = null;
if (notes.length < 10) {
setTimeout(() => {
getForYouFeed(publicKey);
getForYouFeedPromise = Relay.forYouFeed(publicKey);
}, 1000);
}
forYouFeed = {
@ -86,11 +85,6 @@ export const ForYouTab = memo(function ForYouTab() {
created_at: Date.now(),
};
setNotes(notes);
notes.forEach(note => {
queueMicrotask(() => {
System.HandleEvent(note);
});
});
});
};

72
packages/app/src/Db/getForYouFeed.ts → packages/worker-relay/src/forYouFeed.ts
View File

@ -1,49 +1,48 @@
import { NostrEvent, parseZap } from "@snort/system";
import debug from "debug";
import { Relay } from "@/Cache";
import {RelayHandler} from "./types";
const log = debug("getForYouFeed");
export async function getForYouFeed(pubkey: string): Promise<NostrEvent[]> {
export async function getForYouFeed(relay: RelayHandler, pubkey: string): Promise<NostrEvent[]> {
console.time("For You feed generation time");
log("pubkey", pubkey);
// Get events reacted to by me
const myReactedEventIds = await getMyReactedEvents(pubkey);
const myReactedEventIds = await getMyReactedEvents(relay, pubkey);
log("my reacted events", myReactedEventIds);
const myReactedAuthors = await getMyReactedAuthors(myReactedEventIds, pubkey);
const myReactedAuthors = await getMyReactedAuthors(relay, myReactedEventIds, pubkey);
log("my reacted authors", myReactedAuthors);
// Get others who reacted to the same events as me
const othersWhoReacted = await getOthersWhoReacted(myReactedEventIds, pubkey);
const othersWhoReacted = await getOthersWhoReacted(relay, myReactedEventIds, pubkey);
// this tends to be small when the user has just logged in, we should maybe subscribe for more from relays
log("others who reacted", othersWhoReacted);
// Get event ids reacted to by those others
const reactedByOthers = await getEventIdsReactedByOthers(othersWhoReacted, myReactedEventIds, pubkey);
const reactedByOthers = await getEventIdsReactedByOthers(relay, othersWhoReacted, myReactedEventIds, pubkey);
log("reacted by others", reactedByOthers);
// Get full events in sorted order
const feed = await getFeedEvents(reactedByOthers, myReactedAuthors);
const feed = await getFeedEvents(relay, reactedByOthers, myReactedAuthors);
log("feed.length", feed.length);
console.timeEnd("For You feed generation time");
return feed;
}
async function getMyReactedAuthors(myReactedEventIds: Set<string>, myPubkey: string) {
async function getMyReactedAuthors(relay: RelayHandler, myReactedEventIds: Set<string>, myPubkey: string) {
const myReactedAuthors = new Map<string, number>();
const myReactions = await Relay.query([
"REQ",
const myReactions = relay.req(
"getMyReactedAuthors",
{
"#e": Array.from(myReactedEventIds),
},
]);
) as NostrEvent[];
myReactions.forEach(reaction => {
if (reaction.pubkey !== myPubkey) {
@ -54,17 +53,16 @@ async function getMyReactedAuthors(myReactedEventIds: Set<string>, myPubkey: str
return myReactedAuthors;
}
async function getMyReactedEvents(pubkey: string) {
async function getMyReactedEvents(relay: RelayHandler, pubkey: string) {
const myReactedEventIds = new Set<string>();
const myEvents = await Relay.query([
"REQ",
const myEvents = relay.req(
"getMyReactedEventIds",
{
authors: [pubkey],
kinds: [1, 6, 7, 9735],
},
]);
) as NostrEvent[];
myEvents.forEach(ev => {
const targetEventId = ev.kind === 9735 ? parseZap(ev).event?.id : ev.tags.find(tag => tag[0] === "e")?.[1];
if (targetEventId) {
@ -75,16 +73,15 @@ async function getMyReactedEvents(pubkey: string) {
return myReactedEventIds;
}
async function getOthersWhoReacted(myReactedEventIds: Set<string>, myPubkey: string) {
async function getOthersWhoReacted(relay: RelayHandler, myReactedEventIds: Set<string>, myPubkey: string) {
const othersWhoReacted = new Map<string, number>();
const otherReactions = await Relay.query([
"REQ",
const otherReactions = relay.req(
"getOthersWhoReacted",
{
"#e": Array.from(myReactedEventIds),
},
]);
) as NostrEvent[];
otherReactions.forEach(reaction => {
if (reaction.pubkey !== myPubkey) {
@ -96,20 +93,20 @@ async function getOthersWhoReacted(myReactedEventIds: Set<string>, myPubkey: str
}
async function getEventIdsReactedByOthers(
relay: RelayHandler,
othersWhoReacted: Map<string, number>,
myReactedEvents: Set<string>,
myPub: string,
) {
const eventIdsReactedByOthers = new Map<string, number>();
const events = await Relay.query([
"REQ",
const events = relay.req(
"getEventIdsReactedByOthers",
{
authors: [...othersWhoReacted.keys()],
kinds: [1, 6, 7, 9735],
},
]);
) as NostrEvent[];
events.forEach(event => {
if (event.pubkey === myPub || myReactedEvents.has(event.id)) {
@ -127,16 +124,12 @@ async function getEventIdsReactedByOthers(
return eventIdsReactedByOthers;
}
async function getFeedEvents(reactedToIds: Map<string, number>, reactedToAuthors: Map<string, number>) {
const events = await Relay.query([
"REQ",
"getFeedEvents",
{
ids: Array.from(reactedToIds.keys()),
kinds: [1],
since: Math.floor(Date.now() / 1000) - 60 * 60 * 24 * 7,
},
]);
async function getFeedEvents(relay: RelayHandler, reactedToIds: Map<string, number>, reactedToAuthors: Map<string, number>) {
const events = relay.sql(
`select json from events where id in (${Array.from(reactedToIds.keys()).map(() => "?").join(", ")}) and kind = 1 order by seen_at ASC, created DESC limit 1000`,
Array.from(reactedToIds.keys()),
).map(row => JSON.parse(row[0] as string) as NostrEvent);
const seen = new Set<string>(events.map(ev => ev.id));
log("reactedToAuthors", reactedToAuthors);
@ -145,16 +138,11 @@ async function getFeedEvents(reactedToIds: Map<string, number>, reactedToAuthors
.sort((a, b) => reactedToAuthors.get(b)! - reactedToAuthors.get(a)!)
.slice(20);
const eventsByFavoriteAuthors = await Relay.query([
"REQ",
"getFeedEvents",
{
authors: favoriteAuthors,
kinds: [1],
since: Math.floor(Date.now() / 1000) - 60 * 60 * 24,
limit: 100,
},
]);
const eventsByFavoriteAuthors = relay.sql(
`select json from events where pubkey in (${favoriteAuthors.map(() => "?").join(", ")}) and kind = 1 order by seen_at ASC, created DESC limit 100`,
favoriteAuthors,
).map(row => JSON.parse(row[0] as string) as NostrEvent);
eventsByFavoriteAuthors.forEach(ev => {
if (!seen.has(ev.id)) {

4
packages/worker-relay/src/interface.ts
View File

@ -45,6 +45,10 @@ export class WorkerRelayInterface {
return await this.#workerRpc<void, Uint8Array>("dumpDb");
}
async forYouFeed(pubkey: string) {
return await this.#workerRpc<string, Array<NostrEvent>>("forYouFeed", pubkey);
}
#workerRpc<T, R>(cmd: WorkerMessageCommand, args?: T) {
const id = uuid();
const msg = {

3
packages/worker-relay/src/types.ts
View File

@ -9,7 +9,8 @@ export type WorkerMessageCommand =
| "summary"
| "close"
| "dumpDb"
| "emit-event";
| "emit-event"
| "forYouFeed";
export interface WorkerMessage<T> {
id: string;

8
packages/worker-relay/src/worker.ts
View File

@ -4,6 +4,7 @@ import { InMemoryRelay } from "./memory-relay";
import { WorkQueueItem, barrierQueue, processWorkQueue } from "./queue";
import { SqliteRelay } from "./sqlite-relay";
import { NostrEvent, RelayHandler, ReqCommand, ReqFilter, WorkerMessage, unixNowMs } from "./types";
import {getForYouFeed} from "./forYouFeed";
let relay: RelayHandler | undefined;
@ -130,6 +131,13 @@ globalThis.onmessage = async ev => {
});
break;
}
case "forYouFeed": {
await barrierQueue(cmdQueue, async () => {
const res = await getForYouFeed(relay!, msg.args as string);
reply(msg.id, res);
});
break;
}
default: {
reply(msg.id, { error: "Unknown command" });
break;