Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #184 from v0l/feed-cache

Browse Source 
feed cache
This commit is contained in:
Kieran
2023-02-02 13:55:31 +00:00
committed by GitHub
parent 69a4dfffb7 162ab55445
commit 1770e09159
21 changed files with 372 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/Feed/EventPublisher.ts
View File

@ -78,8 +78,8 @@ export default function useEventPublisher() {
}
return {
nip42Auth: async (challenge: string, relay:string) => {
if(pubKey) {
nip42Auth: async (challenge: string, relay: string) => {
if (pubKey) {
const ev = NEvent.ForPubKey(pubKey);
ev.Kind = EventKind.Auth;
ev.Content = "";
@ -112,17 +112,17 @@ export default function useEventPublisher() {
ev.Kind = EventKind.Lists;
ev.Tags.push(new Tag(["d", Lists.Muted], ev.Tags.length))
keys.forEach(p => {
ev.Tags.push(new Tag(["p", p], ev.Tags.length))
ev.Tags.push(new Tag(["p", p], ev.Tags.length))
})
let content = ""
if (priv.length > 0) {
const ps = priv.map(p => ["p", p])
const plaintext = JSON.stringify(ps)
if (hasNip07 && !privKey) {
content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
} else if (privKey) {
content = await ev.EncryptData(plaintext, pubKey, privKey)
}
const ps = priv.map(p => ["p", p])
const plaintext = JSON.stringify(ps)
if (hasNip07 && !privKey) {
content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
} else if (privKey) {
content = await ev.EncryptData(plaintext, pubKey, privKey)
}
}
ev.Content = content;
return await signEvent(ev);

2
src/Feed/FollowersFeed.ts
View File

@ -7,7 +7,7 @@ import useSubscription from "Feed/Subscription";
export default function useFollowersFeed(pubkey: HexKey) {
const sub = useMemo(() => {
let x = new Subscriptions();
x.Id = "followers";
x.Id = `followers:${pubkey.slice(0, 12)}`;
x.Kinds = new Set([EventKind.ContactList]);
x.PTags = new Set([pubkey]);

2
src/Feed/FollowsFeed.ts
View File

@ -7,7 +7,7 @@ import useSubscription, { NoteStore } from "Feed/Subscription";
export default function useFollowsFeed(pubkey: HexKey) {
const sub = useMemo(() => {
let x = new Subscriptions();
x.Id = "follows";
x.Id = `follows:${pubkey.slice(0, 12)}`;
x.Kinds = new Set([EventKind.ContactList]);
x.Authors = new Set([pubkey]);

236
src/Feed/LoginFeed.ts
View File

@ -8,151 +8,149 @@ import Event from "Nostr/Event";
import { Subscriptions } from "Nostr/Subscriptions";
import { addDirectMessage, setFollows, setRelays, setMuted, setBlocked, sendNotification } from "State/Login";
import { RootState } from "State/Store";
import { mapEventToProfile, MetadataCache } from "State/Users";
import { getDb } from "State/Users/Db";
import { mapEventToProfile, MetadataCache } from "State/Users";
import { useDb } from "State/Users/Db";
import useSubscription from "Feed/Subscription";
import { getDisplayName } from "Element/ProfileImage";
import { barierNip07 } from "Feed/EventPublisher";
import { getMutedKeys, getNewest } from "Feed/MuteList";
import { MentionRegex } from "Const";
import useModeration from "Hooks/useModeration";
/**
* Managed loading data for the current logged in user
*/
export default function useLoginFeed() {
const dispatch = useDispatch();
const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login);
const { isMuted } = useModeration();
const dispatch = useDispatch();
const { publicKey: pubKey, privateKey: privKey } = useSelector((s: RootState) => s.login);
const { isMuted } = useModeration();
const db = useDb();
const subMetadata = useMemo(() => {
if (!pubKey) return null;
const subMetadata = useMemo(() => {
if (!pubKey) return null;
let sub = new Subscriptions();
sub.Id = `login:meta`;
sub.Authors = new Set([pubKey]);
sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]);
sub.Limit = 2
let sub = new Subscriptions();
sub.Id = `login:meta`;
sub.Authors = new Set([pubKey]);
sub.Kinds = new Set([EventKind.ContactList, EventKind.SetMetadata]);
sub.Limit = 2
return sub;
}, [pubKey]);
return sub;
}, [pubKey]);
const subNotification = useMemo(() => {
if (!pubKey) return null;
const subNotification = useMemo(() => {
if (!pubKey) return null;
let sub = new Subscriptions();
sub.Id = "login:notifications";
sub.Kinds = new Set([EventKind.TextNote]);
sub.PTags = new Set([pubKey]);
sub.Limit = 1;
return sub;
}, [pubKey]);
let sub = new Subscriptions();
sub.Id = "login:notifications";
sub.Kinds = new Set([EventKind.TextNote]);
sub.PTags = new Set([pubKey]);
sub.Limit = 1;
return sub;
}, [pubKey]);
const subMuted = useMemo(() => {
if (!pubKey) return null;
const subMuted = useMemo(() => {
if (!pubKey) return null;
let sub = new Subscriptions();
sub.Id = "login:muted";
sub.Kinds = new Set([EventKind.Lists]);
sub.Authors = new Set([pubKey]);
sub.DTags = new Set([Lists.Muted]);
sub.Limit = 1;
let sub = new Subscriptions();
sub.Id = "login:muted";
sub.Kinds = new Set([EventKind.Lists]);
sub.Authors = new Set([pubKey]);
sub.DTags = new Set([Lists.Muted]);
sub.Limit = 1;
return sub;
}, [pubKey]);
return sub;
}, [pubKey]);
const subDms = useMemo(() => {
if (!pubKey) return null;
const subDms = useMemo(() => {
if (!pubKey) return null;
let dms = new Subscriptions();
dms.Id = "login:dms";
dms.Kinds = new Set([EventKind.DirectMessage]);
dms.PTags = new Set([pubKey]);
let dms = new Subscriptions();
dms.Id = "login:dms";
dms.Kinds = new Set([EventKind.DirectMessage]);
dms.PTags = new Set([pubKey]);
let dmsFromME = new Subscriptions();
dmsFromME.Authors = new Set([pubKey]);
dmsFromME.Kinds = new Set([EventKind.DirectMessage]);
dms.AddSubscription(dmsFromME);
let dmsFromME = new Subscriptions();
dmsFromME.Authors = new Set([pubKey]);
dmsFromME.Kinds = new Set([EventKind.DirectMessage]);
dms.AddSubscription(dmsFromME);
return dms;
}, [pubKey]);
return dms;
}, [pubKey]);
const metadataFeed = useSubscription(subMetadata, { leaveOpen: true });
const notificationFeed = useSubscription(subNotification, { leaveOpen: true });
const dmsFeed = useSubscription(subDms, { leaveOpen: true });
const mutedFeed = useSubscription(subMuted, { leaveOpen: true });
const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true });
const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true });
const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true });
const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true });
useEffect(() => {
let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);
let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata);
let profiles = metadata.map(a => mapEventToProfile(a))
.filter(a => a !== undefined)
.map(a => a!);
useEffect(() => {
let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);
let metadata = metadataFeed.store.notes.filter(a => a.kind === EventKind.SetMetadata);
let profiles = metadata.map(a => mapEventToProfile(a))
.filter(a => a !== undefined)
.map(a => a!);
for (let cl of contactList) {
if (cl.content !== "" && cl.content !== "{}") {
let relays = JSON.parse(cl.content);
dispatch(setRelays({ relays, createdAt: cl.created_at }));
}
let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]);
dispatch(setFollows({ keys: pTags, createdAt: cl.created_at }));
}
(async () => {
let maxProfile = profiles.reduce((acc, v) => {
if (v.created > acc.created) {
acc.profile = v;
acc.created = v.created;
}
return acc;
}, { created: 0, profile: null as MetadataCache | null });
if (maxProfile.profile) {
const db = getDb()
let existing = await db.find(maxProfile.profile.pubkey);
if ((existing?.created ?? 0) < maxProfile.created) {
await db.put(maxProfile.profile);
}
}
})().catch(console.warn);
}, [dispatch, metadataFeed.store]);
useEffect(() => {
const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey))
replies.forEach(nx => {
makeNotification(nx).then(notification => {
if (notification) {
// @ts-ignore
dispatch(sendNotification(notification))
}
})
})
}, [dispatch, notificationFeed.store]);
useEffect(() => {
const muted = getMutedKeys(mutedFeed.store.notes)
dispatch(setMuted(muted))
const newest = getNewest(mutedFeed.store.notes)
if (newest && newest.content.length > 0 && pubKey) {
decryptBlocked(newest, pubKey, privKey).then((plaintext) => {
try {
const blocked = JSON.parse(plaintext)
const keys = blocked.filter((p:any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1])
dispatch(setBlocked({
keys,
createdAt: newest.created_at,
}))
} catch(error) {
console.debug("Couldn't parse JSON")
}
}).catch((error) => console.warn(error))
for (let cl of contactList) {
if (cl.content !== "" && cl.content !== "{}") {
let relays = JSON.parse(cl.content);
dispatch(setRelays({ relays, createdAt: cl.created_at }));
}
}, [dispatch, mutedFeed.store])
let pTags = cl.tags.filter(a => a[0] === "p").map(a => a[1]);
dispatch(setFollows({ keys: pTags, createdAt: cl.created_at }));
}
useEffect(() => {
let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage);
dispatch(addDirectMessage(dms));
}, [dispatch, dmsFeed.store]);
(async () => {
let maxProfile = profiles.reduce((acc, v) => {
if (v.created > acc.created) {
acc.profile = v;
acc.created = v.created;
}
return acc;
}, { created: 0, profile: null as MetadataCache | null });
if (maxProfile.profile) {
let existing = await db.find(maxProfile.profile.pubkey);
if ((existing?.created ?? 0) < maxProfile.created) {
await db.put(maxProfile.profile);
}
}
})().catch(console.warn);
}, [dispatch, metadataFeed.store, db]);
useEffect(() => {
const replies = notificationFeed.store.notes.filter(a => a.kind === EventKind.TextNote && !isMuted(a.pubkey))
replies.forEach(nx => {
makeNotification(db, nx).then(notification => {
if (notification) {
// @ts-ignore
dispatch(sendNotification(notification))
}
})
})
}, [dispatch, notificationFeed.store, db]);
useEffect(() => {
const muted = getMutedKeys(mutedFeed.store.notes)
dispatch(setMuted(muted))
const newest = getNewest(mutedFeed.store.notes)
if (newest && newest.content.length > 0 && pubKey) {
decryptBlocked(newest, pubKey, privKey).then((plaintext) => {
try {
const blocked = JSON.parse(plaintext)
const keys = blocked.filter((p: any) => p && p.length === 2 && p[0] === "p").map((p: any) => p[1])
dispatch(setBlocked({
keys,
createdAt: newest.created_at,
}))
} catch (error) {
console.debug("Couldn't parse JSON")
}
}).catch((error) => console.warn(error))
}
}, [dispatch, mutedFeed.store])
useEffect(() => {
let dms = dmsFeed.store.notes.filter(a => a.kind === EventKind.DirectMessage);
dispatch(addDirectMessage(dms));
}, [dispatch, dmsFeed.store]);
}

4
src/Feed/ProfileFeed.ts
View File

@ -1,6 +1,4 @@
import { useLiveQuery } from "dexie-react-hooks";
import { useEffect, useMemo } from "react";
import { RootState } from "State/Store";
import { useEffect } from "react";
import { MetadataCache } from "State/Users";
import { useKey, useKeys } from "State/Users/Hooks";
import { HexKey } from "Nostr";

79
src/Feed/Subscription.ts
View File

@ -3,6 +3,7 @@ import { System } from "Nostr/System";
import { TaggedRawEvent } from "Nostr";
import { Subscriptions } from "Nostr/Subscriptions";
import { debounce } from "Util";
import { db } from "Db";
export type NoteStore = {
notes: Array<TaggedRawEvent>,
@ -10,7 +11,8 @@ export type NoteStore = {
};
export type UseSubscriptionOptions = {
leaveOpen: boolean
leaveOpen: boolean,
cache: boolean
}
interface ReducerArg {
@ -77,33 +79,49 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
const [state, dispatch] = useReducer(notesReducer, initStore);
const [debounceOutput, setDebounceOutput] = useState<number>(0);
const [subDebounce, setSubDebounced] = useState<Subscriptions>();
const useCache = useMemo(() => options?.cache === true, [options]);
useEffect(() => {
if (sub) {
return debounce(DebounceMs, () => {
dispatch({
type: "END",
end: false
});
setSubDebounced(sub);
});
}
}, [sub, options]);
useEffect(() => {
if (sub) {
sub.OnEvent = (e) => {
if (subDebounce) {
dispatch({
type: "END",
end: false
});
if (useCache) {
// preload notes from db
PreloadNotes(subDebounce.Id)
.then(ev => {
dispatch({
type: "EVENT",
ev: ev
});
})
.catch(console.warn);
}
subDebounce.OnEvent = (e) => {
dispatch({
type: "EVENT",
ev: e
});
if (useCache) {
db.events.put(e);
}
};
sub.OnEnd = (c) => {
subDebounce.OnEnd = (c) => {
if (!(options?.leaveOpen ?? false)) {
c.RemoveSubscription(sub.Id);
if (sub.IsFinished()) {
System.RemoveSubscription(sub.Id);
c.RemoveSubscription(subDebounce.Id);
if (subDebounce.IsFinished()) {
System.RemoveSubscription(subDebounce.Id);
}
}
dispatch({
@ -112,14 +130,23 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
});
};
console.debug("Adding sub: ", sub.ToObject());
System.AddSubscription(sub);
console.debug("Adding sub: ", subDebounce.ToObject());
System.AddSubscription(subDebounce);
return () => {
console.debug("Removing sub: ", sub.ToObject());
System.RemoveSubscription(sub.Id);
console.debug("Removing sub: ", subDebounce.ToObject());
System.RemoveSubscription(subDebounce.Id);
};
}
}, [subDebounce]);
}, [subDebounce, useCache]);
useEffect(() => {
if (subDebounce && useCache) {
return debounce(500, () => {
TrackNotesInFeed(subDebounce.Id, state.notes)
.catch(console.warn);
});
}
}, [state, useCache]);
useEffect(() => {
return debounce(DebounceMs, () => {
@ -140,4 +167,24 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
});
}
}
}
/**
* Lookup cached copy of feed
*/
const PreloadNotes = async (id: string): Promise<TaggedRawEvent[]> => {
const feed = await db.feeds.get(id);
if (feed) {
const events = await db.events.bulkGet(feed.ids);
return events.filter(a => a !== undefined).map(a => a!);
}
return [];
}
const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => {
const existing = await db.feeds.get(id);
const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)]));
const since = notes.reduce((acc, v) => acc > v.created_at ? v.created_at : acc, +Infinity);
const until = notes.reduce((acc, v) => acc < v.created_at ? v.created_at : acc, -Infinity);
await db.feeds.put({ id, ids, since, until });
}

2
src/Feed/ThreadFeed.ts
View File

@ -38,7 +38,7 @@ export default function useThreadFeed(id: u256) {
return thisSub;
}, [trackingEvents, pref, id]);
const main = useSubscription(sub, { leaveOpen: true });
const main = useSubscription(sub, { leaveOpen: true, cache: true });
useEffect(() => {
if (main.store) {

11
src/Feed/TimelineFeed.ts
View File

@ -14,6 +14,7 @@ export interface TimelineFeedOptions {
export interface TimelineSubject {
type: "pubkey" | "hashtag" | "global" | "ptag" | "keyword",
discriminator: string,
items: string[]
}
@ -32,7 +33,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
}
let sub = new Subscriptions();
sub.Id = `timeline:${subject.type}`;
sub.Id = `timeline:${subject.type}:${subject.discriminator}`;
sub.Kinds = new Set([EventKind.TextNote, EventKind.Repost]);
switch (subject.type) {
case "pubkey": {
@ -54,7 +55,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
}
}
return sub;
}, [subject.type, subject.items]);
}, [subject.type, subject.items, subject.discriminator]);
const sub = useMemo(() => {
let sub = createSub();
@ -86,7 +87,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return sub;
}, [until, since, options.method, pref, createSub]);
const main = useSubscription(sub, { leaveOpen: true });
const main = useSubscription(sub, { leaveOpen: true, cache: true });
const subRealtime = useMemo(() => {
let subLatest = createSub();
@ -98,7 +99,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return subLatest;
}, [pref, createSub]);
const latest = useSubscription(subRealtime, { leaveOpen: true });
const latest = useSubscription(subRealtime, { leaveOpen: true, cache: false });
const subNext = useMemo(() => {
let sub: Subscriptions | undefined;
@ -111,7 +112,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return sub ?? null;
}, [trackingEvents, pref, subject.type]);
const others = useSubscription(subNext, { leaveOpen: true });
const others = useSubscription(subNext, { leaveOpen: true, cache: true });
const subParents = useMemo(() => {
if (trackingParentEvents.length > 0) {