Kieran/snort
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feed cache

Browse Source
This commit is contained in:
Kieran 2023-02-01 13:22:11 +00:00
parent 1b4ac0d711
commit 48407d9198
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
17 changed files with 158 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/Db/index.ts
View File

@ -1,23 +1,35 @@
import Dexie, { Table } from "dexie";
import { TaggedRawEvent, u256 } from "Nostr";
import { MetadataCache } from "State/Users";
import { hexToBech32 } from "Util";
export const NAME = 'snortDB'
export const VERSION = 2
export const VERSION = 3
export interface SubCache {
id: string,
ids: u256[],
until?: number,
since?: number,
}
const STORES = {
users: '++pubkey, name, display_name, picture, nip05, npub'
users: '++pubkey, name, display_name, picture, nip05, npub',
events: '++id, pubkey, created_at',
feeds: '++id'
}
export class SnortDB extends Dexie {
users!: Table<MetadataCache>;
events!: Table<TaggedRawEvent>;
feeds!: Table<SubCache>;
constructor() {
super(NAME);
this.version(VERSION).stores(STORES).upgrade(tx => {
return tx.table("users").toCollection().modify(user => {
this.version(VERSION).stores(STORES).upgrade(async tx => {
await tx.table("users").toCollection().modify(user => {
user.npub = hexToBech32("npub", user.pubkey)
})
});
});
}
}

20
src/Feed/EventPublisher.ts
View File

@ -78,8 +78,8 @@ export default function useEventPublisher() {
}
return {
nip42Auth: async (challenge: string, relay:string) => {
if(pubKey) {
nip42Auth: async (challenge: string, relay: string) => {
if (pubKey) {
const ev = NEvent.ForPubKey(pubKey);
ev.Kind = EventKind.Auth;
ev.Content = "";
@ -112,17 +112,17 @@ export default function useEventPublisher() {
ev.Kind = EventKind.Lists;
ev.Tags.push(new Tag(["d", Lists.Muted], ev.Tags.length))
keys.forEach(p => {
ev.Tags.push(new Tag(["p", p], ev.Tags.length))
ev.Tags.push(new Tag(["p", p], ev.Tags.length))
})
let content = ""
if (priv.length > 0) {
const ps = priv.map(p => ["p", p])
const plaintext = JSON.stringify(ps)
if (hasNip07 && !privKey) {
content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
} else if (privKey) {
content = await ev.EncryptData(plaintext, pubKey, privKey)
}
const ps = priv.map(p => ["p", p])
const plaintext = JSON.stringify(ps)
if (hasNip07 && !privKey) {
content = await barierNip07(() => window.nostr.nip04.encrypt(pubKey, plaintext));
} else if (privKey) {
content = await ev.EncryptData(plaintext, pubKey, privKey)
}
}
ev.Content = content;
return await signEvent(ev);

2
src/Feed/FollowersFeed.ts
View File

@ -7,7 +7,7 @@ import useSubscription from "Feed/Subscription";
export default function useFollowersFeed(pubkey: HexKey) {
const sub = useMemo(() => {
let x = new Subscriptions();
x.Id = "followers";
x.Id = `followers:${pubkey.slice(0, 12)}`;
x.Kinds = new Set([EventKind.ContactList]);
x.PTags = new Set([pubkey]);

2
src/Feed/FollowsFeed.ts
View File

@ -7,7 +7,7 @@ import useSubscription, { NoteStore } from "Feed/Subscription";
export default function useFollowsFeed(pubkey: HexKey) {
const sub = useMemo(() => {
let x = new Subscriptions();
x.Id = "follows";
x.Id = `follows:${pubkey.slice(0, 12)}`;
x.Kinds = new Set([EventKind.ContactList]);
x.Authors = new Set([pubkey]);

8
src/Feed/LoginFeed.ts
View File

@ -77,10 +77,10 @@ export default function useLoginFeed() {
return dms;
}, [pubKey]);
const metadataFeed = useSubscription(subMetadata, { leaveOpen: true });
const notificationFeed = useSubscription(subNotification, { leaveOpen: true });
const dmsFeed = useSubscription(subDms, { leaveOpen: true });
const mutedFeed = useSubscription(subMuted, { leaveOpen: true });
const metadataFeed = useSubscription(subMetadata, { leaveOpen: true, cache: true });
const notificationFeed = useSubscription(subNotification, { leaveOpen: true, cache: true });
const dmsFeed = useSubscription(subDms, { leaveOpen: true, cache: true });
const mutedFeed = useSubscription(subMuted, { leaveOpen: true, cache: true });
useEffect(() => {
let contactList = metadataFeed.store.notes.filter(a => a.kind === EventKind.ContactList);

4
src/Feed/ProfileFeed.ts
View File

@ -1,6 +1,4 @@
import { useLiveQuery } from "dexie-react-hooks";
import { useEffect, useMemo } from "react";
import { RootState } from "State/Store";
import { useEffect } from "react";
import { MetadataCache } from "State/Users";
import { useKey, useKeys } from "State/Users/Hooks";
import { HexKey } from "Nostr";

79
src/Feed/Subscription.ts
View File

@ -3,6 +3,7 @@ import { System } from "Nostr/System";
import { TaggedRawEvent } from "Nostr";
import { Subscriptions } from "Nostr/Subscriptions";
import { debounce } from "Util";
import { db } from "Db";
export type NoteStore = {
notes: Array<TaggedRawEvent>,
@ -10,7 +11,8 @@ export type NoteStore = {
};
export type UseSubscriptionOptions = {
leaveOpen: boolean
leaveOpen: boolean,
cache: boolean
}
interface ReducerArg {
@ -77,33 +79,49 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
const [state, dispatch] = useReducer(notesReducer, initStore);
const [debounceOutput, setDebounceOutput] = useState<number>(0);
const [subDebounce, setSubDebounced] = useState<Subscriptions>();
const useCache = useMemo(() => options?.cache === true, [options]);
useEffect(() => {
if (sub) {
return debounce(DebounceMs, () => {
dispatch({
type: "END",
end: false
});
setSubDebounced(sub);
});
}
}, [sub, options]);
useEffect(() => {
if (sub) {
sub.OnEvent = (e) => {
if (subDebounce) {
dispatch({
type: "END",
end: false
});
if (useCache) {
// preload notes from db
PreloadNotes(subDebounce.Id)
.then(ev => {
dispatch({
type: "EVENT",
ev: ev
});
})
.catch(console.warn);
}
subDebounce.OnEvent = (e) => {
dispatch({
type: "EVENT",
ev: e
});
if (useCache) {
db.events.put(e);
}
};
sub.OnEnd = (c) => {
subDebounce.OnEnd = (c) => {
if (!(options?.leaveOpen ?? false)) {
c.RemoveSubscription(sub.Id);
if (sub.IsFinished()) {
System.RemoveSubscription(sub.Id);
c.RemoveSubscription(subDebounce.Id);
if (subDebounce.IsFinished()) {
System.RemoveSubscription(subDebounce.Id);
}
}
dispatch({
@ -112,14 +130,23 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
});
};
console.debug("Adding sub: ", sub.ToObject());
System.AddSubscription(sub);
console.debug("Adding sub: ", subDebounce.ToObject());
System.AddSubscription(subDebounce);
return () => {
console.debug("Removing sub: ", sub.ToObject());
System.RemoveSubscription(sub.Id);
console.debug("Removing sub: ", subDebounce.ToObject());
System.RemoveSubscription(subDebounce.Id);
};
}
}, [subDebounce]);
}, [subDebounce, useCache]);
useEffect(() => {
if (subDebounce && useCache) {
return debounce(500, () => {
TrackNotesInFeed(subDebounce.Id, state.notes)
.catch(console.warn);
});
}
}, [state, useCache]);
useEffect(() => {
return debounce(DebounceMs, () => {
@ -140,4 +167,24 @@ export default function useSubscription(sub: Subscriptions | null, options?: Use
});
}
}
}
/**
* Lookup cached copy of feed
*/
const PreloadNotes = async (id: string): Promise<TaggedRawEvent[]> => {
const feed = await db.feeds.get(id);
if (feed) {
const events = await db.events.bulkGet(feed.ids);
return events.filter(a => a !== undefined).map(a => a!);
}
return [];
}
const TrackNotesInFeed = async (id: string, notes: TaggedRawEvent[]) => {
const existing = await db.feeds.get(id);
const ids = Array.from(new Set([...(existing?.ids || []), ...notes.map(a => a.id)]));
const since = notes.reduce((acc, v) => acc > v.created_at ? v.created_at : acc, +Infinity);
const until = notes.reduce((acc, v) => acc < v.created_at ? v.created_at : acc, -Infinity);
await db.feeds.put({ id, ids, since, until });
}

2
src/Feed/ThreadFeed.ts
View File

@ -38,7 +38,7 @@ export default function useThreadFeed(id: u256) {
return thisSub;
}, [trackingEvents, pref, id]);
const main = useSubscription(sub, { leaveOpen: true });
const main = useSubscription(sub, { leaveOpen: true, cache: true });
useEffect(() => {
if (main.store) {

11
src/Feed/TimelineFeed.ts
View File

@ -14,6 +14,7 @@ export interface TimelineFeedOptions {
export interface TimelineSubject {
type: "pubkey" | "hashtag" | "global" | "ptag" | "keyword",
discriminator: string,
items: string[]
}
@ -32,7 +33,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
}
let sub = new Subscriptions();
sub.Id = `timeline:${subject.type}`;
sub.Id = `timeline:${subject.type}:${subject.discriminator}`;
sub.Kinds = new Set([EventKind.TextNote, EventKind.Repost]);
switch (subject.type) {
case "pubkey": {
@ -54,7 +55,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
}
}
return sub;
}, [subject.type, subject.items]);
}, [subject.type, subject.items, subject.discriminator]);
const sub = useMemo(() => {
let sub = createSub();
@ -86,7 +87,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return sub;
}, [until, since, options.method, pref, createSub]);
const main = useSubscription(sub, { leaveOpen: true });
const main = useSubscription(sub, { leaveOpen: true, cache: true });
const subRealtime = useMemo(() => {
let subLatest = createSub();
@ -98,7 +99,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return subLatest;
}, [pref, createSub]);
const latest = useSubscription(subRealtime, { leaveOpen: true });
const latest = useSubscription(subRealtime, { leaveOpen: true, cache: false });
const subNext = useMemo(() => {
let sub: Subscriptions | undefined;
@ -111,7 +112,7 @@ export default function useTimelineFeed(subject: TimelineSubject, options: Timel
return sub ?? null;
}, [trackingEvents, pref, subject.type]);
const others = useSubscription(subNext, { leaveOpen: true });
const others = useSubscription(subNext, { leaveOpen: true, cache: true });
const subParents = useMemo(() => {
if (trackingParentEvents.length > 0) {

2
src/Pages/HashTagsPage.tsx
View File

@ -8,7 +8,7 @@ const HashTagsPage = () => {
return (
<>
<h2>#{tag}</h2>
<Timeline key={tag} subject={{ type: "hashtag", items: [tag] }} postsOnly={false} method={"TIME_RANGE"} />
<Timeline key={tag} subject={{ type: "hashtag", items: [tag], discriminator: tag }} postsOnly={false} method={"TIME_RANGE"} />
</>
)
}

2
src/Pages/Notifications.tsx
View File

@ -17,7 +17,7 @@ export default function NotificationsPage() {
return (
<>
{pubkey ?
<Timeline subject={{ type: "ptag", items: [pubkey!] }} postsOnly={false} method={"TIME_RANGE"} />
<Timeline subject={{ type: "ptag", items: [pubkey!], discriminator: pubkey!.slice(0, 12) }} postsOnly={false} method={"TIME_RANGE"} />
: null}
</>
)

4
src/Pages/ProfilePage.tsx
View File

@ -105,7 +105,7 @@ export default function ProfilePage() {
function tabContent() {
switch (tab) {
case ProfileTab.Notes:
return <Timeline key={id} subject={{ type: "pubkey", items: [id] }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
return <Timeline key={id} subject={{ type: "pubkey", items: [id], discriminator: id.slice(0, 12) }} postsOnly={false} method={"LIMIT_UNTIL"} ignoreModeration={true} />;
case ProfileTab.Follows: {
if (isMe) {
return (
@ -195,7 +195,7 @@ export default function ProfilePage() {
return (
<>
<div className="profile flex">
{user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w}/>}
{user?.banner && <ProxyImg alt="banner" className="banner" src={user.banner} size={w} />}
<div className="profile-wrapper flex">
{avatar()}
{userDetails()}

2
src/Pages/Root.tsx
View File

@ -29,7 +29,7 @@ export default function RootPage() {
}
const isGlobal = loggedOut || tab === RootTab.Global;
const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [] } : { type: "pubkey", items: follows };
const timelineSubect: TimelineSubject = isGlobal ? { type: "global", items: [], discriminator: "all" } : { type: "pubkey", items: follows, discriminator: "follows" };
return (
<>
{pubKey ? <>

2
src/Pages/SearchPage.tsx
View File

@ -43,7 +43,7 @@ const SearchPage = () => {
<div className="flex mb10">
<input type="text" className="f-grow mr10" placeholder="Search.." value={search} onChange={e => setSearch(e.target.value)} />
</div>
{keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword] }} postsOnly={false} method={"LIMIT_UNTIL"} />}
{keyword && <Timeline key={keyword} subject={{ type: "keyword", items: [keyword], discriminator: keyword }} postsOnly={false} method={"LIMIT_UNTIL"} />}
</>
)
}

22
src/State/Users.ts
View File

@ -35,20 +35,20 @@ export function mapEventToProfile(ev: TaggedRawEvent) {
...data
} as MetadataCache;
} catch (e) {
console.error("Failed to parse JSON", ev, e);
console.error("Failed to parse JSON", ev, e);
}
}
export interface UsersDb {
isAvailable(): Promise<boolean>
query(str: string): Promise<MetadataCache[]>
find(key: HexKey): Promise<MetadataCache | undefined>
add(user: MetadataCache): Promise<any>
put(user: MetadataCache): Promise<any>
bulkAdd(users: MetadataCache[]): Promise<any>
bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
bulkPut(users: MetadataCache[]): Promise<any>
update(key: HexKey, fields: Record<string, any>): Promise<any>
isAvailable(): Promise<boolean>
query(str: string): Promise<MetadataCache[]>
find(key: HexKey): Promise<MetadataCache | undefined>
add(user: MetadataCache): Promise<any>
put(user: MetadataCache): Promise<any>
bulkAdd(users: MetadataCache[]): Promise<any>
bulkGet(keys: HexKey[]): Promise<MetadataCache[]>
bulkPut(users: MetadataCache[]): Promise<any>
update(key: HexKey, fields: Record<string, any>): Promise<any>
}
export interface UsersStore {
@ -65,7 +65,7 @@ const UsersSlice = createSlice({
initialState: InitState,
reducers: {
setUsers(state, action: PayloadAction<Record<HexKey, MetadataCache>>) {
state.users = action.payload
state.users = action.payload
}
}
});

27
src/State/Users/Db.ts
View File

@ -5,6 +5,8 @@ import { UsersDb, MetadataCache, setUsers } from "State/Users";
import store from "State/Store";
class IndexedDb implements UsersDb {
ready: boolean = false;
isAvailable() {
if ("indexedDB" in window) {
return new Promise<boolean>((resolve) => {
@ -26,12 +28,12 @@ class IndexedDb implements UsersDb {
query(q: string) {
return idb.users
.where("npub").startsWithIgnoreCase(q)
.or("name").startsWithIgnoreCase(q)
.or("display_name").startsWithIgnoreCase(q)
.or("nip05").startsWithIgnoreCase(q)
.limit(5)
.toArray()
.where("npub").startsWithIgnoreCase(q)
.or("name").startsWithIgnoreCase(q)
.or("display_name").startsWithIgnoreCase(q)
.or("nip05").startsWithIgnoreCase(q)
.limit(5)
.toArray()
}
bulkGet(keys: HexKey[]) {
@ -41,7 +43,7 @@ class IndexedDb implements UsersDb {
add(user: MetadataCache) {
return idb.users.add(user)
}
put(user: MetadataCache) {
return idb.users.put(user)
}
@ -88,21 +90,21 @@ class ReduxUsersDb implements UsersDb {
async add(user: MetadataCache) {
const state = store.getState()
const { users } = state.users
store.dispatch(setUsers({...users, [user.pubkey]: user }))
store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
}
async put(user: MetadataCache) {
const state = store.getState()
const { users } = state.users
store.dispatch(setUsers({...users, [user.pubkey]: user }))
store.dispatch(setUsers({ ...users, [user.pubkey]: user }))
}
async bulkAdd(newUserProfiles: MetadataCache[]) {
const state = store.getState()
const { users } = state.users
const newUsers = newUserProfiles.reduce(groupByPubkey, {})
store.dispatch(setUsers({...users, ...newUsers }))
store.dispatch(setUsers({ ...users, ...newUsers }))
}
async bulkGet(keys: HexKey[]) {
@ -118,8 +120,8 @@ class ReduxUsersDb implements UsersDb {
const state = store.getState()
const { users } = state.users
const current = users[key]
const updated = {...current, ...fields }
store.dispatch(setUsers({...users, [key]: updated }))
const updated = { ...current, ...fields }
store.dispatch(setUsers({ ...users, [key]: updated }))
}
async bulkPut(newUsers: MetadataCache[]) {
@ -138,6 +140,7 @@ let db: UsersDb = inMemoryDb
indexedDb.isAvailable().then((available) => {
if (available) {
console.debug('Using Indexed DB')
indexedDb.ready = true;
db = indexedDb;
} else {
console.debug('Using in-memory DB')

44
src/State/Users/Hooks.ts
View File

@ -10,10 +10,10 @@ export function useQuery(query: string, limit: number = 5) {
const allUsers = useLiveQuery(
() => db.query(query)
.catch((err) => {
console.error(err)
return inMemoryDb.query(query)
}),
.catch((err) => {
console.error(err)
return inMemoryDb.query(query)
}),
[query],
)
@ -26,14 +26,14 @@ export function useKey(pubKey: HexKey) {
const defaultUser = users[pubKey]
const user = useLiveQuery(async () => {
if (pubKey) {
try {
return await db.find(pubKey);
} catch (error) {
console.error(error)
return defaultUser
}
}
if (pubKey) {
try {
return await db.find(pubKey);
} catch (error) {
console.error(error)
return defaultUser
}
}
}, [pubKey, defaultUser]);
return user
@ -42,17 +42,17 @@ export function useKey(pubKey: HexKey) {
export function useKeys(pubKeys: HexKey[]): Map<HexKey, MetadataCache> {
const db = getDb()
const dbUsers = useLiveQuery(async () => {
if (pubKeys) {
try {
const ret = await db.bulkGet(pubKeys);
return new Map(ret.map(a => [a.pubkey, a]))
} catch (error) {
console.error(error)
const ret = await inMemoryDb.bulkGet(pubKeys);
return new Map(ret.map(a => [a.pubkey, a]))
}
if (pubKeys) {
try {
const ret = await db.bulkGet(pubKeys);
return new Map(ret.map(a => [a.pubkey, a]))
} catch (error) {
console.error(error)
const ret = await inMemoryDb.bulkGet(pubKeys);
return new Map(ret.map(a => [a.pubkey, a]))
}
return new Map()
}
return new Map()
}, [pubKeys]);
return dbUsers!