diff --git a/src-tauri/migrations/20231028083224_add_ndk_cache_table.sql b/src-tauri/migrations/20231028083224_add_ndk_cache_table.sql new file mode 100644 index 00000000..d1440e3d --- /dev/null +++ b/src-tauri/migrations/20231028083224_add_ndk_cache_table.sql @@ -0,0 +1,27 @@ +-- Add migration script here +CREATE TABLE + ndk_users ( + pubkey TEXT NOT NULL PRIMARY KEY, + profile TEXT, + createdAt NUMBER + ); + +CREATE TABLE + ndk_events ( + id TEXT NOT NULL PRIMARY KEY, + pubkey TEXT, + content TEXT, + kind NUMBER, + createdAt NUMBER, + relay TEXT, + event TEXT + ); + +CREATE TABLE + ndk_eventtags ( + id TEXT NOT NULL PRIMARY KEY, + eventId TEXT, + tag TEXT, + value TEXT, + tagValue TEXT + ); diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index d59d0edc..34938251 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -128,12 +128,20 @@ fn main() { tauri_plugin_sql::Builder::default() .add_migrations( "sqlite:lume_v2.db", - vec![Migration { - version: 20230418013219, - description: "initial data", - sql: include_str!("../migrations/20230418013219_initial_data.sql"), - kind: MigrationKind::Up, - }], + vec![ + Migration { + version: 20230418013219, + description: "initial data", + sql: include_str!("../migrations/20230418013219_initial_data.sql"), + kind: MigrationKind::Up, + }, + Migration { + version: 20231028083224, + description: "add ndk cache table", + sql: include_str!("../migrations/20231028083224_add_ndk_cache_table.sql"), + kind: MigrationKind::Up, + }, + ], ) .build(), ) diff --git a/src/libs/ndk/cache.ts b/src/libs/ndk/cache.ts index 782c0b1a..b5c2450f 100644 --- a/src/libs/ndk/cache.ts +++ b/src/libs/ndk/cache.ts @@ -5,60 +5,30 @@ import type { NDKFilter, NDKSubscription, NDKUserProfile, + NostrEvent, } from '@nostr-dev-kit/ndk'; -import _debug from 'debug'; +import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; -import { LRUCache } from 'typescript-lru-cache'; -import { createDatabase, db } from './db'; +import { LumeStorage } from '@libs/storage/instance'; -export { db } from './db'; - -interface NDKCacheAdapterDexieOptions { - /** - * The name of the database to use - */ - dbName?: string; - - /** - * Debug instance to use for logging - */ - debug?: debug.IDebugger; - - /** - * The number of seconds to store events in Dexie (IndexedDB) before they expire - * Defaults to 3600 seconds (1 hour) - */ - expirationTime?: number; - - /** - * Number of profiles to keep in an LRU cache - */ - profileCacheSize?: number | 'disabled'; -} - -export default class NDKCacheAdapterDexie implements NDKCacheAdapter { - public debug: debug.Debugger; - private expirationTime; - readonly locking; +export default class NDKCacheAdapterTauri implements NDKCacheAdapter { + public db: LumeStorage; public profiles?: LRUCache; private dirtyProfiles: Set = new Set(); + readonly locking: boolean; - constructor(opts: NDKCacheAdapterDexieOptions = {}) { - createDatabase(opts.dbName || 'ndk'); - this.debug = opts.debug || _debug('ndk:dexie-adapter'); + constructor(db: LumeStorage) { + this.db = db; this.locking = true; - this.expirationTime = opts.expirationTime || 3600; - if (opts.profileCacheSize !== 'disabled') { - this.profiles = new LRUCache({ - maxSize: opts.profileCacheSize || 100000, - }); + this.profiles = new LRUCache({ + max: 100000, + }); - setInterval(() => { - this.dumpProfiles(); - }, 1000 * 10); - } + setInterval(() => { + this.dumpProfiles(); + }, 1000 * 10); } public async query(subscription: NDKSubscription): Promise { @@ -73,9 +43,9 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { let profile = this.profiles.get(pubkey); if (!profile) { - const user = await db.users.get({ pubkey }); + const user = await this.db.getCacheUser(pubkey); if (user) { - profile = user.profile; + profile = user.profile as NDKUserProfile; this.profiles.set(pubkey, profile); } } @@ -126,7 +96,7 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { if (event.isParamReplaceable()) { const replaceableId = `${event.kind}:${event.pubkey}:${event.tagId()}`; - const existingEvent = await db.events.where({ id: replaceableId }).first(); + const existingEvent = await this.db.getCacheEvent(replaceableId); if ( existingEvent && event.created_at && @@ -137,7 +107,7 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { } if (addEvent) { - db.events.put({ + this.db.setCacheEvent({ id: event.tagId(), pubkey: event.pubkey, content: event.content, @@ -153,7 +123,7 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { event.tags.forEach((tag) => { if (tag[0].length !== 1) return; - db.eventTags.put({ + this.db.setCacheEventTag({ id: `${event.id}:${tag[0]}:${tag[1]}`, eventId: event.id, tag: tag[0], @@ -182,9 +152,9 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { if (hasAllKeys && filter.authors) { for (const pubkey of filter.authors) { - const events = await db.events.where({ pubkey }).toArray(); + const events = await this.db.getCacheEventsByPubkey(pubkey); for (const event of events) { - let rawEvent; + let rawEvent: NostrEvent; try { rawEvent = JSON.parse(event.event); } catch (e) { @@ -218,9 +188,9 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { if (hasAllKeys && filter.kinds) { for (const kind of filter.kinds) { - const events = await db.events.where({ kind }).toArray(); + const events = await this.db.getCacheEventsByKind(kind); for (const event of events) { - let rawEvent; + let rawEvent: NostrEvent; try { rawEvent = JSON.parse(event.event); } catch (e) { @@ -252,10 +222,10 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { if (hasAllKeys && filter.ids) { for (const id of filter.ids) { - const event = await db.events.where({ id }).first(); + const event = await this.db.getCacheEvent(id); if (!event) continue; - let rawEvent; + let rawEvent: NostrEvent; try { rawEvent = JSON.parse(event.event); } catch (e) { @@ -295,10 +265,10 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { for (const author of filter.authors) { for (const dTag of filter['#d']) { const replaceableId = `${kind}:${author}:${dTag}`; - const event = await db.events.where({ id: replaceableId }).first(); + const event = await this.db.getCacheEvent(replaceableId); if (!event) continue; - let rawEvent; + let rawEvent: NostrEvent; try { rawEvent = JSON.parse(event.event); } catch (e) { @@ -335,10 +305,10 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { if (filter.kinds && filter.authors) { for (const kind of filter.kinds) { for (const author of filter.authors) { - const events = await db.events.where({ kind, pubkey: author }).toArray(); + const events = await this.db.getCacheEventsByKindAndAuthor(kind, author); for (const event of events) { - let rawEvent; + let rawEvent: NostrEvent; try { rawEvent = JSON.parse(event.event); } catch (e) { @@ -400,12 +370,12 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { } for (const value of values) { - const eventTags = await db.eventTags.where({ tagValue: tag + value }).toArray(); + const eventTags = await this.db.getCacheEventTagsByTagValue(tag + value); if (!eventTags.length) continue; const eventIds = eventTags.map((t) => t.eventId); - const events = await db.events.where('id').anyOf(eventIds).toArray(); + const events = await this.db.getCacheEvents(eventIds); for (const event of events) { let rawEvent; try { @@ -441,13 +411,13 @@ export default class NDKCacheAdapterDexie implements NDKCacheAdapter { profiles.push({ pubkey, - profile, + profile: JSON.stringify(profile), createdAt: Date.now(), }); } if (profiles.length) { - await db.users.bulkPut(profiles); + await this.db.setCacheProfiles(profiles); } this.dirtyProfiles.clear(); diff --git a/src/libs/ndk/instance.ts b/src/libs/ndk/instance.ts index 1fc844c9..56d5f0e4 100644 --- a/src/libs/ndk/instance.ts +++ b/src/libs/ndk/instance.ts @@ -1,11 +1,11 @@ import NDK, { NDKNip46Signer, NDKPrivateKeySigner } from '@nostr-dev-kit/ndk'; -import NDKCacheAdapterDexie from '@nostr-dev-kit/ndk-cache-dexie'; import { ndkAdapter } from '@nostr-fetch/adapter-ndk'; import { message } from '@tauri-apps/plugin-dialog'; import { fetch } from '@tauri-apps/plugin-http'; import { NostrFetcher } from 'nostr-fetch'; import { useEffect, useMemo, useState } from 'react'; +import NDKCacheAdapterTauri from '@libs/ndk/cache'; import { useStorage } from '@libs/storage/provider'; export const NDKInstance = () => { @@ -77,7 +77,7 @@ export const NDKInstance = () => { const outboxSetting = await db.getSettingValue('outbox'); const explicitRelayUrls = await getExplicitRelays(); - const dexieAdapter = new NDKCacheAdapterDexie({ dbName: 'lume_ndkcache' }); + const dexieAdapter = new NDKCacheAdapterTauri(db); const instance = new NDK({ explicitRelayUrls, cacheAdapter: dexieAdapter, diff --git a/src/libs/storage/instance.ts b/src/libs/storage/instance.ts index cfe84e83..d973222f 100644 --- a/src/libs/storage/instance.ts +++ b/src/libs/storage/instance.ts @@ -6,7 +6,15 @@ import Database from '@tauri-apps/plugin-sql'; import { FULL_RELAYS } from '@stores/constants'; import { rawEvent } from '@utils/transform'; -import { Account, DBEvent, Relays, Widget } from '@utils/types'; +import type { + Account, + DBEvent, + NDKCacheEvent, + NDKCacheEventTag, + NDKCacheUser, + Relays, + Widget, +} from '@utils/types'; export class LumeStorage { public db: Database; @@ -37,6 +45,115 @@ export class LumeStorage { return await invoke('secure_remove', { key }); } + public async getCacheUser(pubkey: string) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_users WHERE pubkey = $1 ORDER BY pubkey DESC LIMIT 1;', + [pubkey] + ); + + if (results.length < 1) return null; + + if (typeof results[0].profile === 'string') + results[0].profile = JSON.parse(results[0].profile); + + return results[0]; + } + + public async getCacheEvent(id: string) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_events WHERE id = $1 ORDER BY id DESC LIMIT 1;', + [id] + ); + + if (results.length < 1) return null; + return results[0]; + } + + public async getCacheEvents(ids: string[]) { + const idsArr = `'${ids.join("','")}'`; + + const results: Array = await this.db.select( + `SELECT * FROM ndk_events WHERE id IN (${idsArr}) ORDER BY id;` + ); + + if (results.length < 1) return []; + return results; + } + + public async getCacheEventsByPubkey(pubkey: string) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_events WHERE pubkey = $1 ORDER BY id;', + [pubkey] + ); + + if (results.length < 1) return []; + return results; + } + + public async getCacheEventsByKind(kind: number) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_events WHERE kind = $1 ORDER BY id;', + [kind] + ); + + if (results.length < 1) return []; + return results; + } + + public async getCacheEventsByKindAndAuthor(kind: number, pubkey: string) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_events WHERE kind = $1 AND pubkey = $2 ORDER BY id;', + [kind, pubkey] + ); + + if (results.length < 1) return []; + return results; + } + + public async getCacheEventTagsByTagValue(tagValue: string) { + const results: Array = await this.db.select( + 'SELECT * FROM ndk_eventtags WHERE tagValue = $1 ORDER BY id;', + [tagValue] + ); + + if (results.length < 1) return []; + return results; + } + + public async setCacheEvent({ + id, + pubkey, + content, + kind, + createdAt, + relay, + event, + }: NDKCacheEvent) { + return await this.db.execute( + 'INSERT OR IGNORE INTO ndk_events (id, pubkey, content, kind, createdAt, relay, event) VALUES ($1, $2, $3, $4, $5, $6, $7);', + [id, pubkey, content, kind, createdAt, relay, event] + ); + } + + public async setCacheEventTag({ id, eventId, tag, value, tagValue }: NDKCacheEventTag) { + return await this.db.execute( + 'INSERT OR IGNORE INTO ndk_eventtags (id, eventId, tag, value, tagValue) VALUES ($1, $2, $3, $4, $5);', + [id, eventId, tag, value, tagValue] + ); + } + + public async setCacheProfiles(profiles: Array) { + return await Promise.all( + profiles.map( + async (profile) => + await this.db.execute( + 'INSERT OR IGNORE INTO ndk_users (pubkey, profile, createdAt) VALUES ($1, $2, $3);', + [profile.pubkey, profile.profile, profile.createdAt] + ) + ) + ); + } + public async checkAccount() { const result: Array<{ total: string }> = await this.db.select( 'SELECT COUNT(*) AS "total" FROM accounts;' diff --git a/src/shared/user.tsx b/src/shared/user.tsx index 7371be2b..02449461 100644 --- a/src/shared/user.tsx +++ b/src/shared/user.tsx @@ -189,7 +189,7 @@ export const User = memo(function User({ return (
-
+
diff --git a/src/shared/widgets/eventLoader.tsx b/src/shared/widgets/eventLoader.tsx index 76b6e089..3917af3a 100644 --- a/src/shared/widgets/eventLoader.tsx +++ b/src/shared/widgets/eventLoader.tsx @@ -19,19 +19,18 @@ export function EventLoader({ firstTime }: { firstTime: boolean }) { useEffect(() => { async function getEvents() { const events = await getAllEventsSinceLastLogin(); - console.log('total new events has found: ', events.data.length); + console.log('total new events has found: ', events.length); - const promises = await Promise.all( - events.data.map(async (event) => await db.createEvent(event)) - ); - - if (promises) { + if (events) { setProgress(100); setIsFetched(); + // invalidate queries - queryClient.invalidateQueries({ - queryKey: ['local-network-widget'] + await queryClient.invalidateQueries({ + queryKey: ['local-network-widget'], }); + + // update last login time, use for next fetch await db.updateLastLogin(); } } diff --git a/src/shared/widgets/local/notification.tsx b/src/shared/widgets/local/notification.tsx index b0134c56..ce969d37 100644 --- a/src/shared/widgets/local/notification.tsx +++ b/src/shared/widgets/local/notification.tsx @@ -10,6 +10,7 @@ import { TitleBar } from '@shared/titleBar'; import { WidgetWrapper } from '@shared/widgets'; import { useActivities } from '@stores/activities'; +import { useWidgets } from '@stores/widgets'; import { useNostr } from '@utils/hooks/useNostr'; import { Widget } from '@utils/types'; @@ -23,6 +24,8 @@ export function LocalNotificationWidget({ params }: { params: Widget }) { state.setActivities, ]); + const isFetched = useWidgets((state) => state.isFetched); + const renderEvent = useCallback( (event: NDKEvent) => { if (event.pubkey === db.account.pubkey) return null; @@ -37,8 +40,8 @@ export function LocalNotificationWidget({ params }: { params: Widget }) { setActivities(events); } - getActivities(); - }, []); + if (isFetched) getActivities(); + }, [isFetched]); return ( diff --git a/src/utils/hooks/useEvent.ts b/src/utils/hooks/useEvent.ts index 31cfc918..b4b00dab 100644 --- a/src/utils/hooks/useEvent.ts +++ b/src/utils/hooks/useEvent.ts @@ -3,14 +3,12 @@ import { useQuery } from '@tanstack/react-query'; import { AddressPointer } from 'nostr-tools/lib/types/nip19'; import { useNDK } from '@libs/ndk/provider'; -import { useStorage } from '@libs/storage/provider'; export function useEvent( id: undefined | string, naddr?: undefined | AddressPointer, embed?: undefined | string ) { - const { db } = useStorage(); const { ndk } = useNDK(); const { status, data } = useQuery({ queryKey: ['event', id], @@ -33,20 +31,16 @@ export function useEvent( return event; } - // get event from db - const dbEvent = await db.getEventByID(id); - if (dbEvent) return dbEvent; - - // get event from relay if event in db not present + // get event from relay const event = await ndk.fetchEvent(id); if (!event) return Promise.reject(new Error('event not found')); - await db.createEvent(event); - return event; }, - enabled: !!ndk, refetchOnWindowFocus: false, + refetchOnMount: false, + refetchOnReconnect: false, + staleTime: Infinity, }); return { status, data }; diff --git a/src/utils/hooks/useNostr.ts b/src/utils/hooks/useNostr.ts index a02b996e..ed44e08e 100644 --- a/src/utils/hooks/useNostr.ts +++ b/src/utils/hooks/useNostr.ts @@ -186,10 +186,9 @@ export function useNostr() { { since: since } )) as unknown as NDKEvent[]; - return { status: 'ok', message: 'fetch completed', data: events }; + return events; } catch (e) { console.error('prefetch events failed, error: ', e); - return { status: 'failed', message: e }; } }; diff --git a/src/utils/hooks/useProfile.ts b/src/utils/hooks/useProfile.ts index 6f73237e..498d0897 100644 --- a/src/utils/hooks/useProfile.ts +++ b/src/utils/hooks/useProfile.ts @@ -22,12 +22,10 @@ export function useProfile(pubkey: string, embed?: string) { return await user.fetchProfile(); }, - enabled: !!ndk, staleTime: Infinity, refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, - retry: 2, }); return { status, user, error }; diff --git a/src/utils/types.d.ts b/src/utils/types.d.ts index a9e13585..ba8794ae 100644 --- a/src/utils/types.d.ts +++ b/src/utils/types.d.ts @@ -115,3 +115,27 @@ export interface Resources { title: string; data: Array; } + +export interface NDKCacheUser { + pubkey: string; + profile: string | NDKUserProfile; + createdAt: number; +} + +export interface NDKCacheEvent { + id: string; + pubkey: string; + content: string; + kind: number; + createdAt: number; + relay: string; + event: string; +} + +export interface NDKCacheEventTag { + id: string; + eventId: string; + tag: string; + value: string; + tagValue: string; +}