diff --git a/.github/workflows/eslint.yaml b/.github/workflows/eslint.yaml deleted file mode 100644 index 60f3427..0000000 --- a/.github/workflows/eslint.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Linting -on: - pull_request: -jobs: - formatting: - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: 16 - - name: Install Dependencies - run: yarn install - - name: Check Eslint - run: yarn workspace @snort/app eslint diff --git a/.github/workflows/formatting.yaml b/.github/workflows/test-lint.yaml similarity index 61% rename from .github/workflows/formatting.yaml rename to .github/workflows/test-lint.yaml index 27c70bc..352a0a4 100644 --- a/.github/workflows/formatting.yaml +++ b/.github/workflows/test-lint.yaml @@ -1,8 +1,8 @@ -name: Formatting +name: Test+Lint on: pull_request: jobs: - formatting: + test_and_lint: timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -14,5 +14,11 @@ jobs: node-version: 16 - name: Install Dependencies run: yarn install + - name: Build packages + run: yarn workspace @snort/nostr build + - name: Run tests + run: yarn workspace @snort/app test + - name: Check Eslint + run: yarn workspace @snort/app eslint - name: Check Formatting run: yarn workspace @snort/app prettier --check . diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..2291b09 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/Thumbs.db": true, + "**/node_modules": true + } +} \ No newline at end of file diff --git a/packages/app/package.json b/packages/app/package.json index 2606fa5..8a9e094 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -20,6 +20,7 @@ "bech32": "^2.0.0", "dexie": "^3.2.2", "dexie-react-hooks": "^1.1.1", + "events": "^3.3.0", "light-bolt11-decoder": "^2.1.0", "qr-code-styling": "^1.6.0-rc.1", "react": "^18.2.0", @@ -30,9 +31,9 @@ "react-query": "^3.39.2", "react-redux": "^8.0.5", "react-router-dom": "^6.5.0", - "react-scripts": "5.0.1", "react-textarea-autosize": "^8.4.0", "react-twitter-embed": "^4.0.4", + "throttle-debounce": "^5.0.0", "unist-util-visit": "^4.1.2", "use-long-press": "^2.0.3", "uuid": "^9.0.0", @@ -94,6 +95,7 @@ "lint-staged": ">=10", "prettier": "2.8.3", "react-app-rewired": "^2.2.1", + "react-scripts": "5.0.1", "typescript": "^4.9.4" }, "lint-staged": { diff --git a/packages/app/src/Cache/DMCache.ts b/packages/app/src/Cache/DMCache.ts new file mode 100644 index 0000000..2d4e63a --- /dev/null +++ b/packages/app/src/Cache/DMCache.ts @@ -0,0 +1,36 @@ +import { RawEvent } from "@snort/nostr"; +import { db } from "Db"; +import { dedupe } from "Util"; +import FeedCache from "./FeedCache"; + +class DMCache extends FeedCache { + constructor() { + super("DMCache", db.dms); + } + + key(of: RawEvent): string { + return of.id; + } + + override async preload(): Promise { + await super.preload(); + // load all dms to memory + await this.buffer([...this.onTable]); + } + + newest(): number { + let ret = 0; + this.cache.forEach(v => (ret = v.created_at > ret ? v.created_at : ret)); + return ret; + } + + allDms(): Array { + return [...this.cache.values()]; + } + + takeSnapshot(): Array { + return this.allDms(); + } +} + +export const DmCache = new DMCache(); diff --git a/packages/app/src/Cache/FeedCache.ts b/packages/app/src/Cache/FeedCache.ts new file mode 100644 index 0000000..387093a --- /dev/null +++ b/packages/app/src/Cache/FeedCache.ts @@ -0,0 +1,162 @@ +import { db } from "Db"; +import { Table } from "dexie"; +import { unixNowMs, unwrap } from "Util"; + +type HookFn = () => void; + +interface HookFilter { + key: string; + fn: HookFn; +} + +export default abstract class FeedCache { + #name: string; + #table: Table; + #hooks: Array = []; + #snapshot: Readonly> = []; + #changed = true; + protected onTable: Set = new Set(); + protected cache: Map = new Map(); + + constructor(name: string, table: Table) { + this.#name = name; + this.#table = table; + setInterval(() => { + console.debug( + `[${this.#name}] ${this.cache.size} loaded, ${this.onTable.size} on-disk, ${this.#hooks.length} hooks` + ); + }, 5_000); + } + + async preload() { + if (db.ready) { + const keys = await this.#table.toCollection().primaryKeys(); + this.onTable = new Set(keys.map(a => a as string)); + } + } + + hook(fn: HookFn, key: string | undefined) { + if (!key) { + return () => { + //noop + }; + } + + this.#hooks.push({ + key, + fn, + }); + return () => { + const idx = this.#hooks.findIndex(a => a.fn === fn); + if (idx >= 0) { + this.#hooks.splice(idx, 1); + } + }; + } + + getFromCache(key?: string) { + if (key) { + return this.cache.get(key); + } + } + + async get(key?: string) { + if (key && !this.cache.has(key) && db.ready) { + const cached = await this.#table.get(key); + if (cached) { + this.cache.set(this.key(cached), cached); + this.notifyChange([key]); + return cached; + } + } + return key ? this.cache.get(key) : undefined; + } + + async bulkGet(keys: Array) { + const missing = keys.filter(a => !this.cache.has(a)); + if (missing.length > 0 && db.ready) { + const cached = await this.#table.bulkGet(missing); + cached.forEach(a => { + if (a) { + this.cache.set(this.key(a), a); + } + }); + } + return keys + .map(a => this.cache.get(a)) + .filter(a => a) + .map(a => unwrap(a)); + } + + async set(obj: TCached) { + const k = this.key(obj); + this.cache.set(k, obj); + if (db.ready) { + await this.#table.put(obj); + this.onTable.add(k); + } + this.notifyChange([k]); + } + + async bulkSet(obj: Array) { + if (db.ready) { + await this.#table.bulkPut(obj); + obj.forEach(a => this.onTable.add(this.key(a))); + } + obj.forEach(v => this.cache.set(this.key(v), v)); + this.notifyChange(obj.map(a => this.key(a))); + } + + /** + * Loads a list of rows from disk cache + * @param keys List of ids to load + * @returns Keys that do not exist on disk cache + */ + async buffer(keys: Array): Promise> { + const needsBuffer = keys.filter(a => !this.cache.has(a)); + if (db.ready && needsBuffer.length > 0) { + const mapped = needsBuffer.map(a => ({ + has: this.onTable.has(a), + key: a, + })); + const start = unixNowMs(); + const fromCache = await this.#table.bulkGet(mapped.filter(a => a.has).map(a => a.key)); + const fromCacheFiltered = fromCache.filter(a => a !== undefined).map(a => unwrap(a)); + fromCacheFiltered.forEach(a => { + this.cache.set(this.key(a), a); + }); + this.notifyChange(fromCacheFiltered.map(a => this.key(a))); + console.debug( + `[${this.#name}] Loaded ${fromCacheFiltered.length}/${keys.length} in ${( + unixNowMs() - start + ).toLocaleString()} ms` + ); + return mapped.filter(a => !a.has).map(a => a.key); + } + + // no IndexdDB always return all keys + return needsBuffer; + } + + async clear() { + await this.#table.clear(); + this.cache.clear(); + this.onTable.clear(); + } + + snapshot() { + if (this.#changed) { + this.#snapshot = this.takeSnapshot(); + this.#changed = false; + } + return this.#snapshot; + } + + protected notifyChange(keys: Array) { + this.#changed = true; + this.#hooks.filter(a => keys.includes(a.key)).forEach(h => h.fn()); + } + + abstract key(of: TCached): string; + abstract takeSnapshot(): Array; +} diff --git a/packages/app/src/Cache/UserCache.ts b/packages/app/src/Cache/UserCache.ts new file mode 100644 index 0000000..a504030 --- /dev/null +++ b/packages/app/src/Cache/UserCache.ts @@ -0,0 +1,83 @@ +import FeedCache from "Cache/FeedCache"; +import { db } from "Db"; +import { LNURL } from "LNURL"; +import { MetadataCache } from "Cache"; + +class UserProfileCache extends FeedCache { + constructor() { + super("UserCache", db.users); + } + + key(of: MetadataCache): string { + return of.pubkey; + } + + async search(q: string): Promise> { + if (db.ready) { + // on-disk cache will always have more data + return ( + await db.users + .where("npub") + .startsWithIgnoreCase(q) + .or("name") + .startsWithIgnoreCase(q) + .or("display_name") + .startsWithIgnoreCase(q) + .or("nip05") + .startsWithIgnoreCase(q) + .toArray() + ).slice(0, 5); + } else { + return [...this.cache.values()] + .filter(user => { + const profile = user as MetadataCache; + return ( + profile.name?.includes(q) || + profile.npub?.includes(q) || + profile.display_name?.includes(q) || + profile.nip05?.includes(q) + ); + }) + .slice(0, 5); + } + } + + /** + * Try to update the profile metadata cache with a new version + * @param m Profile metadata + * @returns + */ + async update(m: MetadataCache) { + const existing = this.getFromCache(m.pubkey); + const refresh = existing && existing.created === m.created && existing.loaded < m.loaded; + if (!existing || existing.created < m.created || refresh) { + // fetch zapper key + const lnurl = m.lud16 || m.lud06; + if (lnurl) { + try { + const svc = new LNURL(lnurl); + await svc.load(); + m.zapService = svc.zapperPubkey; + } catch { + console.debug("Failed to load LNURL for zapper pubkey", lnurl); + } + // ignored + } + + this.cache.set(m.pubkey, m); + if (db.ready) { + await db.users.put(m); + this.onTable.add(m.pubkey); + } + this.notifyChange([m.pubkey]); + return true; + } + return false; + } + + takeSnapshot(): MetadataCache[] { + return []; + } +} + +export const UserCache = new UserProfileCache(); diff --git a/packages/app/src/State/Users/index.ts b/packages/app/src/Cache/index.ts similarity index 81% rename from packages/app/src/State/Users/index.ts rename to packages/app/src/Cache/index.ts index 8d9fd17..ae61faa 100644 --- a/packages/app/src/State/Users/index.ts +++ b/packages/app/src/Cache/index.ts @@ -1,5 +1,7 @@ import { HexKey, TaggedRawEvent, UserMetadata } from "@snort/nostr"; import { hexToBech32, unixNowMs } from "Util"; +import { DmCache } from "./DMCache"; +import { UserCache } from "./UserCache"; export interface MetadataCache extends UserMetadata { /** @@ -42,3 +44,10 @@ export function mapEventToProfile(ev: TaggedRawEvent) { console.error("Failed to parse JSON", ev, e); } } + +export async function preload() { + await UserCache.preload(); + await DmCache.preload(); +} + +export { UserCache, DmCache }; diff --git a/packages/app/src/Db/index.ts b/packages/app/src/Db/index.ts index 7a3b04e..7b30d58 100644 --- a/packages/app/src/Db/index.ts +++ b/packages/app/src/Db/index.ts @@ -1,9 +1,9 @@ import Dexie, { Table } from "dexie"; -import { u256 } from "@snort/nostr"; -import { MetadataCache } from "State/Users"; +import { FullRelaySettings, HexKey, RawEvent, u256 } from "@snort/nostr"; +import { MetadataCache } from "Cache"; export const NAME = "snortDB"; -export const VERSION = 4; +export const VERSION = 7; export interface SubCache { id: string; @@ -12,13 +12,33 @@ export interface SubCache { since?: number; } +export interface RelayMetrics { + addr: string; + events: number; + disconnects: number; + latency: number[]; +} + +export interface UsersRelays { + pubkey: HexKey; + relays: FullRelaySettings[]; +} + const STORES = { users: "++pubkey, name, display_name, picture, nip05, npub", + relays: "++addr", + userRelays: "++pubkey", + events: "++id, pubkey, created_at", + dms: "++id, pubkey", }; export class SnortDB extends Dexie { ready = false; users!: Table; + relayMetrics!: Table; + userRelays!: Table; + events!: Table; + dms!: Table; constructor() { super(NAME); diff --git a/packages/app/src/Element/Bookmarks.tsx b/packages/app/src/Element/Bookmarks.tsx index e9627ba..cd6ddea 100644 --- a/packages/app/src/Element/Bookmarks.tsx +++ b/packages/app/src/Element/Bookmarks.tsx @@ -1,30 +1,29 @@ import { useState, useMemo, ChangeEvent } from "react"; import { useSelector } from "react-redux"; import { FormattedMessage } from "react-intl"; - -import { dedupeByPubkey } from "Util"; -import Note from "Element/Note"; import { HexKey, TaggedRawEvent } from "@snort/nostr"; + +import Note from "Element/Note"; import { RootState } from "State/Store"; -import { UserCache } from "State/Users/UserCache"; +import { UserCache } from "Cache/UserCache"; import messages from "./messages"; interface BookmarksProps { pubkey: HexKey; - bookmarks: TaggedRawEvent[]; - related: TaggedRawEvent[]; + bookmarks: readonly TaggedRawEvent[]; + related: readonly TaggedRawEvent[]; } const Bookmarks = ({ pubkey, bookmarks, related }: BookmarksProps) => { const [onlyPubkey, setOnlyPubkey] = useState("all"); const loginPubKey = useSelector((s: RootState) => s.login.publicKey); const ps = useMemo(() => { - return dedupeByPubkey(bookmarks).map(ev => ev.pubkey); + return [...new Set(bookmarks.map(ev => ev.pubkey))]; }, [bookmarks]); function renderOption(p: HexKey) { - const profile = UserCache.get(p); + const profile = UserCache.getFromCache(p); return profile ? : null; } diff --git a/packages/app/src/Element/DM.tsx b/packages/app/src/Element/DM.tsx index ddadb92..8ebb549 100644 --- a/packages/app/src/Element/DM.tsx +++ b/packages/app/src/Element/DM.tsx @@ -3,14 +3,13 @@ import { useEffect, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; import { useIntl } from "react-intl"; import { useInView } from "react-intersection-observer"; +import { HexKey, TaggedRawEvent } from "@snort/nostr"; import useEventPublisher from "Feed/EventPublisher"; -import { Event } from "@snort/nostr"; import NoteTime from "Element/NoteTime"; import Text from "Element/Text"; import { setLastReadDm } from "Pages/MessagesPage"; import { RootState } from "State/Store"; -import { HexKey, TaggedRawEvent } from "@snort/nostr"; import { incDmInteraction } from "State/Login"; import { unwrap } from "Util"; @@ -32,11 +31,10 @@ export default function DM(props: DMProps) { const otherPubkey = isMe ? pubKey : unwrap(props.data.tags.find(a => a[0] === "p")?.[1]); async function decrypt() { - const e = new Event(props.data); - const decrypted = await publisher.decryptDm(e); + const decrypted = await publisher.decryptDm(props.data); setContent(decrypted || ""); if (!isMe) { - setLastReadDm(e.PubKey); + setLastReadDm(props.data.pubkey); dispatch(incDmInteraction()); } } diff --git a/packages/app/src/Element/HyperText.tsx b/packages/app/src/Element/HyperText.tsx index e5a9d70..05a16ec 100644 --- a/packages/app/src/Element/HyperText.tsx +++ b/packages/app/src/Element/HyperText.tsx @@ -111,25 +111,12 @@ export default function HyperText({ link, creator }: { link: string; creator: He ); } - } else if (tweetId && !pref.rewriteTwitterPosts) { + } else if (tweetId) { return (
); - } else if (pref.rewriteTwitterPosts && url.hostname == "twitter.com") { - url.host = "nitter.net"; - return ( - e.stopPropagation()} - target="_blank" - rel="noreferrer" - className="ext"> - {url.toString()} - - ); } else if (youtubeId) { return (