diff --git a/packages/app/src/Components/Feed/LocalSearch.tsx b/packages/app/src/Components/Feed/LocalSearch.tsx new file mode 100644 index 00000000..561c71b0 --- /dev/null +++ b/packages/app/src/Components/Feed/LocalSearch.tsx @@ -0,0 +1,46 @@ +import { EventKind, RequestBuilder, TaggedNostrEvent } from "@snort/system"; +import { useRequestBuilder } from "@snort/system-react"; +import { useEffect, useMemo, useState } from "react"; + +import { Relay } from "@/Cache"; +import { SearchRelays } from "@/Utils/Const"; + +import PageSpinner from "../PageSpinner"; +import { TimelineFragment } from "./TimelineFragment"; + +export function LocalSearch({ term, kind }: { term: string; kind: EventKind }) { + const [frag, setFrag] = useState(); + + const r = useMemo(() => { + const rb = new RequestBuilder("search"); + rb.withFilter().search(term).kinds([kind]).relay(SearchRelays).limit(100); + return rb; + }, [term]); + useRequestBuilder(r); + + useEffect(() => { + setFrag(undefined); + if (term) { + Relay.req({ + id: "local-search", + filters: [ + { + kinds: [kind], + limit: 100, + search: term, + }, + ], + }).then(res => { + setFrag({ + refTime: 0, + events: res.result as Array, + }); + }); + } + }, [term, kind]); + + if (frag) { + return ; + } + return ; +} diff --git a/packages/app/src/Hooks/useProfileSearch.tsx b/packages/app/src/Hooks/useProfileSearch.tsx index 7fb9ac85..05629687 100644 --- a/packages/app/src/Hooks/useProfileSearch.tsx +++ b/packages/app/src/Hooks/useProfileSearch.tsx @@ -1,33 +1,9 @@ -import { unixNow } from "@snort/shared"; import { socialGraphInstance } from "@snort/system"; import { useMemo } from "react"; import fuzzySearch from "@/Db/FuzzySearch"; -import useTimelineFeed, { TimelineFeedOptions, TimelineSubject } from "@/Feed/TimelineFeed"; export default function useProfileSearch(search: string) { - const options: TimelineFeedOptions = useMemo( - () => ({ - method: "LIMIT_UNTIL", - window: undefined, - now: unixNow(), - }), - [], - ); - - const subject: TimelineSubject = useMemo( - () => ({ - type: "profile_keyword", - discriminator: search, - items: search ? [search] : [], - relay: undefined, - streams: false, - }), - [search], - ); - - const { main } = useTimelineFeed(subject, options); - const results = useMemo(() => { const searchString = search.trim(); const fuseResults = fuzzySearch.search(searchString); @@ -66,7 +42,7 @@ export default function useProfileSearch(search: string) { }); return combinedResults.map(r => r.item); - }, [search, main]); + }, [search]); return results; } diff --git a/packages/app/src/Pages/SearchPage.tsx b/packages/app/src/Pages/SearchPage.tsx index c261ca04..85946c5a 100644 --- a/packages/app/src/Pages/SearchPage.tsx +++ b/packages/app/src/Pages/SearchPage.tsx @@ -1,25 +1,15 @@ -import { useEffect, useMemo, useState } from "react"; +import { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate, useParams } from "react-router-dom"; -import Timeline from "@/Components/Feed/Timeline"; +import { LocalSearch } from "@/Components/Feed/LocalSearch"; import TabSelectors, { Tab } from "@/Components/TabSelectors/TabSelectors"; import TrendingNotes from "@/Components/Trending/TrendingPosts"; -import TrendingUsers from "@/Components/Trending/TrendingUsers"; -import FollowListBase from "@/Components/User/FollowListBase"; -import useProfileSearch from "@/Hooks/useProfileSearch"; import { debounce } from "@/Utils"; const NOTES = 0; const PROFILES = 1; -const Profiles = ({ keyword }: { keyword: string }) => { - const results = useProfileSearch(keyword); - const ids = useMemo(() => results.map(r => r.pubkey), [results]); - const content = keyword ? : ; - return
{content}
; -}; - const SearchPage = () => { const params = useParams(); const { formatMessage } = useIntl(); @@ -53,18 +43,9 @@ const SearchPage = () => { return debounce(500, () => setKeyword(search)); }, [search]); - const subject = useMemo( - () => ({ - type: "post_keyword", - items: [keyword + (sortPopular ? " sort:popular" : "")], - discriminator: keyword, - }), - [keyword, sortPopular], - ); - function tabContent() { if (tab.value === PROFILES) { - return ; + return ; } if (!keyword) { @@ -74,7 +55,7 @@ const SearchPage = () => { return ( <> {sortOptions()} - + ); } diff --git a/packages/worker-relay/src/relay.ts b/packages/worker-relay/src/sqlite-relay.ts similarity index 82% rename from packages/worker-relay/src/relay.ts rename to packages/worker-relay/src/sqlite-relay.ts index 0326da09..d1d60f06 100644 --- a/packages/worker-relay/src/relay.ts +++ b/packages/worker-relay/src/sqlite-relay.ts @@ -2,7 +2,7 @@ import sqlite3InitModule, { Database, Sqlite3Static } from "@sqlite.org/sqlite-w import { EventEmitter } from "eventemitter3"; import { NostrEvent, RelayHandler, RelayHandlerEvents, ReqFilter, unixNowMs } from "./types"; -export class WorkerRelay extends EventEmitter implements RelayHandler { +export class SqliteRelay extends EventEmitter implements RelayHandler { #sqlite?: Sqlite3Static; #log = (...args: any[]) => console.debug(...args); #db?: Database; @@ -67,6 +67,10 @@ export class WorkerRelay extends EventEmitter implements Rel this.#migrate_v2(); this.#log("Migrated to v2"); } + if (version < 3) { + this.#migrate_v3(); + this.#log("Migrated to v3"); + } } /** @@ -81,9 +85,6 @@ export class WorkerRelay extends EventEmitter implements Rel return false; } - /** - * Run any SQL command - */ sql(sql: string, params: Array) { return this.#db?.selectArrays(sql, params) as Array>; } @@ -94,11 +95,13 @@ export class WorkerRelay extends EventEmitter implements Rel eventBatch(evs: Array) { const start = unixNowMs(); let eventsInserted: Array = []; - for (const ev of evs) { - if (this.#insertEvent(this.#db!, ev)) { - eventsInserted.push(ev); + this.#db?.transaction(db => { + for (const ev of evs) { + if (this.#insertEvent(db, ev)) { + eventsInserted.push(ev); + } } - } + }); if (eventsInserted.length > 0) { this.#log(`Inserted Batch: ${eventsInserted.length}/${evs.length}, ${(unixNowMs() - start).toLocaleString()}ms`); this.emit("event", eventsInserted); @@ -154,13 +157,12 @@ export class WorkerRelay extends EventEmitter implements Rel }); let eventInserted = (this.#db?.changes() as number) > 0; if (eventInserted) { - db.transaction(db => { - for (const t of ev.tags.filter(a => a[0].length === 1)) { - db.exec("insert into tags(event_id, key, value) values(?, ?, ?)", { - bind: [ev.id, t[0], t[1]], - }); - } - }); + for (const t of ev.tags.filter(a => a[0].length === 1)) { + db.exec("insert into tags(event_id, key, value) values(?, ?, ?)", { + bind: [ev.id, t[0], t[1]], + }); + } + this.#insertSearchIndex(db, ev); } this.#seenInserts.add(ev.id); return eventInserted; @@ -245,6 +247,11 @@ export class WorkerRelay extends EventEmitter implements Rel params.push(key.slice(1)); params.push(...vArray); } + if (req.search) { + sql += " inner join search_content on search_content.id = events.id"; + conditions.push("search_content match ?"); + params.push(req.search.replaceAll(".", "+").replaceAll("@", "+")); + } if (req.ids) { conditions.push(`id in (${this.#repeatParams(req.ids.length)})`); params.push(...req.ids); @@ -335,4 +342,50 @@ export class WorkerRelay extends EventEmitter implements Rel }); }); } + + #insertSearchIndex(db: Database, ev: NostrEvent) { + if (ev.kind === 0) { + const profile = JSON.parse(ev.content) as { + name?: string; + display_name?: string; + lud16?: string; + nip05?: string; + website?: string; + about?: string; + }; + if (profile) { + const indexContent = [ + profile.name, + profile.display_name, + profile.about, + profile.website, + profile.lud16, + profile.nip05, + ].join(" "); + db.exec("insert into search_content values(?,?)", { + bind: [ev.id, indexContent], + }); + } + } else if (ev.kind === 1) { + db.exec("insert into search_content values(?,?)", { + bind: [ev.id, ev.content], + }); + } + } + + #migrate_v3() { + this.#db?.transaction(db => { + db.exec("CREATE VIRTUAL TABLE search_content using fts5(id UNINDEXED, content)"); + const events = db.selectArrays("select json from events where kind in (?,?)", [0, 1]); + for (const json of events) { + const ev = JSON.parse(json[0] as string) as NostrEvent; + if (ev) { + this.#insertSearchIndex(db, ev); + } + } + db.exec("insert into __migration values(3, ?)", { + bind: [new Date().getTime() / 1000], + }); + }); + } } diff --git a/packages/worker-relay/src/types.ts b/packages/worker-relay/src/types.ts index bdb168fb..975a5323 100644 --- a/packages/worker-relay/src/types.ts +++ b/packages/worker-relay/src/types.ts @@ -50,6 +50,10 @@ export interface RelayHandler extends EventEmitter { close(): void; event(ev: NostrEvent): boolean; eventBatch(evs: Array): boolean; + + /** + * Run any SQL command + */ sql(sql: string, params: Array): Array>; req(id: string, req: ReqFilter): Array; count(req: ReqFilter): number; diff --git a/packages/worker-relay/src/worker.ts b/packages/worker-relay/src/worker.ts index ad2e0a33..4019c226 100644 --- a/packages/worker-relay/src/worker.ts +++ b/packages/worker-relay/src/worker.ts @@ -2,8 +2,8 @@ import { InMemoryRelay } from "./memory-relay"; import { WorkQueueItem, barrierQueue, processWorkQueue } from "./queue"; -import { WorkerRelay } from "./relay"; -import { NostrEvent, RelayHandler, ReqCommand, ReqFilter, WorkerMessage, eventMatchesFilter } from "./types"; +import { SqliteRelay } from "./sqlite-relay"; +import { NostrEvent, RelayHandler, ReqCommand, ReqFilter, WorkerMessage, eventMatchesFilter, unixNowMs } from "./types"; interface PortedFilter { filters: Array; @@ -33,10 +33,18 @@ async function insertBatch() { // This is to make req's execute first and not block them if (eventWriteQueue.length > 0 && cmdQueue.length === 0) { await barrierQueue(cmdQueue, async () => { + const start = unixNowMs(); + const timeLimit = 1000; if (relay) { - const toWrite = [...eventWriteQueue]; - eventWriteQueue = []; - relay.eventBatch(toWrite); + while (eventWriteQueue.length > 0) { + if (unixNowMs() - start >= timeLimit) { + console.debug("Yield insert, queue length: ", eventWriteQueue.length, ", cmds: ", cmdQueue.length); + break; + } + const batch = eventWriteQueue.splice(0, 10); + eventWriteQueue = eventWriteQueue.slice(batch.length); + relay.eventBatch(batch); + } } }); } @@ -68,7 +76,7 @@ globalThis.onmessage = async ev => { case "init": { await barrierQueue(cmdQueue, async () => { if ("WebAssembly" in globalThis && (await tryOpfs())) { - relay = new WorkerRelay(); + relay = new SqliteRelay(); } else { relay = new InMemoryRelay(); }