From 71be59b2e9f1f2cdfd114d9c6481f3402ad44714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E5=AE=AE=E8=93=AE?= <123083837+reyamir@users.noreply.github.com> Date: Wed, 12 Jun 2024 08:27:53 +0700 Subject: [PATCH] Move the event parser and dedup functions to Rust (#206) * feat: improve js parser * feat: move parser and dedup to rust * fix: parser * fix: get event function * feat: improve parser performance (#207) * feat: improve parser performance * feat: add test for video parsing * feat: finish new parser --------- Co-authored-by: XIAO YU --- .idea/.gitignore | 5 + .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/lume.iml | 11 ++ .idea/modules.xml | 8 + .idea/vcs.xml | 6 + apps/desktop2/src/components/note/content.tsx | 93 ++++----- .../src/components/note/contentLarge.tsx | 183 ++++++------------ apps/desktop2/src/components/repost.tsx | 19 +- biome.json | 6 +- packages/system/src/commands.ts | 14 +- packages/system/src/event.ts | 19 +- packages/system/src/hooks/useEvent.ts | 10 +- packages/system/src/query.ts | 83 ++++++-- packages/types/index.d.ts | 10 + packages/utils/src/parser.ts | 57 +++++- src-tauri/Cargo.lock | 13 ++ src-tauri/Cargo.toml | 14 +- src-tauri/capabilities/main.json | 1 + src-tauri/gen/schemas/capabilities.json | 2 +- src-tauri/src/nostr/event.rs | 120 ++++++++++-- src-tauri/src/nostr/utils.rs | 181 ++++++++++++++++- 21 files changed, 610 insertions(+), 250 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/lume.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..b58b603f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 00000000..a55e7a17 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/lume.iml b/.idea/lume.iml new file mode 100644 index 00000000..53020506 --- /dev/null +++ b/.idea/lume.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..85ca919d --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/apps/desktop2/src/components/note/content.tsx b/apps/desktop2/src/components/note/content.tsx index d0cecc67..45ed1e9a 100644 --- a/apps/desktop2/src/components/note/content.tsx +++ b/apps/desktop2/src/components/note/content.tsx @@ -1,4 +1,4 @@ -import { NOSTR_EVENTS, NOSTR_MENTIONS, cn, parser } from "@lume/utils"; +import { cn } from "@lume/utils"; import { type ReactNode, useMemo } from "react"; import reactStringReplace from "react-string-replace"; import { Hashtag } from "./mentions/hashtag"; @@ -21,54 +21,42 @@ export function NoteContent({ className?: string; }) { const event = useNoteContext(); - const data = useMemo(() => { - const { content, images, videos } = parser(event.content); - const words = content.split(/( |\n)/); - const hashtags = words.filter((word) => word.startsWith("#")); - const events = words.filter((word) => - NOSTR_EVENTS.some((el) => word.startsWith(el)), - ); - const mentions = words.filter((word) => - NOSTR_MENTIONS.some((el) => word.startsWith(el)), - ); - - let richContent: ReactNode[] | string = content; - + const content = useMemo(() => { try { - if (hashtags.length) { - for (const hashtag of hashtags) { - const regex = new RegExp(`(|^)${hashtag}\\b`, "g"); - richContent = reactStringReplace(richContent, regex, (_, index) => { - return ; - }); + // Get parsed meta + const { content, hashtags, events, mentions } = event.meta; + + // Define rich content + let richContent: ReactNode[] | string = content; + + for (const hashtag of hashtags) { + const regex = new RegExp(`(|^)${hashtag}\\b`, "g"); + richContent = reactStringReplace(richContent, regex, (_, index) => { + return ; + }); + } + + for (const event of events) { + if (quote) { + richContent = reactStringReplace(richContent, event, (_, index) => ( + + )); + } + + if (!quote && clean) { + richContent = reactStringReplace(richContent, event, () => null); } } - if (events.length) { - for (const event of events) { - if (quote) { - richContent = reactStringReplace(richContent, event, (_, index) => ( - - )); - } - - if (!quote && clean) { - richContent = reactStringReplace(richContent, event, () => null); - } + for (const user of mentions) { + if (mention) { + richContent = reactStringReplace(richContent, user, (_, index) => ( + + )); } - } - if (mentions.length) { - for (const user of mentions) { - if (mention) { - richContent = reactStringReplace(richContent, user, (_, index) => ( - - )); - } - - if (!mention && clean) { - richContent = reactStringReplace(richContent, user, () => null); - } + if (!mention && clean) { + richContent = reactStringReplace(richContent, user, () => null); } } @@ -81,7 +69,7 @@ export function NoteContent({ href={match} target="_blank" rel="noreferrer" - className="line-clamp-1 text-blue-500 hover:text-blue-600" + className="text-blue-500 line-clamp-1 hover:text-blue-600" > {match} @@ -92,25 +80,26 @@ export function NoteContent({
)); - return { content: richContent, images, videos }; + return richContent; } catch (e) { - return { content, images, videos }; + console.log("[parser]: ", e); + return event.content; } - }, []); + }, [event.content]); return (
500 ? "max-h-[300px] gradient-mask-b-0" : "", + "select-text text-pretty content-break overflow-hidden", + event.content.length > 420 ? "max-h-[250px] gradient-mask-b-0" : "", className, )} > - {data.content} + {content}
- {data.images.length ? : null} - {data.videos.length ? : null} + {event.meta?.images.length ? : null} + {event.meta?.videos.length ? : null}
); } diff --git a/apps/desktop2/src/components/note/contentLarge.tsx b/apps/desktop2/src/components/note/contentLarge.tsx index c8614470..7e88923b 100644 --- a/apps/desktop2/src/components/note/contentLarge.tsx +++ b/apps/desktop2/src/components/note/contentLarge.tsx @@ -1,13 +1,4 @@ -import type { Settings } from "@lume/types"; -import { - AUDIOS, - IMAGES, - NOSTR_EVENTS, - NOSTR_MENTIONS, - VIDEOS, - cn, -} from "@lume/utils"; -import { useRouteContext } from "@tanstack/react-router"; +import { cn } from "@lume/utils"; import { nanoid } from "nanoid"; import { type ReactNode, useMemo } from "react"; import reactStringReplace from "react-string-replace"; @@ -19,135 +10,85 @@ import { VideoPreview } from "./preview/video"; import { useNoteContext } from "./provider"; export function NoteContentLarge({ - compact = true, className, }: { - compact?: boolean; className?: string; }) { - const { settings }: { settings: Settings } = useRouteContext({ - strict: false, - }); const event = useNoteContext(); const content = useMemo(() => { - const text = event.content.trim(); - const words = text.split(/( |\n)/); - - // @ts-ignore, kaboom !!! - let parsedContent: ReactNode[] = compact - ? text.replace(/\n\s*\n/g, "\n") - : text; - - const hashtags = words.filter((word) => word.startsWith("#")); - const events = words.filter((word) => - NOSTR_EVENTS.some((el) => word.startsWith(el)), - ); - const mentions = words.filter((word) => - NOSTR_MENTIONS.some((el) => word.startsWith(el)), - ); - try { - if (hashtags.length) { - for (const hashtag of hashtags) { - const regex = new RegExp(`(|^)${hashtag}\\b`, "g"); - parsedContent = reactStringReplace(parsedContent, regex, () => { - return ; - }); - } - } + // Get parsed meta + const { images, videos, hashtags, events, mentions } = event.meta; - if (events.length) { - for (const event of events) { - parsedContent = reactStringReplace( - parsedContent, - event, - (match, i) => , - ); - } - } + // Define rich content + let richContent: ReactNode[] | string = event.content; - if (mentions.length) { - for (const mention of mentions) { - parsedContent = reactStringReplace( - parsedContent, - mention, - (match, i) => , - ); - } - } - - parsedContent = reactStringReplace( - parsedContent, - /(https?:\/\/\S+)/gi, - (match, i) => { - try { - const url = new URL(match); - const ext = url.pathname.split(".")[1]; - - if (!settings.enhancedPrivacy) { - if (IMAGES.includes(ext)) { - return ; - } - - if (VIDEOS.includes(ext)) { - return ; - } - - if (AUDIOS.includes(ext)) { - return ; - } - } - - return ( - - {match} - - ); - } catch { - return ( - - {match} - - ); - } - }, - ); - - if (compact) { - parsedContent = reactStringReplace(parsedContent, /\n*\n/g, () => ( -
+ for (const hashtag of hashtags) { + const regex = new RegExp(`(|^)${hashtag}\\b`, "g"); + richContent = reactStringReplace(richContent, regex, () => ( + )); } - parsedContent = reactStringReplace( - parsedContent, - /[\r]?\n[\r]?\n/g, - (_, index) =>
, + for (const event of events) { + richContent = reactStringReplace(richContent, event, (match, i) => ( + + )); + } + + for (const mention of mentions) { + richContent = reactStringReplace(richContent, mention, (match, i) => ( + + )); + } + + for (const image of images) { + richContent = reactStringReplace(richContent, image, (match, i) => ( + + )); + } + + for (const video of videos) { + richContent = reactStringReplace(richContent, video, (match, i) => ( + + )); + } + + richContent = reactStringReplace( + richContent, + /(https?:\/\/\S+)/gi, + (match, i) => ( + + {match} + + ), ); - return parsedContent; + richContent = reactStringReplace(richContent, /(\r\n|\r|\n)+/g, () => ( +
+ )); + + return richContent; } catch (e) { - return text; + console.log("[parser]: ", e); + return event.content; } - }, []); + }, [event.content]); return ( -
-
- {content} -
+
+ {content}
); } diff --git a/apps/desktop2/src/components/repost.tsx b/apps/desktop2/src/components/repost.tsx index f08bb662..b93d826c 100644 --- a/apps/desktop2/src/components/repost.tsx +++ b/apps/desktop2/src/components/repost.tsx @@ -3,7 +3,7 @@ import { Note } from "@/components/note"; import { User } from "@/components/user"; import { cn } from "@lume/utils"; import { useQuery } from "@tanstack/react-query"; -import { NostrEvent } from "@lume/types"; +import type { NostrEvent } from "@lume/types"; import { NostrQuery } from "@lume/system"; export function RepostNote({ @@ -21,12 +21,7 @@ export function RepostNote({ queryKey: ["repost", event.id], queryFn: async () => { try { - if (event.content.length > 50) { - const embed: NostrEvent = JSON.parse(event.content); - return embed; - } - - const id = event.tags.find((el) => el[0] === "e")?.[1]; + const id = event.tags.find((el) => el[0] === "e")[1]; const repostEvent = await NostrQuery.getEvent(id); return repostEvent; @@ -50,27 +45,27 @@ export function RepostNote({
Reposted by
- + {isLoading ? ( -
+
Loading event...
) : isError || !repostEvent ? ( -
+
Event not found within your current relay set
) : ( -
+
-
+
diff --git a/biome.json b/biome.json index 7a6d162e..d269b819 100644 --- a/biome.json +++ b/biome.json @@ -11,13 +11,17 @@ "rules": { "recommended": true, "style": { - "noNonNullAssertion": "warn" + "noNonNullAssertion": "warn", + "noUselessElse": "off" }, "correctness": { "useExhaustiveDependencies": "off" }, "a11y": { "noSvgWithoutTitle": "off" + }, + "complexity": { + "noStaticOnlyClass": "off" } } } diff --git a/packages/system/src/commands.ts b/packages/system/src/commands.ts index 84c537fa..ca23003e 100644 --- a/packages/system/src/commands.ts +++ b/packages/system/src/commands.ts @@ -244,7 +244,7 @@ try { else return { status: "error", error: e as any }; } }, -async getEvent(id: string) : Promise> { +async getEvent(id: string) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_event", { id }) }; } catch (e) { @@ -252,7 +252,7 @@ try { else return { status: "error", error: e as any }; } }, -async getReplies(id: string) : Promise> { +async getReplies(id: string) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_replies", { id }) }; } catch (e) { @@ -260,7 +260,7 @@ try { else return { status: "error", error: e as any }; } }, -async getEventsBy(publicKey: string, asOf: string | null) : Promise> { +async getEventsBy(publicKey: string, asOf: string | null) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_events_by", { publicKey, asOf }) }; } catch (e) { @@ -268,7 +268,7 @@ try { else return { status: "error", error: e as any }; } }, -async getLocalEvents(pubkeys: string[], until: string | null) : Promise> { +async getLocalEvents(pubkeys: string[], until: string | null) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_local_events", { pubkeys, until }) }; } catch (e) { @@ -276,7 +276,7 @@ try { else return { status: "error", error: e as any }; } }, -async getGlobalEvents(until: string | null) : Promise> { +async getGlobalEvents(until: string | null) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_global_events", { until }) }; } catch (e) { @@ -284,7 +284,7 @@ try { else return { status: "error", error: e as any }; } }, -async getHashtagEvents(hashtags: string[], until: string | null) : Promise> { +async getHashtagEvents(hashtags: string[], until: string | null) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("get_hashtag_events", { hashtags, until }) }; } catch (e) { @@ -367,7 +367,9 @@ await TAURI_INVOKE("set_badge", { count }); /** user-defined types **/ export type Account = { npub: string; nsec: string } +export type Meta = { content: string; images: string[]; videos: string[]; events: string[]; mentions: string[]; hashtags: string[] } export type Relays = { connected: string[]; read: string[] | null; write: string[] | null; both: string[] | null } +export type RichEvent = { raw: string; parsed: Meta | null } /** tauri-specta globals **/ diff --git a/packages/system/src/event.ts b/packages/system/src/event.ts index 1e27ba80..29c50c16 100644 --- a/packages/system/src/event.ts +++ b/packages/system/src/event.ts @@ -1,4 +1,4 @@ -import { EventWithReplies, Kind, NostrEvent } from "@lume/types"; +import type { EventWithReplies, Kind, Meta, NostrEvent } from "@lume/types"; import { commands } from "./commands"; import { generateContentTags } from "@lume/utils"; @@ -11,6 +11,7 @@ export class LumeEvent { public content: string; public sig: string; public relay?: string; + public meta: Meta; #raw: NostrEvent; constructor(event: NostrEvent) { @@ -74,9 +75,17 @@ export class LumeEvent { const query = await commands.getReplies(id); if (query.status === "ok") { - const events = query.data.map( - (item) => JSON.parse(item) as EventWithReplies, - ); + const events = query.data.map((item) => { + const raw = JSON.parse(item.raw) as EventWithReplies; + + if (item.parsed) { + raw.meta = item.parsed; + } else { + raw.meta = null; + } + + return raw; + }); if (events.length > 0) { const replies = new Set(); @@ -135,7 +144,7 @@ export class LumeEvent { const queryReply = await commands.getEvent(reply_to); if (queryReply.status === "ok") { - const replyEvent = JSON.parse(queryReply.data) as NostrEvent; + const replyEvent = JSON.parse(queryReply.data.raw) as NostrEvent; const relayHint = replyEvent.tags.find((ev) => ev[0] === "e")?.[0][2] ?? ""; diff --git a/packages/system/src/hooks/useEvent.ts b/packages/system/src/hooks/useEvent.ts index 2362de7d..e0705031 100644 --- a/packages/system/src/hooks/useEvent.ts +++ b/packages/system/src/hooks/useEvent.ts @@ -1,18 +1,12 @@ -import type { Event, NostrEvent } from "@lume/types"; import { useQuery } from "@tanstack/react-query"; -import { invoke } from "@tauri-apps/api/core"; +import { NostrQuery } from "../query"; export function useEvent(id: string) { const { isLoading, isError, data } = useQuery({ queryKey: ["event", id], queryFn: async () => { try { - const eventId: string = id - .replace("nostr:", "") - .split("'")[0] - .split(".")[0]; - const cmd: string = await invoke("get_event", { id: eventId }); - const event: NostrEvent = JSON.parse(cmd); + const event = await NostrQuery.getEvent(id); return event; } catch (e) { throw new Error(e); diff --git a/packages/system/src/query.ts b/packages/system/src/query.ts index dd2adc9d..a6a8c997 100644 --- a/packages/system/src/query.ts +++ b/packages/system/src/query.ts @@ -1,10 +1,15 @@ -import { LumeColumn, Metadata, NostrEvent, Relay, Settings } from "@lume/types"; +import type { + LumeColumn, + Metadata, + NostrEvent, + Relay, + Settings, +} from "@lume/types"; import { commands } from "./commands"; import { resolveResource } from "@tauri-apps/api/path"; import { readFile, readTextFile } from "@tauri-apps/plugin-fs"; import { isPermissionGranted } from "@tauri-apps/plugin-notification"; import { open } from "@tauri-apps/plugin-dialog"; -import { dedupEvents } from "./dedup"; import { invoke } from "@tauri-apps/api/core"; import { relaunch } from "@tauri-apps/plugin-process"; @@ -98,9 +103,16 @@ export class NostrQuery { const query = await commands.getEvent(normalize); if (query.status === "ok") { - const event: NostrEvent = JSON.parse(query.data); - return event; + const data = query.data; + const raw = JSON.parse(data.raw) as NostrEvent; + + if (data?.parsed) { + raw.meta = data.parsed; + } + + return raw; } else { + console.log("[getEvent]: ", query.error); return null; } } @@ -110,8 +122,19 @@ export class NostrQuery { const query = await commands.getEventsBy(pubkey, until); if (query.status === "ok") { - const events = query.data.map((item) => JSON.parse(item) as NostrEvent); - return events; + const data = query.data.map((item) => { + const raw = JSON.parse(item.raw) as NostrEvent; + + if (item.parsed) { + raw.meta = item.parsed; + } else { + raw.meta = null; + } + + return raw; + }); + + return data; } else { return []; } @@ -122,10 +145,19 @@ export class NostrQuery { const query = await commands.getLocalEvents(pubkeys, until); if (query.status === "ok") { - const events = query.data.map((item) => JSON.parse(item) as NostrEvent); - const dedup = dedupEvents(events); + const data = query.data.map((item) => { + const raw = JSON.parse(item.raw) as NostrEvent; - return dedup; + if (item.parsed) { + raw.meta = item.parsed; + } else { + raw.meta = null; + } + + return raw; + }); + + return data; } else { return []; } @@ -136,10 +168,19 @@ export class NostrQuery { const query = await commands.getGlobalEvents(until); if (query.status === "ok") { - const events = query.data.map((item) => JSON.parse(item) as NostrEvent); - const dedup = dedupEvents(events); + const data = query.data.map((item) => { + const raw = JSON.parse(item.raw) as NostrEvent; - return dedup; + if (item.parsed) { + raw.meta = item.parsed; + } else { + raw.meta = null; + } + + return raw; + }); + + return data; } else { return []; } @@ -151,10 +192,19 @@ export class NostrQuery { const query = await commands.getHashtagEvents(nostrTags, until); if (query.status === "ok") { - const events = query.data.map((item) => JSON.parse(item) as NostrEvent); - const dedup = dedupEvents(events); + const data = query.data.map((item) => { + const raw = JSON.parse(item.raw) as NostrEvent; - return dedup; + if (item.parsed) { + raw.meta = item.parsed; + } else { + raw.meta = null; + } + + return raw; + }); + + return data; } else { return []; } @@ -311,8 +361,7 @@ export class NostrQuery { const query = await commands.getBootstrapRelays(); if (query.status === "ok") { - let relays: Relay[] = []; - console.log(query.data); + const relays: Relay[] = []; for (const item of query.data) { const line = item.split(","); diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts index bdbc0264..44d623c8 100644 --- a/packages/types/index.d.ts +++ b/packages/types/index.d.ts @@ -28,6 +28,15 @@ export enum Kind { // #TODO: Add all nostr kinds } +export interface Meta { + content: string; + images: string[]; + videos: string[]; + events: string[]; + mentions: string[]; + hashtags: string[]; +} + export interface NostrEvent { id: string; pubkey: string; @@ -36,6 +45,7 @@ export interface NostrEvent { tags: string[][]; content: string; sig: string; + meta: Meta; } export interface EventWithReplies extends NostrEvent { diff --git a/packages/utils/src/parser.ts b/packages/utils/src/parser.ts index 4c81768e..bfe3c9a1 100644 --- a/packages/utils/src/parser.ts +++ b/packages/utils/src/parser.ts @@ -1,12 +1,31 @@ -import { IMAGES, VIDEOS } from "./constants"; +import { Meta } from "@lume/types"; +import { IMAGES, NOSTR_EVENTS, NOSTR_MENTIONS, VIDEOS } from "./constants"; +import { fetch } from "@tauri-apps/plugin-http"; -export function parser(content: string) { - // Get clean content +export async function parser( + content: string, + abortController?: AbortController, +) { + const words = content.split(/( |\n)/); const urls = content.match(/(https?:\/\/\S+)/gi); + // Extract hashtags + const hashtags = words.filter((word) => word.startsWith("#")); + + // Extract nostr events + const events = words.filter((word) => + NOSTR_EVENTS.some((el) => word.startsWith(el)), + ); + + // Extract nostr mentions + const mentions = words.filter((word) => + NOSTR_MENTIONS.some((el) => word.startsWith(el)), + ); + // Extract images and videos from content const images: string[] = []; const videos: string[] = []; + let text: string = content; if (urls) { @@ -16,20 +35,44 @@ export function parser(content: string) { if (IMAGES.includes(ext)) { text = text.replace(url, ""); images.push(url); + break; } if (VIDEOS.includes(ext)) { text = text.replace(url, ""); videos.push(url); + break; + } + + if (urls.length <= 3) { + try { + const res = await fetch(url, { + method: "HEAD", + priority: "high", + signal: abortController.signal, + // proxy: settings.proxy; + }); + + if (res.headers.get("Content-Type").startsWith("image")) { + text = text.replace(url, ""); + images.push(url); + break; + } + } catch { + break; + } } } } - const trimContent = text.trim(); - - return { - content: trimContent, + const meta: Meta = { + content: text.trim(), images, videos, + events, + mentions, + hashtags, }; + + return meta; } diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 61ffeedd..8f294b85 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2707,6 +2707,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e" +[[package]] +name = "linkify" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dfa36d52c581e9ec783a7ce2a5e0143da6237be5811a0b3153fedfdbe9f780" +dependencies = [ + "memchr", +] + [[package]] name = "linux-keyutils" version = "0.2.4" @@ -2786,12 +2795,15 @@ name = "lume" version = "4.0.0" dependencies = [ "cocoa", + "futures", "keyring", "keyring-search", + "linkify", "monitor", "nostr-sdk", "objc", "rand 0.8.5", + "reqwest", "serde", "serde_json", "specta", @@ -2812,6 +2824,7 @@ dependencies = [ "tauri-plugin-upload", "tauri-specta", "tokio", + "url", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 2ce3233e..04783e61 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -17,11 +17,11 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } monitor = { git = "https://github.com/ahkohd/tauri-toolkit", branch = "v2" } tauri = { version = "2.0.0-beta", features = [ - "unstable", - "tray-icon", - "macos-private-api", - "native-tls-vendored", - "protocol-asset", + "unstable", + "tray-icon", + "macos-private-api", + "native-tls-vendored", + "protocol-asset", ] } tauri-plugin-clipboard-manager = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v2" } tauri-plugin-dialog = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v2" } @@ -40,6 +40,10 @@ tauri-plugin-decorum = "0.1.0" specta = "^2.0.0-rc.12" keyring = "2" keyring-search = "0.2.0" +reqwest = "0.12.4" +url = "2.5.0" +futures = "0.3.30" +linkify = "0.10.0" [target.'cfg(target_os = "macos")'.dependencies] cocoa = "0.25.0" diff --git a/src-tauri/capabilities/main.json b/src-tauri/capabilities/main.json index 69e83613..34462be5 100644 --- a/src-tauri/capabilities/main.json +++ b/src-tauri/capabilities/main.json @@ -59,6 +59,7 @@ "fs:allow-read-file", "theme:allow-set-theme", "theme:allow-get-theme", + "http:default", "shell:allow-open", { "identifier": "http:default", diff --git a/src-tauri/gen/schemas/capabilities.json b/src-tauri/gen/schemas/capabilities.json index 4875d4ba..d74af3be 100644 --- a/src-tauri/gen/schemas/capabilities.json +++ b/src-tauri/gen/schemas/capabilities.json @@ -1 +1 @@ -{"desktop-capability":{"identifier":"desktop-capability","description":"Capability for the desktop","local":true,"windows":["main","panel","splash","settings","search","nwc","activity","zap-*","event-*","user-*","editor-*","column-*"],"permissions":["path:default","event:default","window:default","app:default","resources:default","menu:default","tray:default","notification:allow-is-permission-granted","notification:allow-request-permission","notification:default","os:allow-locale","os:allow-platform","os:allow-os-type","updater:default","updater:allow-check","updater:allow-download-and-install","window:allow-start-dragging","window:allow-create","window:allow-close","window:allow-set-focus","window:allow-center","window:allow-minimize","window:allow-maximize","window:allow-set-size","window:allow-set-focus","window:allow-start-dragging","decorum:allow-show-snap-overlay","clipboard-manager:allow-write-text","clipboard-manager:allow-read-text","webview:allow-create-webview-window","webview:allow-create-webview","webview:allow-set-webview-size","webview:allow-set-webview-position","webview:allow-webview-close","dialog:allow-open","dialog:allow-ask","dialog:allow-message","process:allow-restart","fs:allow-read-file","theme:allow-set-theme","theme:allow-get-theme","shell:allow-open",{"identifier":"http:default","allow":[{"url":"http://**/"},{"url":"https://**/"}]},{"identifier":"fs:allow-read-text-file","allow":[{"path":"$RESOURCE/locales/*"},{"path":"$RESOURCE/resources/*"}]}],"platforms":["linux","macOS","windows"]}} \ No newline at end of file +{"desktop-capability":{"identifier":"desktop-capability","description":"Capability for the desktop","local":true,"windows":["main","panel","splash","settings","search","nwc","activity","zap-*","event-*","user-*","editor-*","column-*"],"permissions":["path:default","event:default","window:default","app:default","resources:default","menu:default","tray:default","notification:allow-is-permission-granted","notification:allow-request-permission","notification:default","os:allow-locale","os:allow-platform","os:allow-os-type","updater:default","updater:allow-check","updater:allow-download-and-install","window:allow-start-dragging","window:allow-create","window:allow-close","window:allow-set-focus","window:allow-center","window:allow-minimize","window:allow-maximize","window:allow-set-size","window:allow-set-focus","window:allow-start-dragging","decorum:allow-show-snap-overlay","clipboard-manager:allow-write-text","clipboard-manager:allow-read-text","webview:allow-create-webview-window","webview:allow-create-webview","webview:allow-set-webview-size","webview:allow-set-webview-position","webview:allow-webview-close","dialog:allow-open","dialog:allow-ask","dialog:allow-message","process:allow-restart","fs:allow-read-file","theme:allow-set-theme","theme:allow-get-theme","http:default","shell:allow-open",{"identifier":"http:default","allow":[{"url":"http://**/"},{"url":"https://**/"}]},{"identifier":"fs:allow-read-text-file","allow":[{"path":"$RESOURCE/locales/*"},{"path":"$RESOURCE/resources/*"}]}],"platforms":["linux","macOS","windows"]}} \ No newline at end of file diff --git a/src-tauri/src/nostr/event.rs b/src-tauri/src/nostr/event.rs index 969d3fb3..e349531e 100644 --- a/src-tauri/src/nostr/event.rs +++ b/src-tauri/src/nostr/event.rs @@ -1,11 +1,23 @@ -use crate::Nostr; -use nostr_sdk::prelude::*; use std::{str::FromStr, time::Duration}; + +use futures::future::join_all; +use nostr_sdk::prelude::*; +use serde::Serialize; +use specta::Type; use tauri::State; +use crate::Nostr; +use crate::nostr::utils::{dedup_event, Meta, parse_event}; + +#[derive(Debug, Serialize, Type)] +pub struct RichEvent { + pub raw: String, + pub parsed: Option, +} + #[tauri::command] #[specta::specta] -pub async fn get_event(id: &str, state: State<'_, Nostr>) -> Result { +pub async fn get_event(id: &str, state: State<'_, Nostr>) -> Result { let client = &state.client; let event_id: Option = match Nip19::from_bech32(id) { Ok(val) => match val { @@ -36,7 +48,14 @@ pub async fn get_event(id: &str, state: State<'_, Nostr>) -> Result { if let Some(event) = events.first() { - Ok(event.as_json()) + let raw = event.as_json(); + let parsed = if event.kind == Kind::TextNote { + Some(parse_event(&event.content).await) + } else { + None + }; + + Ok(RichEvent { raw, parsed }) } else { Err("Cannot found this event with current relay list".into()) } @@ -50,7 +69,7 @@ pub async fn get_event(id: &str, state: State<'_, Nostr>) -> Result) -> Result, String> { +pub async fn get_replies(id: &str, state: State<'_, Nostr>) -> Result, String> { let client = &state.client; match EventId::from_hex(id) { @@ -58,7 +77,21 @@ pub async fn get_replies(id: &str, state: State<'_, Nostr>) -> Result Ok(events.into_iter().map(|ev| ev.as_json()).collect()), + Ok(events) => { + let futures = events.into_iter().map(|ev| async move { + let raw = ev.as_json(); + let parsed = if ev.kind == Kind::TextNote { + Some(parse_event(&ev.content).await) + } else { + None + }; + + RichEvent { raw, parsed } + }); + let rich_events = join_all(futures).await; + + Ok(rich_events) + } Err(err) => Err(err.to_string()), } } @@ -72,7 +105,7 @@ pub async fn get_events_by( public_key: &str, as_of: Option<&str>, state: State<'_, Nostr>, -) -> Result, String> { +) -> Result, String> { let client = &state.client; match PublicKey::from_str(public_key) { @@ -88,7 +121,21 @@ pub async fn get_events_by( .until(until); match client.get_events_of(vec![filter], None).await { - Ok(events) => Ok(events.into_iter().map(|ev| ev.as_json()).collect()), + Ok(events) => { + let futures = events.into_iter().map(|ev| async move { + let raw = ev.as_json(); + let parsed = if ev.kind == Kind::TextNote { + Some(parse_event(&ev.content).await) + } else { + None + }; + + RichEvent { raw, parsed } + }); + let rich_events = join_all(futures).await; + + Ok(rich_events) + } Err(err) => Err(err.to_string()), } } @@ -102,7 +149,7 @@ pub async fn get_local_events( pubkeys: Vec, until: Option<&str>, state: State<'_, Nostr>, -) -> Result, String> { +) -> Result, String> { let client = &state.client; let as_of = match until { Some(until) => Timestamp::from_str(until).unwrap(), @@ -128,7 +175,22 @@ pub async fn get_local_events( .get_events_of(vec![filter], Some(Duration::from_secs(10))) .await { - Ok(events) => Ok(events.into_iter().map(|ev| ev.as_json()).collect()), + Ok(events) => { + let dedup = dedup_event(&events, false); + let futures = dedup.into_iter().map(|ev| async move { + let raw = ev.as_json(); + let parsed = if ev.kind == Kind::TextNote { + Some(parse_event(&ev.content).await) + } else { + None + }; + + RichEvent { raw, parsed } + }); + let rich_events = join_all(futures).await; + + Ok(rich_events) + } Err(err) => Err(err.to_string()), } } @@ -138,7 +200,7 @@ pub async fn get_local_events( pub async fn get_global_events( until: Option<&str>, state: State<'_, Nostr>, -) -> Result, String> { +) -> Result, String> { let client = &state.client; let as_of = match until { Some(until) => Timestamp::from_str(until).unwrap(), @@ -154,7 +216,22 @@ pub async fn get_global_events( .get_events_of(vec![filter], Some(Duration::from_secs(8))) .await { - Ok(events) => Ok(events.into_iter().map(|ev| ev.as_json()).collect()), + Ok(events) => { + let dedup = dedup_event(&events, false); + let futures = dedup.into_iter().map(|ev| async move { + let raw = ev.as_json(); + let parsed = if ev.kind == Kind::TextNote { + Some(parse_event(&ev.content).await) + } else { + None + }; + + RichEvent { raw, parsed } + }); + let rich_events = join_all(futures).await; + + Ok(rich_events) + } Err(err) => Err(err.to_string()), } } @@ -165,7 +242,7 @@ pub async fn get_hashtag_events( hashtags: Vec<&str>, until: Option<&str>, state: State<'_, Nostr>, -) -> Result, String> { +) -> Result, String> { let client = &state.client; let as_of = match until { Some(until) => Timestamp::from_str(until).unwrap(), @@ -178,7 +255,22 @@ pub async fn get_hashtag_events( .hashtags(hashtags); match client.get_events_of(vec![filter], None).await { - Ok(events) => Ok(events.into_iter().map(|ev| ev.as_json()).collect()), + Ok(events) => { + let dedup = dedup_event(&events, false); + let futures = dedup.into_iter().map(|ev| async move { + let raw = ev.as_json(); + let parsed = if ev.kind == Kind::TextNote { + Some(parse_event(&ev.content).await) + } else { + None + }; + + RichEvent { raw, parsed } + }); + let rich_events = join_all(futures).await; + + Ok(rich_events) + } Err(err) => Err(err.to_string()), } } diff --git a/src-tauri/src/nostr/utils.rs b/src-tauri/src/nostr/utils.rs index e5e18fa3..7e88f248 100644 --- a/src-tauri/src/nostr/utils.rs +++ b/src-tauri/src/nostr/utils.rs @@ -1,5 +1,184 @@ -use nostr_sdk::Event; +use std::collections::HashSet; +use std::str::FromStr; + +use linkify::LinkFinder; +use nostr_sdk::{Alphabet, Event, SingleLetterTag, Tag, TagKind}; +use reqwest::Client; +use serde::Serialize; +use specta::Type; +use url::Url; + +#[derive(Debug, Serialize, Type)] +pub struct Meta { + pub content: String, + pub images: Vec, + pub videos: Vec, + pub events: Vec, + pub mentions: Vec, + pub hashtags: Vec, +} + +const NOSTR_EVENTS: [&str; 10] = [ + "@nevent1", + "@note1", + "@nostr:note1", + "@nostr:nevent1", + "nostr:note1", + "note1", + "nostr:nevent1", + "nevent1", + "Nostr:note1", + "Nostr:nevent1", +]; +const NOSTR_MENTIONS: [&str; 10] = [ + "@npub1", + "nostr:npub1", + "nostr:nprofile1", + "nostr:naddr1", + "npub1", + "nprofile1", + "naddr1", + "Nostr:npub1", + "Nostr:nprofile1", + "Nostr:naddr1", +]; +const IMAGES: [&str; 7] = ["jpg", "jpeg", "gif", "png", "webp", "avif", "tiff"]; +const VIDEOS: [&str; 5] = ["mp4", "mov", "avi", "webm", "mkv"]; pub fn get_latest_event(events: &[Event]) -> Option<&Event> { events.iter().max_by_key(|event| event.created_at()) } + +pub fn dedup_event(events: &[Event], nsfw: bool) -> Vec { + let mut seen_ids = HashSet::new(); + events + .iter() + .filter(|&event| { + let e = TagKind::SingleLetter(SingleLetterTag::lowercase(Alphabet::E)); + let e_tags: Vec<&Tag> = event.tags.iter().filter(|el| el.kind() == e).collect(); + let ids: Vec<&str> = e_tags.iter().filter_map(|tag| tag.content()).collect(); + let is_dup = ids.iter().any(|id| seen_ids.contains(*id)); + + for id in &ids { + seen_ids.insert(*id); + } + + if nsfw { + let w_tags: Vec<&Tag> = event + .tags + .iter() + .filter(|el| el.kind() == TagKind::ContentWarning) + .collect(); + !is_dup && w_tags.is_empty() + } else { + !is_dup + } + }) + .cloned() + .collect() +} + +pub async fn parse_event(content: &str) -> Meta { + let words: Vec<_> = content.split_whitespace().collect(); + let mut finder = LinkFinder::new(); + finder.url_must_have_scheme(false); + let urls: Vec<_> = finder.links(content).collect(); + + let hashtags = words + .iter() + .filter(|&&word| word.starts_with('#')) + .map(|&s| s.to_string()) + .collect::>(); + let events = words + .iter() + .filter(|&&word| NOSTR_EVENTS.iter().any(|&el| word.starts_with(el))) + .map(|&s| s.to_string()) + .collect::>(); + let mentions = words + .iter() + .filter(|&&word| NOSTR_MENTIONS.iter().any(|&el| word.starts_with(el))) + .map(|&s| s.to_string()) + .collect::>(); + + let mut images = Vec::new(); + let mut videos = Vec::new(); + let mut text = content.to_string(); + + if !urls.is_empty() { + let client = Client::new(); + + for url in urls { + let url_str = url.as_str(); + + if let Ok(parsed_url) = Url::from_str(url_str) { + if let Some(ext) = parsed_url + .path_segments() + .and_then(|segments| segments.last().and_then(|s| s.split('.').last())) + { + if IMAGES.contains(&ext) { + text = text.replace(url_str, ""); + images.push(url_str.to_string()); + break; + } + if VIDEOS.contains(&ext) { + text = text.replace(url_str, ""); + videos.push(url_str.to_string()); + break; + } + } + + // Check the content type of URL via HEAD request + if let Ok(res) = client.head(url_str).send().await { + if let Some(content_type) = res.headers().get("Content-Type") { + if content_type.to_str().unwrap_or("").starts_with("image") { + text = text.replace(url_str, ""); + images.push(url_str.to_string()); + break; + } + } + } + } + } + } + + // Clean up the resulting content string to remove extra spaces + let cleaned_text = text.trim().to_string(); + + Meta { + content: cleaned_text, + events, + mentions, + hashtags, + images, + videos, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_event() { + let content = "Check this image: https://example.com/image.jpg #cool @npub1"; + let meta = parse_event(content).await; + + assert_eq!(meta.content, "Check this image: #cool @npub1"); + assert_eq!(meta.images, vec!["https://example.com/image.jpg"]); + assert_eq!(meta.videos, Vec::::new()); + assert_eq!(meta.hashtags, vec!["#cool"]); + assert_eq!(meta.mentions, vec!["@npub1"]); + } + + #[tokio::test] + async fn test_parse_video() { + let content = "Check this video: https://example.com/video.mp4 #cool @npub1"; + let meta = parse_event(content).await; + + assert_eq!(meta.content, "Check this video: #cool @npub1"); + assert_eq!(meta.images, Vec::::new()); + assert_eq!(meta.videos, vec!["https://example.com/video.mp4"]); + assert_eq!(meta.hashtags, vec!["#cool"]); + assert_eq!(meta.mentions, vec!["@npub1"]); + } +}