Optimize Parse Event Content (#83)

Co-authored-by: Foodstr <foodstr@proton.me>
This commit is contained in:
BlowaterNostr 2023-07-16 23:04:23 +08:00 committed by GitHub
parent 87d6d68f9b
commit bd96687205
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 188 additions and 73 deletions

View File

@ -308,6 +308,7 @@ export function AppComponent(props: {
eventEmitter={app.eventBus}
profilesSyncer={app.profileSyncer}
eventSyncer={app.eventSyncer}
allUserInfo={app.allUsersInfo.userInfos}
/>
</div>
);

View File

@ -7,7 +7,7 @@ import * as csp from "https://raw.githubusercontent.com/BlowaterNostr/csp/master
import { Database_Contextual_View } from "../database.ts";
import { convertEventsToChatMessages } from "./dm.ts";
import { get_Kind4_Events_Between, sendDMandImages, sendSocialPost } from "../features/dm.ts";
import { sendDMandImages, sendSocialPost } from "../features/dm.ts";
import { notify } from "./notification.ts";
import { EventBus, EventEmitter } from "../event-bus.ts";
import { ContactUpdate } from "./contact-list.tsx";
@ -21,7 +21,6 @@ import { fromEvents, LamportTime } from "../time.ts";
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import {
NostrAccountContext,
NostrEvent,
NostrKind,
prepareCustomAppDataEvent,
} from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nostr.ts";

View File

@ -1,5 +1,10 @@
import { NostrEvent } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nostr.ts";
import { getTags, groupImageEvents, reassembleBase64ImageFromEvents } from "../nostr.ts";
import {
getTags,
groupImageEvents,
PlainText_Nostr_Event,
reassembleBase64ImageFromEvents,
} from "../nostr.ts";
import { ChatMessage } from "./message.ts";
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { ContactGroup } from "./contact-list.tsx";
@ -17,7 +22,7 @@ export type DM_Container_Model = {
};
export function convertEventsToChatMessages(
events: Iterable<NostrEvent>,
events: Iterable<PlainText_Nostr_Event>,
userProfiles: Map<string, UserInfo>,
): ChatMessage[] {
const messages: ChatMessage[] = [];

View File

@ -109,6 +109,7 @@ export function DirectMessageContainer(props: DirectMessageContainerProps) {
db: props.db,
profilesSyncer: props.profilesSyncer,
eventSyncer: props.eventSyncer,
allUserInfo: props.allUserInfo,
});
}

View File

@ -9,9 +9,10 @@ import { verifyEvent } from "https://raw.githubusercontent.com/BlowaterNostr/nos
export class EventSyncer {
constructor(private readonly pool: ConnectionPool, private readonly db: Database_Contextual_View) {}
syncEvent(id: NoteID) {
const iter = Array.from(this.db.filterEvents((e) => e.id == id.hex));
if (iter.length > 0) {
return iter[0];
for (const e of this.db.events) {
if (e.id == id.hex) {
return e;
}
}
return (async () => {
let events = await this.pool.newSub("EventSyncer", {

View File

@ -9,7 +9,7 @@ import { DividerClass, IconButtonClass } from "./components/tw.ts";
import { sleep } from "https://raw.githubusercontent.com/BlowaterNostr/csp/master/csp.ts";
import { EventEmitter } from "../event-bus.ts";
import { ChatMessage, groupContinuousMessages, parseContent, sortMessage, urlIsImage } from "./message.ts";
import { ChatMessage, groupContinuousMessages, sortMessage, urlIsImage } from "./message.ts";
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import {
NostrEvent,
@ -28,7 +28,7 @@ import { UserDetail } from "./user-detail.tsx";
import { MessageThreadPanel } from "./message-thread-panel.tsx";
import { Database_Contextual_View } from "../database.ts";
import { HoverButtonBackgroudColor, LinkColor, PrimaryTextColor } from "./style/colors.ts";
import { ProfilesSyncer } from "./contact-list.ts";
import { ProfilesSyncer, UserInfo } from "./contact-list.ts";
import { NoteID } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nip19.ts";
import { EventSyncer } from "./event_syncer.ts";
@ -55,6 +55,7 @@ interface DirectMessagePanelProps {
>;
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
allUserInfo: Map<string, UserInfo>;
}
export type RightPanelModel = {
@ -102,6 +103,7 @@ export function MessagePanel(props: DirectMessagePanelProps) {
editorModel={props.focusedContent.editor}
profilesSyncer={props.profilesSyncer}
eventSyncer={props.eventSyncer}
allUserInfo={props.allUserInfo}
/>
);
} else if (props.focusedContent.type == "ProfileData") {
@ -140,6 +142,7 @@ export function MessagePanel(props: DirectMessagePanelProps) {
db={props.db}
profilesSyncer={props.profilesSyncer}
eventSyncer={props.eventSyncer}
allUserInfo={props.allUserInfo}
/>
}
{
@ -187,6 +190,7 @@ interface MessageListProps {
eventEmitter: EventEmitter<DirectMessagePanelUpdate>;
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
allUserInfo: Map<string, UserInfo>;
}
interface MessageListState {
@ -246,10 +250,12 @@ export class MessageList extends Component<MessageListProps, MessageListState> {
});
console.log("MessageList:groupContinuousMessages", Date.now() - t);
const messageBoxGroups = [];
let i = 0;
for (const threads of groups) {
messageBoxGroups.push(
MessageBoxGroup({
messageGroup: threads.map((thread) => {
i++;
return {
msg: thread.root,
replyCount: thread.replies.length,
@ -260,10 +266,11 @@ export class MessageList extends Component<MessageListProps, MessageListState> {
db: this.props.db,
profilesSyncer: this.props.profilesSyncer,
eventSyncer: this.props.eventSyncer,
allUserInfo: this.props.allUserInfo,
}),
);
}
console.log("MessageList:elements", Date.now() - t);
console.log(`MessageList:elements ${i}`, Date.now() - t);
const vNode = (
<div
@ -312,6 +319,7 @@ function MessageBoxGroup(props: {
}[];
myPublicKey: PublicKey;
db: Database_Contextual_View;
allUserInfo: Map<string, UserInfo>;
eventEmitter: EventEmitter<DirectMessagePanelUpdate | ViewUserDetail>;
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
@ -355,7 +363,13 @@ function MessageBoxGroup(props: {
<pre
class={tw`text-[#DCDDDE] whitespace-pre-wrap break-words font-roboto`}
>
{ParseMessageContent(msg.msg, props.db, props.profilesSyncer, props.eventSyncer, props.eventEmitter)}
{ParseMessageContent(
msg.msg,
props.allUserInfo,
props.profilesSyncer,
props.eventSyncer,
props.eventEmitter,
)}
</pre>
{msg.replyCount > 0
? (
@ -443,7 +457,7 @@ export function NameAndTime(message: ChatMessage, index: number, myPublicKey: Pu
export function ParseMessageContent(
message: ChatMessage,
db: Database_Contextual_View,
allUserInfo: Map<string, UserInfo>,
profilesSyncer: ProfilesSyncer,
eventSyncer: EventSyncer,
eventEmitter: EventEmitter<ViewUserDetail | ViewThread>,
@ -454,44 +468,52 @@ export function ParseMessageContent(
const vnode = [];
let start = 0;
for (const item of parseContent(message.content)) {
const itemStr = message.content.slice(item.start, item.end + 1);
for (const item of message.event.parsedContentItems) {
vnode.push(message.content.slice(start, item.start));
const itemStr = message.content.slice(item.start, item.end + 1);
switch (item.type) {
case "url":
if (urlIsImage(itemStr)) {
vnode.push(<img src={itemStr} />);
} else {
vnode.push(
<a target="_blank" class={tw`hover:underline text-[${LinkColor}]`} href={itemStr}>
{itemStr}
</a>,
);
{
if (urlIsImage(itemStr)) {
vnode.push(<img src={itemStr} />);
} else {
vnode.push(
<a target="_blank" class={tw`hover:underline text-[${LinkColor}]`} href={itemStr}>
{itemStr}
</a>,
);
}
}
break;
case "npub":
const pubkey = PublicKey.FromBech32(itemStr);
if (pubkey instanceof Error) {
continue;
}
const profile = getProfileEvent(db, pubkey);
if (profile) {
vnode.push(ProfileCard(profile.profile, pubkey, eventEmitter));
} else {
profilesSyncer.add(pubkey.hex);
{
const userInfo = allUserInfo.get(item.pubkey);
if (userInfo) {
const profile = userInfo.profile;
if (profile) {
vnode.push(
ProfileCard(
profile.profile,
PublicKey.FromHex(item.pubkey) as PublicKey,
eventEmitter,
),
);
} else {
profilesSyncer.add(item.pubkey);
}
} else {
profilesSyncer.add(item.pubkey);
}
}
break;
case "note":
const note = NoteID.FromBech32(itemStr);
if (note instanceof Error) {
console.error(note);
break;
{
const event = eventSyncer.syncEvent(item.noteID);
if (event instanceof Promise) {
break;
}
vnode.push(NoteCard(event, eventEmitter, allUserInfo));
}
const event = eventSyncer.syncEvent(note);
if (event instanceof Promise) {
break;
}
vnode.push(NoteCard(event, eventEmitter, db));
break;
case "tag":
// todo
@ -519,7 +541,7 @@ function ProfileCard(profile: ProfileData, pubkey: PublicKey, eventEmitter: Even
<div class={tw`flex`}>
<Avatar class={tw`w-10 h-10`} picture={profile.picture}></Avatar>
<p class={tw`text-[1.2rem] font-blod leading-10 truncate ml-2`}>
{profile.name || pubkey.bech32}
{profile.name || pubkey.bech32()}
</p>
</div>
<div class={tw`${DividerClass} my-[0.5rem]`}></div>
@ -531,14 +553,14 @@ function ProfileCard(profile: ProfileData, pubkey: PublicKey, eventEmitter: Even
function NoteCard(
event: Profile_Nostr_Event | PlainText_Nostr_Event | Decrypted_Nostr_Event,
eventEmitter: EventEmitter<ViewThread | ViewUserDetail>,
db: Database_Contextual_View,
allUserInfo: Map<string, UserInfo>,
) {
switch (event.kind) {
case NostrKind.META_DATA:
return ProfileCard(event.profile, event.publicKey, eventEmitter);
case NostrKind.TEXT_NOTE:
case NostrKind.DIRECT_MESSAGE:
const profile = getProfileEvent(db, event.publicKey);
const profile = allUserInfo.get(event.pubkey)?.profile;
return (
<div class={tw`px-4 my-1 py-2 border-2 border-[${PrimaryTextColor}4D] rounded-lg py-1 flex`}>
<Avatar class={tw`w-10 h-10`} picture={profile?.profile.picture} />

View File

@ -14,7 +14,7 @@ import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr
import { ChatMessage, groupContinuousMessages } from "./message.ts";
import { Editor, EditorEvent, EditorModel } from "./editor.tsx";
import { Database_Contextual_View } from "../database.ts";
import { ProfilesSyncer } from "./contact-list.ts";
import { ProfilesSyncer, UserInfo } from "./contact-list.ts";
import { EventSyncer } from "./event_syncer.ts";
interface MessageThreadProps {
@ -25,6 +25,7 @@ interface MessageThreadProps {
editorModel: EditorModel;
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
allUserInfo: Map<string, UserInfo>;
}
export function MessageThreadPanel(props: MessageThreadProps) {
@ -48,6 +49,7 @@ export function MessageThreadPanel(props: MessageThreadProps) {
profilesSyncer={props.profilesSyncer}
eventSyncer={props.eventSyncer}
eventEmitter={props.eventEmitter}
allUserInfo={props.allUserInfo}
/>
</div>
@ -68,6 +70,7 @@ function MessageThreadList(props: {
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
eventEmitter: EventEmitter<ViewUserDetail | ViewThread>;
allUserInfo: Map<string, UserInfo>;
}) {
let groups = groupContinuousMessages(props.messages, (pre, cur) => {
const sameAuthor = pre.event.pubkey == cur.event.pubkey;
@ -84,6 +87,7 @@ function MessageThreadList(props: {
profilesSyncer={props.profilesSyncer}
eventSyncer={props.eventSyncer}
eventEmitter={props.eventEmitter}
allUserInfo={props.allUserInfo}
/>,
);
}
@ -103,6 +107,7 @@ function MessageThreadBoxGroup(props: {
profilesSyncer: ProfilesSyncer;
eventSyncer: EventSyncer;
eventEmitter: EventEmitter<ViewUserDetail | ViewThread>;
allUserInfo: Map<string, UserInfo>;
}) {
const vnode = (
<ul class={tw`py-2`}>
@ -120,7 +125,7 @@ function MessageThreadBoxGroup(props: {
<pre
class={tw`text-[#DCDDDE] whitespace-pre-wrap break-words font-roboto`}
>
{ParseMessageContent(msg, props.db, props.profilesSyncer, props.eventSyncer, props.eventEmitter)}
{ParseMessageContent(msg, props.allUserInfo, props.profilesSyncer, props.eventSyncer, props.eventEmitter)}
</pre>
</div>
</li>

View File

@ -1,7 +1,6 @@
import { assertEquals } from "https://deno.land/std@0.176.0/testing/asserts.ts";
import { ChatMessage, groupContinuousMessages, parseContent } from "./message.ts";
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { PrivateKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { PrivateKey, PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
Deno.test("inline parse", async (t) => {
const data = [
@ -69,6 +68,7 @@ Deno.test("inline parse", async (t) => {
input: `nostr:npub17dxnfw2vrhgtk4fgqdmpuqxv05u9raau3w0shay7msmr0dzs4m7s6ng4ylログボ`,
output: [{
type: "npub",
pubkey: "f34d34b94c1dd0bb552803761e00cc7d3851f7bc8b9f0bf49edc3637b450aefd",
start: 6,
end: 68,
}],
@ -117,6 +117,12 @@ Deno.test("message group", () => {
pubkey: "",
sig: "",
tags: [],
parsedContentItems: [],
parsedTags: {
e: [],
p: [],
},
publicKey: PrivateKey.Generate().toPublicKey(),
},
"content": "sendDirectMessage",
"type": "text",

View File

@ -1,6 +1,7 @@
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { MessageThread } from "./dm.tsx";
import { NostrEvent } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nostr.ts";
import { PlainText_Nostr_Event } from "../nostr.ts";
import { NoteID } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nip19.ts";
export function* parseContent(content: string) {
// URLs
@ -27,24 +28,61 @@ function* match(regex: RegExp, content: string, type: ItemType): Generator<Conte
return;
}
const urlEndPosition = urlStartPosition + match[0].length - 1;
yield {
type: type,
start: urlStartPosition,
end: urlEndPosition,
};
if (type == "note") {
const noteID = NoteID.FromBech32(content.slice(urlStartPosition, urlEndPosition + 1));
if (noteID instanceof Error) {
// ignore
} else {
yield {
type: type,
noteID: noteID,
start: urlStartPosition,
end: urlEndPosition,
};
}
} else if (type == "npub") {
const pubkey = PublicKey.FromBech32(content.slice(urlStartPosition, urlEndPosition + 1));
if (pubkey instanceof Error) {
// ignore
} else {
yield {
type: type,
pubkey: pubkey.hex,
start: urlStartPosition,
end: urlEndPosition,
};
}
} else {
yield {
type: type,
start: urlStartPosition,
end: urlEndPosition,
};
}
}
}
type ItemType = "url" | "npub" | "tag" | "note";
type otherItemType = "url" | "tag";
type ItemType = otherItemType | "note" | "npub";
export type ContentItem = {
type: ItemType;
type: otherItemType;
start: number;
end: number;
} | {
type: "npub";
pubkey: string;
start: number;
end: number;
} | {
type: "note";
noteID: NoteID;
start: number;
end: number;
};
// Think of ChatMessage as an materialized view of NostrEvent
export interface ChatMessage {
readonly event: NostrEvent;
readonly event: PlainText_Nostr_Event;
readonly type: "image" | "text";
readonly created_at: Date;
readonly lamport: number | undefined;

View File

@ -11,14 +11,15 @@ import {
Decryptable_Nostr_Event,
Decrypted_Nostr_Event,
getTags,
Parsed_Event,
PlainText_Nostr_Event,
Profile_Nostr_Event,
Tag,
} from "./nostr.ts";
import * as csp from "https://raw.githubusercontent.com/BlowaterNostr/csp/master/csp.ts";
import { DexieDatabase } from "./UI/dexie-db.ts";
import { parseProfileData, ProfileFromNostrEvent } from "./features/profile.ts";
import { parseProfileData } from "./features/profile.ts";
import { parseContent } from "./UI/message.ts";
import { NoteID } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nip19.ts";
export const NotFound = Symbol("Not Found");
const buffer_size = 1000;
@ -89,6 +90,7 @@ export class Database_Contextual_View {
tags: event.tags,
parsedTags: getTags(event),
publicKey: pubkey,
parsedContentItems: Array.from(parseContent(event.content)),
};
cache.push(e);
}
@ -147,6 +149,7 @@ export class Database_Contextual_View {
tags: event.tags,
parsedTags: getTags(event),
publicKey: pubkey,
parsedContentItems: Array.from(parseContent(event.content)),
};
cache.push(e);
await db.sourceOfChange.put(e);
@ -230,6 +233,7 @@ export class Database_Contextual_View {
tags: event.tags,
parsedTags: getTags(event),
publicKey: pubkey,
parsedContentItems: Array.from(parseContent(event.content)),
};
}
}

View File

@ -4,10 +4,7 @@ import {
ConnectionPool,
newSubID,
} from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/relay.ts";
import {
PublicKey,
publicKeyHexFromNpub,
} from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import {
groupBy,
NostrAccountContext,
@ -15,7 +12,7 @@ import {
NostrKind,
prepareNormalNostrEvent,
} from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nostr.ts";
import { Parsed_Event, PlainText_Nostr_Event, Profile_Nostr_Event, Tag } from "../nostr.ts";
import { Parsed_Event, Profile_Nostr_Event } from "../nostr.ts";
// nip01 meta data
// https://github.com/nostr-protocol/nips/blob/master/05.md

View File

@ -13,7 +13,12 @@ export function getSocialPosts(
allUsersInfo: Map<string, UserInfo>,
) {
const t = Date.now();
const events = db.filterEvents((e) => e.kind == NostrKind.TEXT_NOTE);
const events = [];
for (const e of db.events) {
if (e.kind == NostrKind.TEXT_NOTE) {
events.push(e);
}
}
console.log("getSocialPosts:filterEvents", Date.now() - t);
const threads = computeThreads(events);
console.log("getSocialPosts:computeThreads", Date.now() - t);

View File

@ -19,13 +19,15 @@ import {
groupImageEvents,
Parsed_Event,
parsedTagsEvent,
PlainText_Nostr_Event,
prepareNostrImageEvents,
prepareReplyEvent,
reassembleBase64ImageFromEvents,
} from "./nostr.ts";
import { LamportTime } from "./time.ts";
import { PrivateKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { PrivateKey, PublicKey } from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/key.ts";
import { ParseMessageContent } from "./UI/message-panel.tsx";
import { parseContent } from "./UI/message.ts";
Deno.test("prepareNostrImageEvents", async (t) => {
const pri = PrivateKey.Generate();
@ -109,14 +111,37 @@ Deno.test("groupImageEvents", async () => {
fail(imgEvents2.message);
}
const [events2, id2] = imgEvents2;
const groups = groupImageEvents(events1.concat(events2));
const groups = groupImageEvents(
events1.concat(events2).map((e): Parsed_Event => ({
...e,
kind: e.kind as NostrKind.DIRECT_MESSAGE,
publicKey: PublicKey.FromHex(e.pubkey) as PublicKey,
parsedTags: getTags(e),
})),
);
assertEquals(groups.size, 2);
const group1 = groups.get(id1);
const group1 = groups.get(id1)?.map((e): NostrEvent => ({
content: e.content,
created_at: e.created_at,
id: e.id,
kind: e.kind,
pubkey: e.pubkey,
sig: e.sig,
tags: e.tags,
}));
assertNotEquals(group1, undefined);
assertEquals(group1, events1);
const group2 = groups.get(id2);
const group2 = groups.get(id2)?.map((e): NostrEvent => ({
content: e.content,
created_at: e.created_at,
id: e.id,
kind: e.kind,
pubkey: e.pubkey,
sig: e.sig,
tags: e.tags,
}));
assertNotEquals(group2, undefined);
assertEquals(group2, events2);
});

View File

@ -11,6 +11,7 @@ import {
TagPubKey,
} from "https://raw.githubusercontent.com/BlowaterNostr/nostr.ts/main/nostr.ts";
import { ProfileData } from "./features/profile.ts";
import { ContentItem } from "./UI/message.ts";
type TotolChunks = string;
type ChunkIndex = string; // 0-indexed
@ -145,9 +146,9 @@ export function reassembleBase64ImageFromEvents(
return chunks.join("");
}
export function groupImageEvents(events: Iterable<nostr.NostrEvent>) {
export function groupImageEvents<T extends Parsed_Event>(events: Iterable<T>) {
return groupBy(events, (event) => {
const tags = getTags(event);
const tags = event.parsedTags;
const imageTag = tags.image;
if (imageTag == undefined) {
return undefined;
@ -287,9 +288,14 @@ export type Decrypted_Nostr_Event = Parsed_Event<NostrKind.CustomAppData> & {
export type Decryptable_Nostr_Event = nostr.NostrEvent<NostrKind.CustomAppData>;
export type PlainText_Nostr_Event = Parsed_Event<
Exclude<NostrKind, NostrKind.CustomAppData | NostrKind.META_DATA> // todo: exclude DM as well
>;
export type PlainText_Nostr_Event =
& Parsed_Event<
Exclude<NostrKind, NostrKind.CustomAppData | NostrKind.META_DATA> // todo: exclude DM as well
>
& {
parsedContentItems: ContentItem[];
};
export type Profile_Nostr_Event = Parsed_Event<NostrKind.META_DATA> & {
profile: ProfileData;
};