#pragma once #include "re2/re2.h" #include "Bech32Utils.h" #include "WebUtils.h" #include "AlgoScanner.h" #include "WebTemplates.h" #include "DBQuery.h" std::string stripUrls(std::string &content); inline void preprocessMetaFieldContent(std::string &content) { static RE2 matcher(R"((?is)(.*?)(https?://\S+))"); std::string output; std::string_view contentSv(content); re2::StringPiece input(contentSv); re2::StringPiece prefix, match; auto sv = [](re2::StringPiece s){ return std::string_view(s.data(), s.size()); }; auto appendLink = [&](std::string_view url, std::string_view text){ output += ""; output += text; output += ""; }; while (RE2::Consume(&input, matcher, &prefix, &match)) { output += sv(prefix); if (match.starts_with("http")) { appendLink(sv(match), sv(match)); } } if (output.size()) { output += std::string_view(input.data(), input.size()); std::swap(output, content); } } struct User { std::string pubkey; std::string npubId; std::string username; std::optional kind0Json; std::optional kind3Event; User(lmdb::txn &txn, Decompressor &decomp, const std::string &pubkey) : pubkey(pubkey) { npubId = encodeBech32Simple("npub", pubkey); kind0Json = loadKindJson(txn, decomp, 0); try { if (kind0Json) username = kind0Json->at("name").get_string(); } catch (std::exception &e) { } if (username.size() == 0) username = to_hex(pubkey.substr(0,4)); if (username.size() > 50) username = username.substr(0, 50) + "..."; } std::optional loadKindJson(lmdb::txn &txn, Decompressor &decomp, uint64_t kind) { std::optional output; env.generic_foreachFull(txn, env.dbi_Event__pubkeyKind, makeKey_StringUint64Uint64(pubkey, kind, 0), "", [&](std::string_view k, std::string_view v){ ParsedKey_StringUint64Uint64 parsedKey(k); if (parsedKey.s == pubkey && parsedKey.n1 == kind) { auto levId = lmdb::from_sv(v); tao::json::value json = tao::json::from_string(getEventJson(txn, decomp, levId)); try { output = tao::json::from_string(json.at("content").get_string()); if (!output->is_object()) output = std::nullopt; } catch (std::exception &e) { } } return false; }); return output; } std::optional loadKindEvent(lmdb::txn &txn, Decompressor &decomp, uint64_t kind) { std::optional output; env.generic_foreachFull(txn, env.dbi_Event__pubkeyKind, makeKey_StringUint64Uint64(pubkey, kind, 0), "", [&](std::string_view k, std::string_view v){ ParsedKey_StringUint64Uint64 parsedKey(k); if (parsedKey.s == pubkey && parsedKey.n1 == kind) { auto levId = lmdb::from_sv(v); output = tao::json::from_string(getEventJson(txn, decomp, levId)); } return false; }); return output; } bool kind0Found() const { return !!kind0Json; } std::string getMeta(std::string_view field) const { std::string output; if (!kind0Json) throw herr("can't getMeta because user doesn't have kind 0"); if (kind0Json->get_object().contains(field) && kind0Json->at(field).is_string()) output = kind0Json->at(field).get_string(); output = templarInternal::htmlEscape(output, false); preprocessMetaFieldContent(output); return output; } void populateContactList(lmdb::txn &txn, Decompressor &decomp) { kind3Event = loadKindEvent(txn, decomp, 3); } std::vector getFollowers(lmdb::txn &txn, Decompressor &decomp, const std::string &pubkey) { std::vector output; flat_hash_set alreadySeen; std::string prefix = "p"; prefix += pubkey; env.generic_foreachFull(txn, env.dbi_Event__tag, prefix, "", [&](std::string_view k, std::string_view v){ ParsedKey_StringUint64 parsedKey(k); if (parsedKey.s != prefix) return false; auto levId = lmdb::from_sv(v); auto ev = lookupEventByLevId(txn, levId); if (ev.flat_nested()->kind() == 3) { auto pubkey = std::string(sv(ev.flat_nested()->pubkey())); if (!alreadySeen.contains(pubkey)) { alreadySeen.insert(pubkey); output.emplace_back(std::move(pubkey)); } } return true; }); return output; } }; struct UserCache { std::unordered_map cache; const User *getUser(lmdb::txn &txn, Decompressor &decomp, const std::string &pubkey) { auto u = cache.find(pubkey); if (u != cache.end()) return &u->second; cache.emplace(pubkey, User(txn, decomp, pubkey)); return &cache.at(pubkey); } }; struct Event { defaultDb::environment::View_Event ev; tao::json::value json = tao::json::null; std::string parent; std::string root; uint64_t upVotes = 0; uint64_t downVotes = 0; Event(defaultDb::environment::View_Event ev) : ev(ev) { } static Event fromLevId(lmdb::txn &txn, uint64_t levId) { return Event(lookupEventByLevId(txn, levId)); } static Event fromId(lmdb::txn &txn, std::string_view id) { auto existing = lookupEventById(txn, id); if (!existing) throw herr("unable to find event"); return Event(std::move(*existing)); } static Event fromIdExternal(lmdb::txn &txn, std::string_view id) { if (id.starts_with("note1")) { return fromId(txn, decodeBech32Simple(id)); } else { return fromId(txn, from_hex(id)); } } std::string getId() const { return std::string(sv(ev.flat_nested()->id())); } uint64_t getKind() const { return ev.flat_nested()->kind(); } uint64_t getCreatedAt() const { return ev.flat_nested()->created_at(); } std::string getPubkey() const { return std::string(sv(ev.flat_nested()->pubkey())); } std::string getNoteId() const { return encodeBech32Simple("note", getId()); } std::string getParentNoteId() const { return encodeBech32Simple("note", parent); } std::string getRootNoteId() const { return encodeBech32Simple("note", root); } // FIXME: Use "subject" tag if present? // FIXME: Don't truncate UTF-8 mid-sequence // FIXME: Don't put ellipsis if truncated text ends in punctuation std::string summaryHtml() const { std::string content = json.at("content").get_string(); auto firstUrl = stripUrls(content); auto textAbbrev = [](std::string &str, size_t maxLen){ if (str.size() > maxLen) str = str.substr(0, maxLen-3) + "..."; }; textAbbrev(content, 100); templarInternal::htmlEscape(content, true); if (firstUrl.size()) { while (content.size() && isspace(content.back())) content.pop_back(); if (content.empty()) { content = firstUrl; textAbbrev(content, 100); templarInternal::htmlEscape(content, true); } return std::string("" + content + ""; } return content; } void populateJson(lmdb::txn &txn, Decompressor &decomp) { if (!json.is_null()) return; json = tao::json::from_string(getEventJson(txn, decomp, ev.primaryKeyId)); } void populateRootParent(lmdb::txn &txn, Decompressor &decomp) { populateJson(txn, decomp); const auto &tags = json.at("tags").get_array(); // Try to find a e-tags with root/reply types for (const auto &t : tags) { const auto &tArr = t.get_array(); if (tArr.at(0) == "e" && tArr.size() >= 4 && tArr.at(3) == "root") { root = from_hex(tArr.at(1).get_string()); } else if (tArr.at(0) == "e" && tArr.size() >= 4 && tArr.at(3) == "reply") { parent = from_hex(tArr.at(1).get_string()); } } if (!root.size()) { // Otherwise, assume first e tag is root for (auto it = tags.begin(); it != tags.end(); ++it) { const auto &tArr = it->get_array(); if (tArr.at(0) == "e") { root = from_hex(tArr.at(1).get_string()); break; } } } if (!parent.size()) { // Otherwise, assume last e tag is root for (auto it = tags.rbegin(); it != tags.rend(); ++it) { const auto &tArr = it->get_array(); if (tArr.at(0) == "e") { parent = from_hex(tArr.at(1).get_string()); break; } } } } }; inline void preprocessEventContent(lmdb::txn &txn, Decompressor &decomp, const Event &ev, UserCache &userCache, std::string &content) { static RE2 matcher(R"((?is)(.*?)(https?://\S+|#\[\d+\]|nostr:(?:note|npub)1\w+))"); std::string output; std::string_view contentSv(content); re2::StringPiece input(contentSv); re2::StringPiece prefix, match; auto sv = [](re2::StringPiece s){ return std::string_view(s.data(), s.size()); }; auto appendLink = [&](std::string_view url, std::string_view text){ output += ""; output += text; output += ""; }; while (RE2::Consume(&input, matcher, &prefix, &match)) { output += sv(prefix); if (match.starts_with("http")) { appendLink(sv(match), sv(match)); } else if (match.starts_with("nostr:note1")) { std::string path = "/e/"; path += sv(match).substr(6); appendLink(path, sv(match)); } else if (match.starts_with("nostr:npub1")) { bool didTransform = false; try { const auto *u = userCache.getUser(txn, decomp, decodeBech32Simple(sv(match).substr(6))); appendLink(std::string("/u/") + u->npubId, std::string("@") + u->username); didTransform = true; } catch(std::exception &e) { //LW << "tag parse error: " << e.what(); } if (!didTransform) output += sv(match); } else if (match.starts_with("#[")) { bool didTransform = false; auto offset = std::stoull(std::string(sv(match)).substr(2, match.size() - 3)); const auto &tags = ev.json.at("tags").get_array(); try { const auto &tag = tags.at(offset).get_array(); if (tag.at(0) == "p") { const auto *u = userCache.getUser(txn, decomp, from_hex(tag.at(1).get_string())); appendLink(std::string("/u/") + u->npubId, u->username); didTransform = true; } else if (tag.at(0) == "e") { appendLink(std::string("/e/") + encodeBech32Simple("note", from_hex(tag.at(1).get_string())), sv(match)); didTransform = true; } } catch(std::exception &e) { //LW << "tag parse error: " << e.what(); } if (!didTransform) output += sv(match); } } if (output.size()) { output += std::string_view(input.data(), input.size()); std::swap(output, content); } } inline std::string stripUrls(std::string &content) { static RE2 matcher(R"((?is)(.*?)(https?://\S+))"); std::string output; std::string firstUrl; std::string_view contentSv(content); re2::StringPiece input(contentSv); re2::StringPiece prefix, match; auto sv = [](re2::StringPiece s){ return std::string_view(s.data(), s.size()); }; while (RE2::Consume(&input, matcher, &prefix, &match)) { output += sv(prefix); if (firstUrl.empty()) { firstUrl = std::string(sv(match)); } } output += std::string_view(input.data(), input.size()); std::swap(output, content); return firstUrl; } struct ReplyCtx { uint64_t timestamp; TemplarResult rendered; }; struct RenderedEventCtx { std::string content; std::string timestamp; const Event *ev = nullptr; const User *user = nullptr; bool isFullThreadLoaded = false; bool eventPresent = true; bool abbrev = false; bool highlight = false; bool showActions = true; std::vector replies; }; struct EventThread { std::string rootEventId; bool isRootEventThreadRoot; flat_hash_map eventCache; flat_hash_map> children; // parentEventId -> childEventIds std::string pubkeyHighlight; bool isFullThreadLoaded = false; // Load all events under an eventId EventThread(std::string rootEventId, bool isRootEventThreadRoot, flat_hash_map &&eventCache) : rootEventId(rootEventId), isRootEventThreadRoot(isRootEventThreadRoot), eventCache(eventCache) {} EventThread(lmdb::txn &txn, Decompressor &decomp, std::string_view id_) : rootEventId(std::string(id_)) { try { eventCache.emplace(rootEventId, Event::fromId(txn, rootEventId)); } catch (std::exception &e) { return; } eventCache.at(rootEventId).populateRootParent(txn, decomp); isRootEventThreadRoot = eventCache.at(rootEventId).root.empty(); isFullThreadLoaded = true; std::vector pendingQueue; pendingQueue.emplace_back(rootEventId); while (pendingQueue.size()) { auto currId = std::move(pendingQueue.back()); pendingQueue.pop_back(); std::string prefix = "e"; prefix += currId; env.generic_foreachFull(txn, env.dbi_Event__tag, prefix, "", [&](std::string_view k, std::string_view v){ ParsedKey_StringUint64 parsedKey(k); if (parsedKey.s != prefix) return false; auto levId = lmdb::from_sv(v); Event e = Event::fromLevId(txn, levId); std::string childEventId = e.getId(); if (eventCache.contains(childEventId)) return true; eventCache.emplace(childEventId, std::move(e)); if (!isRootEventThreadRoot) pendingQueue.emplace_back(childEventId); return true; }); } for (auto &[id, e] : eventCache) { e.populateRootParent(txn, decomp); auto kind = e.getKind(); if (e.parent.size()) { if (kind == 1) { if (!children.contains(e.parent)) children.emplace(std::piecewise_construct, std::make_tuple(e.parent), std::make_tuple()); children.at(e.parent).insert(id); } else if (kind == 7) { auto p = eventCache.find(e.parent); if (p != eventCache.end()) { auto &parent = p->second; if (e.json.at("content").get_string() == "-") { parent.downVotes++; } else { parent.upVotes++; } } } } } } TemplarResult render(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, std::optional focusOnPubkey = std::nullopt) { auto now = hoytech::curr_time_s(); flat_hash_set processedLevIds; std::function process = [&](const std::string &id){ RenderedEventCtx ctx; auto p = eventCache.find(id); if (p != eventCache.end()) { const auto &elem = p->second; processedLevIds.insert(elem.ev.primaryKeyId); auto pubkey = elem.getPubkey(); ctx.timestamp = renderTimestamp(now, elem.getCreatedAt()); ctx.user = userCache.getUser(txn, decomp, elem.getPubkey()); ctx.isFullThreadLoaded = isFullThreadLoaded; ctx.eventPresent = true; ctx.highlight = (pubkey == pubkeyHighlight); ctx.abbrev = focusOnPubkey && *focusOnPubkey != pubkey; if (ctx.abbrev) { ctx.content = elem.summaryHtml(); } else { ctx.content = templarInternal::htmlEscape(elem.json.at("content").get_string(), false); preprocessEventContent(txn, decomp, elem, userCache, ctx.content); } ctx.ev = &elem; } else { ctx.eventPresent = false; } if (children.contains(id)) { for (const auto &childId : children.at(id)) { auto timestamp = MAX_U64; auto p = eventCache.find(childId); if (p != eventCache.end()) timestamp = p->second.getCreatedAt(); ctx.replies.emplace_back(timestamp, process(childId)); } std::sort(ctx.replies.begin(), ctx.replies.end(), [](auto &a, auto &b){ return a.timestamp < b.timestamp; }); } return tmpl::event::event(ctx); }; struct { TemplarResult foundEvents; std::vector orphanNodes; } ctx; ctx.foundEvents = process(rootEventId); for (auto &[id, e] : eventCache) { if (processedLevIds.contains(e.ev.primaryKeyId)) continue; if (e.getKind() != 1) continue; ctx.orphanNodes.emplace_back(e.getCreatedAt(), process(id)); } std::sort(ctx.orphanNodes.begin(), ctx.orphanNodes.end(), [](auto &a, auto &b){ return a.timestamp < b.timestamp; }); return tmpl::events(ctx); } }; struct UserEvents { User u; struct EventCluster { std::string rootEventId; flat_hash_map eventCache; // eventId (non-root) -> Event bool isRootEventFromUser = false; bool isRootPresent = false; uint64_t rootEventTimestamp = 0; EventCluster(std::string rootEventId) : rootEventId(rootEventId) {} }; std::vector eventClusterArr; UserEvents(lmdb::txn &txn, Decompressor &decomp, const std::string &pubkey) : u(txn, decomp, pubkey) { flat_hash_map eventClusters; // eventId (root) -> EventCluster env.generic_foreachFull(txn, env.dbi_Event__pubkeyKind, makeKey_StringUint64Uint64(pubkey, 1, MAX_U64), "", [&](std::string_view k, std::string_view v){ ParsedKey_StringUint64Uint64 parsedKey(k); if (parsedKey.s != pubkey || parsedKey.n1 != 1) return false; Event ev = Event::fromLevId(txn, lmdb::from_sv(v)); ev.populateRootParent(txn, decomp); auto id = ev.getId(); auto installRoot = [&](std::string rootId, Event &&rootEvent){ rootEvent.populateRootParent(txn, decomp); eventClusters.emplace(rootId, rootId); auto &cluster = eventClusters.at(rootId); cluster.isRootPresent = true; cluster.isRootEventFromUser = rootEvent.getPubkey() == u.pubkey; cluster.rootEventTimestamp = rootEvent.getCreatedAt(); cluster.eventCache.emplace(rootId, std::move(rootEvent)); }; if (ev.root.size()) { // Event is not root if (!eventClusters.contains(ev.root)) { try { installRoot(ev.root, Event::fromId(txn, ev.root)); } catch (std::exception &e) { // no root event eventClusters.emplace(ev.root, ev.root); auto &cluster = eventClusters.at(ev.root); cluster.isRootPresent = true; } } eventClusters.at(ev.root).eventCache.emplace(id, std::move(ev)); } else { // Event is root if (!eventClusters.contains(ev.root)) { installRoot(id, std::move(ev)); } } return true; }, true); for (auto &[k, v] : eventClusters) { eventClusterArr.emplace_back(std::move(v)); } std::sort(eventClusterArr.begin(), eventClusterArr.end(), [](auto &a, auto &b){ return b.rootEventTimestamp < a.rootEventTimestamp; }); } TemplarResult render(lmdb::txn &txn, Decompressor &decomp) { std::vector renderedThreads; UserCache userCache; for (auto &cluster : eventClusterArr) { EventThread eventThread(cluster.rootEventId, cluster.isRootEventFromUser, std::move(cluster.eventCache)); eventThread.pubkeyHighlight = u.pubkey; renderedThreads.emplace_back(eventThread.render(txn, decomp, userCache, u.pubkey)); } struct { std::vector &renderedThreads; User &u; } ctx = { renderedThreads, u, }; return tmpl::user::comments(ctx); } }; struct CommunitySpec { bool valid = true; tao::json::value raw; std::string name; std::string desc; std::string algo; std::string adminNpub; std::string adminUsername; std::string adminTopic; }; inline CommunitySpec lookupCommunitySpec(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, std::string_view algoDescriptor) { CommunitySpec spec; size_t pos = algoDescriptor.find("/"); if (pos == std::string_view::npos) throw herr("bad algo descriptor"); spec.adminNpub = std::string(algoDescriptor.substr(0, pos)); std::string authorPubkey = decodeBech32Simple(spec.adminNpub); spec.adminTopic = algoDescriptor.substr(pos + 1); tao::json::value filter = tao::json::value({ { "authors", tao::json::value::array({ to_hex(authorPubkey) }) }, { "kinds", tao::json::value::array({ uint64_t(33700) }) }, { "#d", tao::json::value::array({ spec.adminTopic }) }, }); bool found = false; foreachByFilter(txn, filter, [&](uint64_t levId){ tao::json::value ev = tao::json::from_string(getEventJson(txn, decomp, levId)); spec.algo = ev.at("content").get_string(); found = true; return false; }); if (!found) throw herr("unable to find algo"); spec.raw = tao::json::from_string(spec.algo); spec.name = spec.raw.at("name").get_string(); spec.desc = spec.raw.at("desc").get_string(); spec.algo = spec.raw.at("algo").get_string(); auto *user = userCache.getUser(txn, decomp, authorPubkey); spec.adminUsername = user->username; return spec; }