From 70aafae383a9ba7779a1e2e87b2dcd8d39e9ac03 Mon Sep 17 00:00:00 2001 From: Doug Hoyte Date: Tue, 10 Dec 2024 16:03:55 -0500 Subject: [PATCH] wip on feeds --- src/apps/web/FeedReader.h | 53 ++++++++++++++++++++++++------- src/apps/web/WebReader.cpp | 30 +++++++++++++---- src/apps/web/cmd_stories.cpp | 43 +++++++++++++++++++------ src/apps/web/static/oddbean.css | 5 +++ src/apps/web/tmpls/about.tmpl | 4 +-- src/apps/web/tmpls/feed/list.tmpl | 7 +++- 6 files changed, 112 insertions(+), 30 deletions(-) diff --git a/src/apps/web/FeedReader.h b/src/apps/web/FeedReader.h index b39bbd8..b9d2060 100644 --- a/src/apps/web/FeedReader.h +++ b/src/apps/web/FeedReader.h @@ -13,7 +13,10 @@ struct FeedReader { EventInfo info; }; - std::vector getEvents(lmdb::txn &txn, Decompressor &decomp, const std::string &feedId) { + std::vector getEvents(lmdb::txn &txn, Decompressor &decomp, const std::string &feedId, uint64_t resultsPerPage, uint64_t page) { + uint64_t skip = page * resultsPerPage; + uint64_t remaining = resultsPerPage; + size_t pos = feedId.find("."); if (pos == std::string_view::npos) throw herr("bad feedId: ", feedId); std::string pubkey = from_hex(feedId.substr(0, pos)); @@ -38,20 +41,46 @@ struct FeedReader { std::vector output; - const auto &tags = feedJson.at("tags").get_array(); + while (true) { + const auto &tags = feedJson.at("tags").get_array(); + std::string prev; - for (const auto &tag : tags) { - if (tag[0] != "e") continue; - std::string id = from_hex(tag[1].get_string()); + for (const auto &tag : tags) { + if (tag[0] == "prev") { + prev = from_hex(tag[1].get_string()); + continue; + } - auto ev = lookupEventById(txn, id); - if (!ev) continue; + if (tag[0] != "e") continue; + std::string id = from_hex(tag[1].get_string()); - output.push_back({ - ev->primaryKeyId, - id, - buildEventInfo(txn, id), - }); + auto ev = lookupEventById(txn, id); + if (!ev) continue; + + if (skip) { + skip--; + continue; + } + + output.push_back({ + ev->primaryKeyId, + id, + buildEventInfo(txn, id), + }); + + remaining--; + if (!remaining) break; + } + + if (remaining && prev.size()) { + auto ev = lookupEventById(txn, prev); + if (ev) { + feedJson = tao::json::from_string(getEventJson(txn, decomp, ev->primaryKeyId)); + continue; + } + } + + break; } return output; diff --git a/src/apps/web/WebReader.cpp b/src/apps/web/WebReader.cpp index 8b2c36c..8f68e24 100644 --- a/src/apps/web/WebReader.cpp +++ b/src/apps/web/WebReader.cpp @@ -92,13 +92,14 @@ void doSearch(lmdb::txn &txn, Decompressor &decomp, std::string_view search, std -TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, const std::string &feedId) { +TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, const std::string &feedId, uint64_t resultsPerPage, uint64_t page) { FeedReader feedReader; - auto events = feedReader.getEvents(txn, decomp, feedId); + auto events = feedReader.getEvents(txn, decomp, feedId, resultsPerPage, page); std::vector rendered; auto now = hoytech::curr_time_s(); - uint64_t n = 1; + uint64_t offset = (page * resultsPerPage) + 1; + uint64_t n = 0; for (auto &fe : events) { auto ev = Event::fromLevId(txn, fe.levId); @@ -111,7 +112,7 @@ TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCa std::string timestamp; FeedReader::EventInfo &info; } ctx = { - n, + offset + n, ev, *userCache.getUser(txn, decomp, ev.getPubkey()), renderTimestamp(now, ev.getCreatedAt()), @@ -122,7 +123,19 @@ TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCa n++; } - return tmpl::feed::list(rendered); + struct { + const std::vector &items; + uint64_t n; + uint64_t resultsPerPage; + uint64_t page; + } ctx = { + rendered, + n, + resultsPerPage, + page, + }; + + return tmpl::feed::list(ctx); } @@ -153,7 +166,12 @@ HTTPResponse WebServer::generateReadResponse(lmdb::txn &txn, Decompressor &decom std::optional rawBody; if (u.path.size() == 0) { - body = renderFeed(txn, decomp, userCache, cfg().web__homepageFeedId); + uint64_t resultsPerPage = 30; + uint64_t page = 0; + auto pageStr = u.lookupQuery("p"); + if (pageStr) page = std::stoull(std::string(*pageStr)); + + body = renderFeed(txn, decomp, userCache, cfg().web__homepageFeedId, resultsPerPage, page); httpResp.extraHeaders += "Cache-Control: max-age=600\r\n"; } else if (u.path[0] == "e") { diff --git a/src/apps/web/cmd_stories.cpp b/src/apps/web/cmd_stories.cpp index 8844d48..80e62dd 100644 --- a/src/apps/web/cmd_stories.cpp +++ b/src/apps/web/cmd_stories.cpp @@ -12,7 +12,7 @@ static const char USAGE[] = R"( Usage: - stories + stories [--top=] [--days=] [--oddbean] )"; @@ -32,16 +32,22 @@ struct FilteredEvent { void cmd_stories(const std::vector &subArgs) { std::map args = docopt::docopt(USAGE, subArgs, true, ""); + uint64_t top = 10; + if (args["--top"]) top = args["--top"].asLong(); + uint64_t days = 2; + if (args["--days"]) days = args["--days"].asLong(); + bool oddbeanOnly = args["--oddbean"].asBool(); + + uint64_t limit = 10000; + uint64_t timeWindow = 86400*days; + uint64_t threshold = 10; + Decompressor decomp; auto txn = env.txn_ro(); flat_hash_map eventInfoCache; std::vector output; - uint64_t limit = 10000; - uint64_t timeWindow = 86400*2; - uint64_t threshold = 10; - uint64_t now = hoytech::curr_time_s(); env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv(MAX_U64), lmdb::to_sv(MAX_U64), [&](auto k, auto v) { @@ -55,6 +61,7 @@ void cmd_stories(const std::vector &subArgs) { if (kind == 1) { bool foundETag = false; + bool isOddbeanTopic = false; packed.foreachTag([&](char tagName, std::string_view tagVal){ if (tagName == 'e') { auto tagEventId = tagVal; @@ -62,6 +69,7 @@ void cmd_stories(const std::vector &subArgs) { eventInfoCache[tagEventId].comments++; foundETag = true; } + if (tagName == 't' && tagVal == "oddbean") isOddbeanTopic = true; return true; }); if (foundETag) return true; // not root event @@ -69,7 +77,19 @@ void cmd_stories(const std::vector &subArgs) { eventInfoCache.emplace(id, EventInfo{}); auto &eventInfo = eventInfoCache[id]; - if (eventInfo.reactions < threshold) return true; + if (oddbeanOnly) { + if (!isOddbeanTopic) return true; + + // Filter out posts from oddbot clients + tao::json::value json = tao::json::from_string(getEventJson(txn, decomp, ev.primaryKeyId)); + const auto &tags = json.at("tags").get_array(); + for (const auto &t : tags) { + const auto &tArr = t.get_array(); + if (tArr.at(0) == "client" && tArr.size() >= 2 && tArr.at(1) == "oddbot") return true; + } + } else { + if (eventInfo.reactions < threshold) return true; + } output.emplace_back(FilteredEvent{ev.primaryKeyId, id, packed.created_at(), eventInfo}); } else if (kind == 7) { @@ -93,11 +113,16 @@ void cmd_stories(const std::vector &subArgs) { return o.created_at < (now - timeWindow); }), output.end()); - std::sort(output.begin(), output.end(), [](const auto &a, const auto &b){ - return a.info.reactions > b.info.reactions; - }); + if (!oddbeanOnly) { + std::sort(output.begin(), output.end(), [](const auto &a, const auto &b){ + return a.info.reactions > b.info.reactions; + }); + } for (const auto &o : output) { + if (top == 0) break; + top--; + tao::json::value ev = tao::json::from_string(getEventJson(txn, decomp, o.levId)); std::string content = ev.at("content").get_string(); diff --git a/src/apps/web/static/oddbean.css b/src/apps/web/static/oddbean.css index 01b6937..a35b6e0 100644 --- a/src/apps/web/static/oddbean.css +++ b/src/apps/web/static/oddbean.css @@ -278,6 +278,11 @@ table.vert { } } +.feed-nav-links { + margin-top: 20px; + margin-left: 40px; +} + /* voting */ diff --git a/src/apps/web/tmpls/about.tmpl b/src/apps/web/tmpls/about.tmpl index 1abe1dc..adbfbf9 100644 --- a/src/apps/web/tmpls/about.tmpl +++ b/src/apps/web/tmpls/about.tmpl @@ -4,7 +4,7 @@

- Why nostr? Nostr is a protocol for building decentralised systems. Is it perfect? No, but today it is the closest thing we have to an honest-to-goodness decentralised social media protocol. Oddbean is just one of many interfaces to the same network. Even though most of the posts don't originate here, Oddbean users can still interact with anyone else on the network, thanks to the interoperability of nostr. If you get tired of Oddbean, you can easily move to another client. + Why nostr? Nostr is a protocol for building decentralised systems. Is it perfect? No, but today it is the closest thing we have to an honest-to-goodness decentralised social media protocol. Oddbean is just one of many interfaces to the same network. Even though many of the posts don't originate here, Oddbean users can still interact with anyone else on the network, thanks to the interoperability of nostr. If you get tired of Oddbean, you can easily move to another client.

@@ -20,7 +20,7 @@

- Is there an algorithm? Yes, computers don't work without algorithms. The goal of Oddbean is to make the algorithms used for ranking and filtering posts and comments transparent. When communities are ready, you'll be able to click on "algo" at the top of a community page and see exactly how it works. If you don't like it, you can fork it to your own community. The homepage is just another community, except one that I control. Don't like it? Don't use it. + Feeds. Oddbean is built around the concept of a "feeds", which are lists of nostr events. Feeds can be curated by people, by algorithms, or both (computers don't work without algorithms). The goal of Oddbean is to let you publish your own feeds according to whatever policies you or a community decide -- similar to a subreddit. A specification on how to create your own feeds is coming soon. The homepage is just a feed like any other, except one that I control. Don't like it? Don't use it.

diff --git a/src/apps/web/tmpls/feed/list.tmpl b/src/apps/web/tmpls/feed/list.tmpl index cdd9457..ee02af9 100644 --- a/src/apps/web/tmpls/feed/list.tmpl +++ b/src/apps/web/tmpls/feed/list.tmpl @@ -1,3 +1,8 @@ -

@(const auto &r : ctx) +
@(const auto &r : ctx.items) $(r)
+ +
+ More ?(ctx.n != 0) + The end. ?(ctx.n == 0) +