wip on feeds

src/apps/web/FeedReader.h

@@ -13,7 +13,10 @@ struct FeedReader {
         EventInfo info;
     };
 
-    std::vector<FeedEvent> getEvents(lmdb::txn &txn, Decompressor &decomp, const std::string &feedId) {
+    std::vector<FeedEvent> getEvents(lmdb::txn &txn, Decompressor &decomp, const std::string &feedId, uint64_t resultsPerPage, uint64_t page) {
+        uint64_t skip = page * resultsPerPage;
+        uint64_t remaining = resultsPerPage;
+
         size_t pos = feedId.find(".");
         if (pos == std::string_view::npos) throw herr("bad feedId: ", feedId);
         std::string pubkey = from_hex(feedId.substr(0, pos));
@@ -38,20 +41,46 @@ struct FeedReader {
 
         std::vector<FeedEvent> output;
 
-        const auto &tags = feedJson.at("tags").get_array();
+        while (true) {
+            const auto &tags = feedJson.at("tags").get_array();
+            std::string prev;
 
-        for (const auto &tag : tags) {
+            for (const auto &tag : tags) {
-            if (tag[0] != "e") continue;
+                if (tag[0] == "prev") {
-            std::string id = from_hex(tag[1].get_string());
+                    prev = from_hex(tag[1].get_string());
+                    continue;
+                }
 
-            auto ev = lookupEventById(txn, id);
+                if (tag[0] != "e") continue;
-            if (!ev) continue;
+                std::string id = from_hex(tag[1].get_string());
 
-            output.push_back({
+                auto ev = lookupEventById(txn, id);
-                ev->primaryKeyId,
+                if (!ev) continue;
-                id,
+
-                buildEventInfo(txn, id),
+                if (skip) {
-            });
+                    skip--;
+                    continue;
+                }
+
+                output.push_back({
+                    ev->primaryKeyId,
+                    id,
+                    buildEventInfo(txn, id),
+                });
+
+                remaining--;
+                if (!remaining) break;
+            }
+
+            if (remaining && prev.size()) {
+                auto ev = lookupEventById(txn, prev);
+                if (ev) {
+                    feedJson = tao::json::from_string(getEventJson(txn, decomp, ev->primaryKeyId));
+                    continue;
+                }
+            }
+
+            break;
         }
 
         return output;

src/apps/web/WebReader.cpp

@@ -92,13 +92,14 @@ void doSearch(lmdb::txn &txn, Decompressor &decomp, std::string_view search, std
 
 
 
-TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, const std::string &feedId) {
+TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCache, const std::string &feedId, uint64_t resultsPerPage, uint64_t page) {
     FeedReader feedReader;
-    auto events = feedReader.getEvents(txn, decomp, feedId);
+    auto events = feedReader.getEvents(txn, decomp, feedId, resultsPerPage, page);
 
     std::vector<TemplarResult> rendered;
     auto now = hoytech::curr_time_s();
-    uint64_t n = 1;
+    uint64_t offset = (page * resultsPerPage) + 1;
+    uint64_t n = 0;
 
     for (auto &fe : events) {
         auto ev = Event::fromLevId(txn, fe.levId);
@@ -111,7 +112,7 @@ TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCa
             std::string timestamp;
             FeedReader::EventInfo &info;
         } ctx = {
-            n,
+            offset + n,
             ev,
             *userCache.getUser(txn, decomp, ev.getPubkey()),
             renderTimestamp(now, ev.getCreatedAt()),
@@ -122,7 +123,19 @@ TemplarResult renderFeed(lmdb::txn &txn, Decompressor &decomp, UserCache &userCa
         n++;
     }
 
-    return tmpl::feed::list(rendered);
+    struct {
+        const std::vector<TemplarResult> &items;
+        uint64_t n;
+        uint64_t resultsPerPage;
+        uint64_t page;
+    } ctx = {
+        rendered,
+        n,
+        resultsPerPage,
+        page,
+    };
+
+    return tmpl::feed::list(ctx);
 }
 
 
@@ -153,7 +166,12 @@ HTTPResponse WebServer::generateReadResponse(lmdb::txn &txn, Decompressor &decom
     std::optional<std::string> rawBody;
 
     if (u.path.size() == 0) {
-        body = renderFeed(txn, decomp, userCache, cfg().web__homepageFeedId);
+        uint64_t resultsPerPage = 30;
+        uint64_t page = 0;
+        auto pageStr = u.lookupQuery("p");
+        if (pageStr) page = std::stoull(std::string(*pageStr));
+
+        body = renderFeed(txn, decomp, userCache, cfg().web__homepageFeedId, resultsPerPage, page);
 
         httpResp.extraHeaders += "Cache-Control: max-age=600\r\n";
     } else if (u.path[0] == "e") {

src/apps/web/cmd_stories.cpp

@@ -12,7 +12,7 @@
 static const char USAGE[] =
 R"(
     Usage:
-      stories
+      stories [--top=<top>] [--days=<days>] [--oddbean]
 )";
 
 
@@ -32,16 +32,22 @@ struct FilteredEvent {
 void cmd_stories(const std::vector<std::string> &subArgs) {
     std::map<std::string, docopt::value> args = docopt::docopt(USAGE, subArgs, true, "");
 
+    uint64_t top = 10;
+    if (args["--top"]) top = args["--top"].asLong();
+    uint64_t days = 2;
+    if (args["--days"]) days = args["--days"].asLong();
+    bool oddbeanOnly = args["--oddbean"].asBool();
+
+    uint64_t limit = 10000;
+    uint64_t timeWindow = 86400*days;
+    uint64_t threshold = 10;
+
     Decompressor decomp;
     auto txn = env.txn_ro();
 
     flat_hash_map<Bytes32, EventInfo> eventInfoCache;
     std::vector<FilteredEvent> output;
 
-    uint64_t limit = 10000;
-    uint64_t timeWindow = 86400*2;
-    uint64_t threshold = 10;
-
     uint64_t now = hoytech::curr_time_s();
 
     env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(MAX_U64), lmdb::to_sv<uint64_t>(MAX_U64), [&](auto k, auto v) {
@@ -55,6 +61,7 @@ void cmd_stories(const std::vector<std::string> &subArgs) {
 
         if (kind == 1) {
             bool foundETag = false;
+            bool isOddbeanTopic = false;
             packed.foreachTag([&](char tagName, std::string_view tagVal){
                 if (tagName == 'e') {
                     auto tagEventId = tagVal;
@@ -62,6 +69,7 @@ void cmd_stories(const std::vector<std::string> &subArgs) {
                     eventInfoCache[tagEventId].comments++;
                     foundETag = true;
                 }
+                if (tagName == 't' && tagVal == "oddbean") isOddbeanTopic = true;
                 return true;
             });
             if (foundETag) return true; // not root event
@@ -69,7 +77,19 @@ void cmd_stories(const std::vector<std::string> &subArgs) {
             eventInfoCache.emplace(id, EventInfo{});
             auto &eventInfo = eventInfoCache[id];
 
-            if (eventInfo.reactions < threshold) return true;
+            if (oddbeanOnly) {
+                if (!isOddbeanTopic) return true;
+
+                // Filter out posts from oddbot clients
+                tao::json::value json = tao::json::from_string(getEventJson(txn, decomp, ev.primaryKeyId));
+                const auto &tags = json.at("tags").get_array();
+                for (const auto &t : tags) {
+                    const auto &tArr = t.get_array();
+                    if (tArr.at(0) == "client" && tArr.size() >= 2 && tArr.at(1) == "oddbot") return true;
+                }
+            } else {
+                if (eventInfo.reactions < threshold) return true;
+            }
 
             output.emplace_back(FilteredEvent{ev.primaryKeyId, id, packed.created_at(), eventInfo});
         } else if (kind == 7) {
@@ -93,11 +113,16 @@ void cmd_stories(const std::vector<std::string> &subArgs) {
         return o.created_at < (now - timeWindow);
     }), output.end());
 
-    std::sort(output.begin(), output.end(), [](const auto &a, const auto &b){
+    if (!oddbeanOnly) {
-        return a.info.reactions > b.info.reactions;
+        std::sort(output.begin(), output.end(), [](const auto &a, const auto &b){
-    });
+            return a.info.reactions > b.info.reactions;
+        });
+    }
 
     for (const auto &o : output) {
+        if (top == 0) break;
+        top--;
+
         tao::json::value ev = tao::json::from_string(getEventJson(txn, decomp, o.levId));
 
         std::string content = ev.at("content").get_string();

src/apps/web/static/oddbean.css

@@ -278,6 +278,11 @@ table.vert {
     }
 }
 
+.feed-nav-links {
+    margin-top: 20px;
+    margin-left: 40px;
+}
+
 
 /* voting */
 

src/apps/web/tmpls/about.tmpl

@@ -4,7 +4,7 @@
     </p>
 
     <p>
-        <b>Why nostr?</b> Nostr is a protocol for building decentralised systems. Is it perfect? No, but today it is the closest thing we have to an honest-to-goodness decentralised social media protocol. Oddbean is just one of many interfaces to the same network. Even though most of the posts don't originate here, Oddbean users can still interact with anyone else on the network, thanks to the interoperability of nostr. If you get tired of Oddbean, you can easily move to another client.
+        <b>Why nostr?</b> Nostr is a protocol for building decentralised systems. Is it perfect? No, but today it is the closest thing we have to an honest-to-goodness decentralised social media protocol. Oddbean is just one of many interfaces to the same network. Even though many of the posts don't originate here, Oddbean users can still interact with anyone else on the network, thanks to the interoperability of nostr. If you get tired of Oddbean, you can easily move to another client.
     </p>
 
     <p>
@@ -20,7 +20,7 @@
     </p>
 
     <p>
-        <b>Is there an algorithm?</b> Yes, <a href="https://www.tbray.org/ongoing/When/202x/2022/11/28/On-Algorithms">computers don't work without algorithms</a>. The goal of Oddbean is to make the algorithms used for ranking and filtering posts and comments transparent. When communities are ready, you'll be able to click on "algo" at the top of a community page and see exactly how it works. If you don't like it, you can fork it to your own community. The homepage is just another community, except one that I control. Don't like it? Don't use it.
+        <b>Feeds.</b> Oddbean is built around the concept of a "feeds", which are lists of nostr events. Feeds can be curated by people, by algorithms, or both (<a href="https://www.tbray.org/ongoing/When/202x/2022/11/28/On-Algorithms">computers don't work without algorithms</a>). The goal of Oddbean is to let you publish your own feeds according to whatever policies you or a community decide -- similar to a subreddit. A specification on how to create your own feeds is coming soon. The homepage is just a feed like any other, except one that I control. Don't like it? Don't use it.
     </p>
 
     <p>

src/apps/web/tmpls/feed/list.tmpl

@@ -1,3 +1,8 @@
-<div> @(const auto &r : ctx)
+<div> @(const auto &r : ctx.items)
     $(r)
 </div>
+
+<div class="feed-nav-links">
+    <a href="./?p=$(ctx.page + 1)">More</a> ?(ctx.n != 0)
+    <span>The end.</span> ?(ctx.n == 0)
+</div>