sync optimisations, DBQuery no longer loads eventPayload

- It is now up to the caller to do so
- QueryScheduler now can optionally not bother to ensure that the events are fresh

src/apps/dbutils/cmd_delete.cpp

@@ -50,7 +50,7 @@ void cmd_delete(const std::vector<std::string> &subArgs) {
         auto txn = env.txn_ro();
 
         while (1) {
-            bool complete = query.process(txn, [&](const auto &sub, uint64_t levId, std::string_view){
+            bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){
                 levIds.insert(levId);
             });
 

src/apps/dbutils/cmd_dict.cpp

@@ -50,7 +50,7 @@ void cmd_dict(const std::vector<std::string> &subArgs) {
     DBQuery query(tao::json::from_string(filterStr));
 
     while (1) {
-        bool complete = query.process(txn, [&](const auto &sub, uint64_t levId, std::string_view){
+        bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){
             levIds.push_back(levId);
         });
 

src/apps/dbutils/cmd_scan.cpp

@@ -37,9 +37,9 @@ void cmd_scan(const std::vector<std::string> &subArgs) {
     exitOnSigPipe();
 
     while (1) {
-        bool complete = query.process(txn, [&](const auto &sub, uint64_t levId, std::string_view eventPayload){
+        bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){
             if (count) numEvents++;
-            else std::cout << getEventJson(txn, decomp, levId, eventPayload) << "\n";
+            else std::cout << getEventJson(txn, decomp, levId) << "\n";
         }, pause ? pause : MAX_U64, metrics);
 
         if (complete) break;

src/apps/mesh/cmd_sync.cpp

@@ -57,18 +57,24 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
         auto txn = env.txn_ro();
 
         uint64_t numEvents = 0;
+        std::vector<uint64_t> levIds;
 
         while (1) {
-            bool complete = query.process(txn, [&](const auto &sub, uint64_t levId, std::string_view eventPayload){
-                auto ev = lookupEventByLevId(txn, levId);
-                ne.addItem(ev.flat_nested()->created_at(), sv(ev.flat_nested()->id()).substr(0, ne.idSize));
-
+            bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){
+                levIds.push_back(levId);
                 numEvents++;
             });
 
             if (complete) break;
         }
 
+        std::sort(levIds.begin(), levIds.end());
+
+        for (auto levId : levIds) {
+            auto ev = lookupEventByLevId(txn, levId);
+            ne.addItem(ev.flat_nested()->created_at(), sv(ev.flat_nested()->id()).substr(0, ne.idSize));
+        }
+
         LI << "Filter matches " << numEvents << " events";
     }
 

src/apps/relay/RelayNegentropy.cpp

@@ -1,7 +1,6 @@
 #include <Negentropy.h>
 
 #include "RelayServer.h"
-#include "DBQuery.h"
 #include "QueryScheduler.h"
 
 
@@ -9,6 +8,7 @@ struct NegentropyViews {
     struct UserView {
         Negentropy ne;
         std::string initialMsg;
+        std::vector<uint64_t> levIds;
         uint64_t startTime = hoytech::curr_time_us();
     };
 
@@ -63,17 +63,18 @@ void RelayServer::runNegentropy(ThreadPool<MsgNegentropy>::Thread &thr) {
     QueryScheduler queries;
     NegentropyViews views;
 
+    queries.ensureExists = false;
+
     queries.onEventBatch = [&](lmdb::txn &txn, const auto &sub, const std::vector<uint64_t> &levIds){
         auto *view = views.findView(sub.connId, sub.subId);
         if (!view) return;
 
         for (auto levId : levIds) {
-            auto ev = lookupEventByLevId(txn, levId);
-            view->ne.addItem(ev.flat_nested()->created_at(), sv(ev.flat_nested()->id()).substr(0, view->ne.idSize));
+            view->levIds.push_back(levId);
         }
     };
 
-    queries.onComplete = [&](Subscription &sub){
+    queries.onComplete = [&](lmdb::txn &txn, Subscription &sub){
         auto *view = views.findView(sub.connId, sub.subId);
         if (!view) return;
 
@@ -94,6 +95,20 @@ void RelayServer::runNegentropy(ThreadPool<MsgNegentropy>::Thread &thr) {
             return;
         }
 
+        std::sort(view->levIds.begin(), view->levIds.end());
+
+        for (auto levId : view->levIds) {
+            try {
+                auto ev = lookupEventByLevId(txn, levId);
+                view->ne.addItem(ev.flat_nested()->created_at(), sv(ev.flat_nested()->id()).substr(0, view->ne.idSize));
+            } catch (std::exception &) {
+                // levId was deleted when query was paused
+            }
+        }
+
+        view->levIds.clear();
+        view->levIds.shrink_to_fit();
+
         view->ne.seal();
 
         auto resp = view->ne.reconcile(view->initialMsg);

src/apps/relay/RelayReqWorker.cpp

@@ -1,5 +1,4 @@
 #include "RelayServer.h"
-#include "DBQuery.h"
 #include "QueryScheduler.h"
 
 
@@ -11,7 +10,7 @@ void RelayServer::runReqWorker(ThreadPool<MsgReqWorker>::Thread &thr) {
         sendEvent(sub.connId, sub.subId, decodeEventPayload(txn, decomp, eventPayload, nullptr, nullptr));
     };
 
-    queries.onComplete = [&](Subscription &sub){
+    queries.onComplete = [&](lmdb::txn &, Subscription &sub){
         sendToConn(sub.connId, tao::json::to_string(tao::json::value::array({ "EOSE", sub.subId.str() })));
         tpReqMonitor.dispatch(sub.connId, MsgReqMonitor{MsgReqMonitor::NewSub{std::move(sub)}});
     };