quadrable node gc, clean-up writes, consolidate deletion

src/RelayCron.cpp

@@ -52,11 +52,8 @@ void RelayServer::cleanupOldEvents() {
         for (auto levId : expiredLevIds) {
             auto view = env.lookup_Event(txn, levId);
             if (!view) continue; // Deleted in between transactions
-
+            deleteEvent(txn, changes, *view);
             numDeleted++;
-            changes.del(flatEventToQuadrableKey(view->flat_nested()));
-            env.delete_Event(txn, levId);
-            env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(levId));
         }
 
         changes.apply(txn);

src/cmd_compact.cpp

@@ -4,27 +4,64 @@
 #include <docopt.h>
 #include "golpe.h"
 
+#include "render.h"
+
 
 static const char USAGE[] =
 R"(
     Usage:
-      compact <output_file>
+      compact export <output_file>
+      compact quad-gc
 )";
 
 
 void cmd_compact(const std::vector<std::string> &subArgs) {
     std::map<std::string, docopt::value> args = docopt::docopt(USAGE, subArgs, true, "");
 
-    std::string outputFile = args["<output_file>"].asString();
+    if (args["export"].asBool()) {
+        std::string outputFile = args["<output_file>"].asString();
 
-    if (outputFile == "-") {
+        if (outputFile == "-") {
-        env.copy_fd(1);
+            env.copy_fd(1);
-    } else {
+        } else {
-        if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting");
+            if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting");
 
-        auto *f = ::fopen(outputFile.c_str(), "w");
+            auto *f = ::fopen(outputFile.c_str(), "w");
-        if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno));
+            if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno));
 
-        env.copy_fd(::fileno(f));
+            env.copy_fd(::fileno(f));
+        }
+    } else if (args["quad-gc"].asBool()) {
+        quadrable::Quadrable qdb;
+        {
+            auto txn = env.txn_ro();
+            qdb.init(txn);
+        }
+        qdb.checkout("events");
+
+        quadrable::Quadrable::GarbageCollector gc(qdb);
+
+        {
+            auto txn = env.txn_ro();
+            gc.markAllHeads(txn);
+        }
+
+        {
+            auto txn = env.txn_rw();
+
+            auto stats = gc.sweep(txn);
+            /*
+            auto stats = gc.sweep(txn, [&](uint64_t nodeId){
+                quadrable::Quadrable::ParsedNode node(&qdb, txn, nodeId);
+                if (!node.isBranch()) throw herr("unexpected quadrable node type during gc: ", (int)node.nodeType);
+                return true;
+            });
+            */
+
+            txn.commit();
+
+            LI << "Total nodes: " << stats.total;
+            LI << "Collected:   " << stats.collected << " (" << renderPercent((double)stats.collected / stats.total) << ")";
+        }
     }
 }

src/events.cpp

@@ -195,7 +195,7 @@ std::string_view decodeEventPayload(lmdb::txn &txn, Decompressor &decomp, std::s
     }
 }
 
-// Return result only valid until on of: next call to getEventJson/decodeEventPayload, write on or closing of txn, or any action on decomp object
+// Return result only valid until one of: next call to getEventJson/decodeEventPayload, write to/closing of txn, or any action on decomp object
 
 std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t levId) {
     std::string_view raw;
@@ -209,12 +209,20 @@ std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t lev
 
 
 
+
+void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev) {
+    changes.del(flatEventToQuadrableKey(ev.flat_nested()));
+    env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(ev.primaryKeyId));
+    env.delete_Event(txn, ev.primaryKeyId);
+}
+
+
+
 void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs) {
     std::sort(evs.begin(), evs.end(), [](auto &a, auto &b) { return a.quadKey < b.quadKey; });
 
     auto changes = qdb.change();
-
+    std::string tmpBuf;
-    std::vector<uint64_t> levIdsToDelete;
 
     for (size_t i = 0; i < evs.size(); i++) {
         auto &ev = evs[i];
@@ -250,8 +258,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
             if (otherLevId) {
                 auto otherEv = env.lookup_Event(txn, otherLevId);
                 if (!otherEv) throw herr("missing event from index, corrupt DB?");
-                changes.del(flatEventToQuadrableKey(otherEv->flat_nested()));
+                deleteEvent(txn, changes, *otherEv);
-                levIdsToDelete.push_back(otherLevId);
             }
         }
 
@@ -262,26 +269,12 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
                     auto otherEv = lookupEventById(txn, sv(tagPair->val()));
                     if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) {
                         LI << "Deleting event. id=" << to_hex(sv(tagPair->val()));
-                        changes.del(flatEventToQuadrableKey(otherEv->flat_nested()));
+                        deleteEvent(txn, changes, *otherEv);
-                        levIdsToDelete.push_back(otherEv->primaryKeyId);
                     }
                 }
             }
         }
 
-        if (ev.status == EventWriteStatus::Pending) changes.put(ev.quadKey, "");
-    }
-
-    changes.apply(txn);
-
-    for (auto levId : levIdsToDelete) {
-        env.delete_Event(txn, levId);
-        env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(levId));
-    }
-
-    std::string tmpBuf;
-
-    for (auto &ev : evs) {
         if (ev.status == EventWriteStatus::Pending) {
             ev.levId = env.insert_Event(txn, ev.receivedAt, ev.flatStr);
 
@@ -290,7 +283,11 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
             tmpBuf += ev.jsonStr;
             env.dbi_EventPayload.put(txn, lmdb::to_sv<uint64_t>(ev.levId), tmpBuf);
 
+            changes.put(ev.quadKey, "");
+
             ev.status = EventWriteStatus::Written;
         }
     }
+
+    changes.apply(txn);
 }

src/events.h

@@ -87,3 +87,4 @@ struct EventToWrite {
 
 
 void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs);
+void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev);