GC improvements

src/RelayCron.cpp

@@ -1,5 +1,7 @@
 #include "RelayServer.h"
 
+#include "gc.h"
+
 
 void RelayServer::cleanupOldEvents() {
     std::vector<uint64_t> expiredLevIds;
@@ -63,3 +65,14 @@ void RelayServer::cleanupOldEvents() {
         if (numDeleted) LI << "Deleted " << numDeleted << " ephemeral events";
     }
 }
+
+void RelayServer::garbageCollect() {
+    quadrable::Quadrable qdb;
+    {
+        auto txn = env.txn_ro();
+        qdb.init(txn);
+    }
+    qdb.checkout("events");
+
+    quadrableGarbageCollect(qdb, 1);
+}

src/RelayServer.h

@@ -160,6 +160,7 @@ struct RelayServer {
     void runYesstr(ThreadPool<MsgYesstr>::Thread &thr);
 
     void cleanupOldEvents();
+    void garbageCollect();
 
     // Utils (can be called by any thread)
 

src/cmd_compact.cpp

@@ -4,7 +4,7 @@
 #include <docopt.h>
 #include "golpe.h"
 
-#include "render.h"
+#include "gc.h"
 
 
 static const char USAGE[] =
@@ -39,29 +39,6 @@ void cmd_compact(const std::vector<std::string> &subArgs) {
         }
         qdb.checkout("events");
 
-        quadrable::Quadrable::GarbageCollector gc(qdb);
+        quadrableGarbageCollect(qdb, 2);
-
-        {
-            auto txn = env.txn_ro();
-            gc.markAllHeads(txn);
-        }
-
-        {
-            auto txn = env.txn_rw();
-
-            auto stats = gc.sweep(txn);
-            /*
-            auto stats = gc.sweep(txn, [&](uint64_t nodeId){
-                quadrable::Quadrable::ParsedNode node(&qdb, txn, nodeId);
-                if (!node.isBranch()) throw herr("unexpected quadrable node type during gc: ", (int)node.nodeType);
-                return true;
-            });
-            */
-
-            txn.commit();
-
-            LI << "Total nodes: " << stats.total;
-            LI << "Collected:   " << stats.collected << " (" << renderPercent((double)stats.collected / stats.total) << ")";
-        }
     }
 }

src/cmd_import.cpp

@@ -5,6 +5,7 @@
 
 #include "events.h"
 #include "filters.h"
+#include "gc.h"
 
 
 static const char USAGE[] =
@@ -59,6 +60,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
 
         logStatus();
         LI << "Committing " << numCommits << " records";
+
         txn.commit();
 
         txn = env.txn_rw();
@@ -91,5 +93,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
 
     flushChanges();
 
+    quadrableGarbageCollect(qdb, 2);
+
     txn.commit();
 }

src/cmd_relay.cpp

@@ -48,6 +48,10 @@ void RelayServer::run() {
         cleanupOldEvents();
     });
 
+    cron.repeat(60 * 60 * 1'000'000UL, [&]{
+        garbageCollect();
+    });
+
     cron.setupCb = []{ setThreadName("cron"); };
 
     cron.run();

src/gc.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <parallel_hashmap/phmap.h>
+
+#include "golpe.h"
+
+#include "render.h"
+
+
+inline void quadrableGarbageCollect(quadrable::Quadrable &qdb, int logLevel = 0) {
+    quadrable::Quadrable::GarbageCollector<phmap::flat_hash_set<uint64_t>> gc(qdb);
+    quadrable::Quadrable::GCStats stats;
+
+    if (logLevel >= 2) LI << "Running garbage collection";
+
+    {
+        auto txn = env.txn_ro();
+
+        if (logLevel >= 2) LI << "GC: mark phase";
+        gc.markAllHeads(txn);
+        if (logLevel >= 2) LI << "GC: sweep phase";
+        stats = gc.sweep(txn);
+    }
+
+    if (logLevel >= 2) {
+        LI << "GC: Total nodes:   " << stats.total;
+        LI << "GC: Garbage nodes: " << stats.garbage << " (" << renderPercent((double)stats.garbage / stats.total) << ")";
+    }
+
+    if (stats.garbage) {
+        auto txn = env.txn_rw();
+        if (logLevel >= 1) LI << "GC: deleting " << stats.garbage << " garbage nodes";
+        gc.deleteNodes(txn);
+        txn.commit();
+    }
+
+}