From 485abee8edfbd440645ce382abcedf3f3645917c Mon Sep 17 00:00:00 2001 From: Doug Hoyte Date: Wed, 25 Jan 2023 15:16:39 -0500 Subject: [PATCH] quadrable node gc, clean-up writes, consolidate deletion --- golpe | 2 +- src/RelayCron.cpp | 5 +---- src/cmd_compact.cpp | 55 +++++++++++++++++++++++++++++++++++++-------- src/events.cpp | 37 ++++++++++++++---------------- src/events.h | 1 + 5 files changed, 66 insertions(+), 34 deletions(-) diff --git a/golpe b/golpe index 9d0519c..a655f8f 160000 --- a/golpe +++ b/golpe @@ -1 +1 @@ -Subproject commit 9d0519c1b019577dae72e1c0f58bce276ba769cd +Subproject commit a655f8f5b2dc90034ad62095f111ec2635a4f000 diff --git a/src/RelayCron.cpp b/src/RelayCron.cpp index 1f11cd0..ef1d80c 100644 --- a/src/RelayCron.cpp +++ b/src/RelayCron.cpp @@ -52,11 +52,8 @@ void RelayServer::cleanupOldEvents() { for (auto levId : expiredLevIds) { auto view = env.lookup_Event(txn, levId); if (!view) continue; // Deleted in between transactions - + deleteEvent(txn, changes, *view); numDeleted++; - changes.del(flatEventToQuadrableKey(view->flat_nested())); - env.delete_Event(txn, levId); - env.dbi_EventPayload.del(txn, lmdb::to_sv(levId)); } changes.apply(txn); diff --git a/src/cmd_compact.cpp b/src/cmd_compact.cpp index 11604f6..efffdb1 100644 --- a/src/cmd_compact.cpp +++ b/src/cmd_compact.cpp @@ -4,27 +4,64 @@ #include #include "golpe.h" +#include "render.h" + static const char USAGE[] = R"( Usage: - compact + compact export + compact quad-gc )"; void cmd_compact(const std::vector &subArgs) { std::map args = docopt::docopt(USAGE, subArgs, true, ""); - std::string outputFile = args[""].asString(); + if (args["export"].asBool()) { + std::string outputFile = args[""].asString(); - if (outputFile == "-") { - env.copy_fd(1); - } else { - if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting"); + if (outputFile == "-") { + env.copy_fd(1); + } else { + if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting"); - auto *f = ::fopen(outputFile.c_str(), "w"); - if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno)); + auto *f = ::fopen(outputFile.c_str(), "w"); + if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno)); - env.copy_fd(::fileno(f)); + env.copy_fd(::fileno(f)); + } + } else if (args["quad-gc"].asBool()) { + quadrable::Quadrable qdb; + { + auto txn = env.txn_ro(); + qdb.init(txn); + } + qdb.checkout("events"); + + quadrable::Quadrable::GarbageCollector gc(qdb); + + { + auto txn = env.txn_ro(); + gc.markAllHeads(txn); + } + + { + auto txn = env.txn_rw(); + + auto stats = gc.sweep(txn); + /* + auto stats = gc.sweep(txn, [&](uint64_t nodeId){ + quadrable::Quadrable::ParsedNode node(&qdb, txn, nodeId); + if (!node.isBranch()) throw herr("unexpected quadrable node type during gc: ", (int)node.nodeType); + return true; + }); + */ + + txn.commit(); + + LI << "Total nodes: " << stats.total; + LI << "Collected: " << stats.collected << " (" << renderPercent((double)stats.collected / stats.total) << ")"; + } } } diff --git a/src/events.cpp b/src/events.cpp index 80364e6..a7403fc 100644 --- a/src/events.cpp +++ b/src/events.cpp @@ -195,7 +195,7 @@ std::string_view decodeEventPayload(lmdb::txn &txn, Decompressor &decomp, std::s } } -// Return result only valid until on of: next call to getEventJson/decodeEventPayload, write on or closing of txn, or any action on decomp object +// Return result only valid until one of: next call to getEventJson/decodeEventPayload, write to/closing of txn, or any action on decomp object std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t levId) { std::string_view raw; @@ -209,12 +209,20 @@ std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t lev + +void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev) { + changes.del(flatEventToQuadrableKey(ev.flat_nested())); + env.dbi_EventPayload.del(txn, lmdb::to_sv(ev.primaryKeyId)); + env.delete_Event(txn, ev.primaryKeyId); +} + + + void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector &evs) { std::sort(evs.begin(), evs.end(), [](auto &a, auto &b) { return a.quadKey < b.quadKey; }); auto changes = qdb.change(); - - std::vector levIdsToDelete; + std::string tmpBuf; for (size_t i = 0; i < evs.size(); i++) { auto &ev = evs[i]; @@ -250,8 +258,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vectorflat_nested())); - levIdsToDelete.push_back(otherLevId); + deleteEvent(txn, changes, *otherEv); } } @@ -262,26 +269,12 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vectorval())); if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) { LI << "Deleting event. id=" << to_hex(sv(tagPair->val())); - changes.del(flatEventToQuadrableKey(otherEv->flat_nested())); - levIdsToDelete.push_back(otherEv->primaryKeyId); + deleteEvent(txn, changes, *otherEv); } } } } - if (ev.status == EventWriteStatus::Pending) changes.put(ev.quadKey, ""); - } - - changes.apply(txn); - - for (auto levId : levIdsToDelete) { - env.delete_Event(txn, levId); - env.dbi_EventPayload.del(txn, lmdb::to_sv(levId)); - } - - std::string tmpBuf; - - for (auto &ev : evs) { if (ev.status == EventWriteStatus::Pending) { ev.levId = env.insert_Event(txn, ev.receivedAt, ev.flatStr); @@ -290,7 +283,11 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector(ev.levId), tmpBuf); + changes.put(ev.quadKey, ""); + ev.status = EventWriteStatus::Written; } } + + changes.apply(txn); } diff --git a/src/events.h b/src/events.h index b87b49b..a8f5eab 100644 --- a/src/events.h +++ b/src/events.h @@ -87,3 +87,4 @@ struct EventToWrite { void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector &evs); +void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev);