nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-16 16:28:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

quadrable node gc, clean-up writes, consolidate deletion

Browse Source
This commit is contained in:
Doug Hoyte
2023-01-25 15:16:39 -05:00
parent eac2095c83
commit 485abee8ed
5 changed files with 66 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
golpe

Submodule golpe updated: 9d0519c1b0...a655f8f5b2

5
src/RelayCron.cpp
View File

@ -52,11 +52,8 @@ void RelayServer::cleanupOldEvents() {
for (auto levId : expiredLevIds) {
auto view = env.lookup_Event(txn, levId);
if (!view) continue; // Deleted in between transactions
deleteEvent(txn, changes, *view);
numDeleted++;
changes.del(flatEventToQuadrableKey(view->flat_nested()));
env.delete_Event(txn, levId);
env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(levId));
}
changes.apply(txn);

55
src/cmd_compact.cpp
View File

@ -4,27 +4,64 @@
#include <docopt.h>
#include "golpe.h"
#include "render.h"
static const char USAGE[] =
R"(
Usage:
compact <output_file>
compact export <output_file>
compact quad-gc
)";
void cmd_compact(const std::vector<std::string> &subArgs) {
std::map<std::string, docopt::value> args = docopt::docopt(USAGE, subArgs, true, "");
std::string outputFile = args["<output_file>"].asString();
if (args["export"].asBool()) {
std::string outputFile = args["<output_file>"].asString();
if (outputFile == "-") {
env.copy_fd(1);
} else {
if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting");
if (outputFile == "-") {
env.copy_fd(1);
} else {
if (access(outputFile.c_str(), F_OK) == 0) throw herr("output file '", outputFile, "' exists, not overwriting");
auto *f = ::fopen(outputFile.c_str(), "w");
if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno));
auto *f = ::fopen(outputFile.c_str(), "w");
if (!f) throw herr("opening output file '", outputFile, "' failed: ", strerror(errno));
env.copy_fd(::fileno(f));
env.copy_fd(::fileno(f));
}
} else if (args["quad-gc"].asBool()) {
quadrable::Quadrable qdb;
{
auto txn = env.txn_ro();
qdb.init(txn);
}
qdb.checkout("events");
quadrable::Quadrable::GarbageCollector gc(qdb);
{
auto txn = env.txn_ro();
gc.markAllHeads(txn);
}
{
auto txn = env.txn_rw();
auto stats = gc.sweep(txn);
/*
auto stats = gc.sweep(txn, [&](uint64_t nodeId){
quadrable::Quadrable::ParsedNode node(&qdb, txn, nodeId);
if (!node.isBranch()) throw herr("unexpected quadrable node type during gc: ", (int)node.nodeType);
return true;
});
*/
txn.commit();
LI << "Total nodes: " << stats.total;
LI << "Collected: " << stats.collected << " (" << renderPercent((double)stats.collected / stats.total) << ")";
}
}
}

37
src/events.cpp
View File

@ -195,7 +195,7 @@ std::string_view decodeEventPayload(lmdb::txn &txn, Decompressor &decomp, std::s
}
}
// Return result only valid until on of: next call to getEventJson/decodeEventPayload, write on or closing of txn, or any action on decomp object
// Return result only valid until one of: next call to getEventJson/decodeEventPayload, write to/closing of txn, or any action on decomp object
std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t levId) {
std::string_view raw;
@ -209,12 +209,20 @@ std::string_view getEventJson(lmdb::txn &txn, Decompressor &decomp, uint64_t lev
void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev) {
changes.del(flatEventToQuadrableKey(ev.flat_nested()));
env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(ev.primaryKeyId));
env.delete_Event(txn, ev.primaryKeyId);
}
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs) {
std::sort(evs.begin(), evs.end(), [](auto &a, auto &b) { return a.quadKey < b.quadKey; });
auto changes = qdb.change();
std::vector<uint64_t> levIdsToDelete;
std::string tmpBuf;
for (size_t i = 0; i < evs.size(); i++) {
auto &ev = evs[i];
@ -250,8 +258,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
if (otherLevId) {
auto otherEv = env.lookup_Event(txn, otherLevId);
if (!otherEv) throw herr("missing event from index, corrupt DB?");
changes.del(flatEventToQuadrableKey(otherEv->flat_nested()));
levIdsToDelete.push_back(otherLevId);
deleteEvent(txn, changes, *otherEv);
}
}
@ -262,26 +269,12 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
auto otherEv = lookupEventById(txn, sv(tagPair->val()));
if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) {
LI << "Deleting event. id=" << to_hex(sv(tagPair->val()));
changes.del(flatEventToQuadrableKey(otherEv->flat_nested()));
levIdsToDelete.push_back(otherEv->primaryKeyId);
deleteEvent(txn, changes, *otherEv);
}
}
}
}
if (ev.status == EventWriteStatus::Pending) changes.put(ev.quadKey, "");
}
changes.apply(txn);
for (auto levId : levIdsToDelete) {
env.delete_Event(txn, levId);
env.dbi_EventPayload.del(txn, lmdb::to_sv<uint64_t>(levId));
}
std::string tmpBuf;
for (auto &ev : evs) {
if (ev.status == EventWriteStatus::Pending) {
ev.levId = env.insert_Event(txn, ev.receivedAt, ev.flatStr);
@ -290,7 +283,11 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
tmpBuf += ev.jsonStr;
env.dbi_EventPayload.put(txn, lmdb::to_sv<uint64_t>(ev.levId), tmpBuf);
changes.put(ev.quadKey, "");
ev.status = EventWriteStatus::Written;
}
}
changes.apply(txn);
}

1
src/events.h
View File

@ -87,3 +87,4 @@ struct EventToWrite {
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs);
void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev);