diff --git a/src/WriterPipeline.h b/src/WriterPipeline.h index 8d43de0..a85108e 100644 --- a/src/WriterPipeline.h +++ b/src/WriterPipeline.h @@ -184,6 +184,12 @@ struct WriterPipeline { validatorInbox.push_move(std::move(inp)); } + void write(EventToWrite &&inp) { + totalProcessed++; + numLive++; + writerInbox.push_move(std::move(inp)); + } + void flush() { validatorInbox.push_move({ tao::json::null, }); flushInbox.wait(); diff --git a/src/apps/dbutils/cmd_export.cpp b/src/apps/dbutils/cmd_export.cpp index f3cac44..ac879e5 100644 --- a/src/apps/dbutils/cmd_export.cpp +++ b/src/apps/dbutils/cmd_export.cpp @@ -9,7 +9,7 @@ static const char USAGE[] = R"( Usage: - export [--since=] [--until=] [--reverse] + export [--since=] [--until=] [--reverse] [--fried] )"; @@ -20,6 +20,7 @@ void cmd_export(const std::vector &subArgs) { if (args["--since"]) since = args["--since"].asLong(); if (args["--until"]) until = args["--until"].asLong(); bool reverse = args["--reverse"].asBool(); + bool fried = args["--fried"].asBool(); Decompressor decomp; @@ -34,6 +35,8 @@ void cmd_export(const std::vector &subArgs) { exitOnSigPipe(); + std::string o; + env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv(start), lmdb::to_sv(startDup), [&](auto k, auto v) { if (reverse) { if (lmdb::from_sv(k) < since) return false; @@ -42,8 +45,23 @@ void cmd_export(const std::vector &subArgs) { } auto levId = lmdb::from_sv(v); + std::string_view json = getEventJson(txn, decomp, levId); - std::cout << getEventJson(txn, decomp, levId) << "\n"; + if (fried) { + auto ev = lookupEventByLevId(txn, levId); + + o.clear(); + o.reserve(json.size() + ev.buf.size() * 2 + 100); + o = json; + o.resize(o.size() - 1); + o += ",\"fried\":\""; + o += to_hex(ev.buf); + o += "\"}\n"; + + std::cout << o; + } else { + std::cout << json << "\n"; + } return true; }, reverse); diff --git a/src/apps/dbutils/cmd_import.cpp b/src/apps/dbutils/cmd_import.cpp index 1ff0b04..5144175 100644 --- a/src/apps/dbutils/cmd_import.cpp +++ b/src/apps/dbutils/cmd_import.cpp @@ -9,10 +9,29 @@ static const char USAGE[] = R"( Usage: - import [--show-rejected] [--no-verify] [--debounce-millis=] [--write-batch=] + import [--show-rejected] [--no-verify] [--debounce-millis=] [--write-batch=] [--fried] )"; + +EventToWrite parseFried(std::string &line) { + if (line.size() < 64) throw herr("fried too small"); + if (!line.ends_with("\"}")) throw herr("fried parse error"); + + size_t i; + for (i = line.size() - 3; i > 0 && line[i] != '"'; i--) {} + + if (!std::string_view(line).substr(0, i + 1).ends_with(",\"fried\":\"")) throw herr("fried parse error"); + + std::string packed = from_hex(std::string_view(line).substr(i + 1, line.size() - i - 3)); + + line[i - 9] = '}'; + line.resize(i - 8); + + return { std::move(packed), std::move(line), }; +} + + void cmd_import(const std::vector &subArgs) { std::map args = docopt::docopt(USAGE, subArgs, true, ""); @@ -22,6 +41,7 @@ void cmd_import(const std::vector &subArgs) { if (args["--debounce-millis"]) debounceMillis = args["--debounce-millis"].asLong(); uint64_t writeBatch = 10'000; if (args["--write-batch"]) writeBatch = args["--write-batch"].asLong(); + bool fried = args["--fried"].asBool(); if (noVerify) LW << "not verifying event IDs or signatures!"; @@ -46,16 +66,25 @@ void cmd_import(const std::vector &subArgs) { std::getline(std::cin, line); if (!line.size()) continue; - tao::json::value evJson; + if (fried) { + try { + writer.write(parseFried(line)); + } catch (std::exception &e) { + LW << "Unable to parse fried JSON on line " << currLine; + continue; + } + } else { + tao::json::value evJson; - try { - evJson = tao::json::from_string(line); - } catch (std::exception &e) { - LW << "Unable to parse JSON on line " << currLine; - continue; + try { + evJson = tao::json::from_string(line); + } catch (std::exception &e) { + LW << "Unable to parse JSON on line " << currLine; + continue; + } + + writer.write({ std::move(evJson), }); } - - writer.write({ std::move(evJson), }); writer.wait(); }