fried feature

src/WriterPipeline.h

@@ -184,6 +184,12 @@ struct WriterPipeline {
         validatorInbox.push_move(std::move(inp));
     }
 
+    void write(EventToWrite &&inp) {
+        totalProcessed++;
+        numLive++;
+        writerInbox.push_move(std::move(inp));
+    }
+
     void flush() {
         validatorInbox.push_move({ tao::json::null, });
         flushInbox.wait();

src/apps/dbutils/cmd_export.cpp

@@ -9,7 +9,7 @@
 static const char USAGE[] =
 R"(
     Usage:
-      export [--since=<since>] [--until=<until>] [--reverse]
+      export [--since=<since>] [--until=<until>] [--reverse] [--fried]
 )";
 
 
@@ -20,6 +20,7 @@ void cmd_export(const std::vector<std::string> &subArgs) {
     if (args["--since"]) since = args["--since"].asLong();
     if (args["--until"]) until = args["--until"].asLong();
     bool reverse = args["--reverse"].asBool();
+    bool fried = args["--fried"].asBool();
 
     Decompressor decomp;
 
@@ -34,6 +35,8 @@ void cmd_export(const std::vector<std::string> &subArgs) {
 
     exitOnSigPipe();
 
+    std::string o;
+
     env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(start), lmdb::to_sv<uint64_t>(startDup), [&](auto k, auto v) {
         if (reverse) {
             if (lmdb::from_sv<uint64_t>(k) < since) return false;
@@ -42,8 +45,23 @@ void cmd_export(const std::vector<std::string> &subArgs) {
         }
 
         auto levId = lmdb::from_sv<uint64_t>(v);
+        std::string_view json = getEventJson(txn, decomp, levId);
 
-        std::cout << getEventJson(txn, decomp, levId) << "\n";
+        if (fried) {
+            auto ev = lookupEventByLevId(txn, levId);
+
+            o.clear();
+            o.reserve(json.size() + ev.buf.size() * 2 + 100);
+            o = json;
+            o.resize(o.size() - 1);
+            o += ",\"fried\":\"";
+            o += to_hex(ev.buf);
+            o += "\"}\n";
+
+            std::cout << o;
+        } else {
+            std::cout << json << "\n";
+        }
 
         return true;
     }, reverse);

src/apps/dbutils/cmd_import.cpp

@@ -9,10 +9,29 @@
 static const char USAGE[] =
 R"(
     Usage:
-      import [--show-rejected] [--no-verify] [--debounce-millis=<debounce-millis>] [--write-batch=<write-batch>]
+      import [--show-rejected] [--no-verify] [--debounce-millis=<debounce-millis>] [--write-batch=<write-batch>] [--fried]
 )";
 
 
+
+EventToWrite parseFried(std::string &line) {
+    if (line.size() < 64) throw herr("fried too small");
+    if (!line.ends_with("\"}")) throw herr("fried parse error");
+
+    size_t i;
+    for (i = line.size() - 3; i > 0 && line[i] != '"'; i--) {}
+
+    if (!std::string_view(line).substr(0, i + 1).ends_with(",\"fried\":\"")) throw herr("fried parse error");
+
+    std::string packed = from_hex(std::string_view(line).substr(i + 1, line.size() - i - 3));
+
+    line[i - 9] = '}';
+    line.resize(i - 8);
+
+    return { std::move(packed), std::move(line), };
+}
+
+
 void cmd_import(const std::vector<std::string> &subArgs) {
     std::map<std::string, docopt::value> args = docopt::docopt(USAGE, subArgs, true, "");
 
@@ -22,6 +41,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
     if (args["--debounce-millis"]) debounceMillis = args["--debounce-millis"].asLong();
     uint64_t writeBatch = 10'000;
     if (args["--write-batch"]) writeBatch = args["--write-batch"].asLong();
+    bool fried = args["--fried"].asBool();
 
     if (noVerify) LW << "not verifying event IDs or signatures!";
 
@@ -46,16 +66,25 @@ void cmd_import(const std::vector<std::string> &subArgs) {
         std::getline(std::cin, line);
         if (!line.size()) continue;
 
-        tao::json::value evJson;
+        if (fried) {
+            try {
+                writer.write(parseFried(line));
+            } catch (std::exception &e) {
+                LW << "Unable to parse fried JSON on line " << currLine;
+                continue;
+            }
+        } else {
+            tao::json::value evJson;
 
-        try {
-            evJson = tao::json::from_string(line);
-        } catch (std::exception &e) {
-            LW << "Unable to parse JSON on line " << currLine;
-            continue;
+            try {
+                evJson = tao::json::from_string(line);
+            } catch (std::exception &e) {
+                LW << "Unable to parse JSON on line " << currLine;
+                continue;
+            }
+
+            writer.write({ std::move(evJson), });
         }
-
-        writer.write({ std::move(evJson), });
         writer.wait();
     }