ensure you can export a version 0 DB using a new binary

TODO

@@ -1,5 +1,4 @@
 0.1 release
-  ensure DB upgrade flow works
   when disk is full it should log warning but not crash
   disable sync
 

golpe.yaml

@@ -31,7 +31,7 @@ tables:
         type: ubytes
 
     indices:
-      createdAt:
+      created_at:
         integer: true
       receivedAt:
         integer: true
@@ -56,8 +56,8 @@ tables:
 
     indexPrelude: |
         auto *flat = v.flat_nested();
-        createdAt = flat->created_at();
+        created_at = flat->created_at();
-        uint64_t indexTime = *createdAt;
+        uint64_t indexTime = *created_at;
         receivedAt = v.receivedAt();
 
         id = makeKey_StringUint64(sv(flat->id()), indexTime);

src/DBScan.h

@@ -187,7 +187,7 @@ struct DBScan {
         } else {
             scanState = CreatedAtScan{};
             auto *state = std::get_if<CreatedAtScan>(&scanState);
-            indexDbi = env.dbi_Event__createdAt;
+            indexDbi = env.dbi_Event__created_at;
 
             isComplete = [&, state]{
                 return state->done;

src/cmd_export.cpp

@@ -24,12 +24,23 @@ void cmd_export(const std::vector<std::string> &subArgs) {
 
     auto txn = env.txn_ro();
 
-    env.generic_foreachFull(txn, env.dbi_Event__createdAt, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
+    auto dbVersion = getDBVersion(txn);
+    auto qdb = getQdbInstance(txn);
+
+    env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
         if (lmdb::from_sv<uint64_t>(k) > until) return false;
 
         auto view = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));
         if (!view) throw herr("missing event from index, corrupt DB?");
 
+        if (dbVersion == 0) {
+            std::string_view raw;
+            bool found = qdb.dbi_nodesLeaf.get(txn, lmdb::to_sv<uint64_t>(view->primaryKeyId), raw);
+            if (!found) throw herr("couldn't find leaf node in quadrable, corrupted DB?");
+            std::cout << raw.substr(8 + 32 + 32) << "\n";
+            return true;
+        }
+
         if (!args["--include-ephemeral"].asBool()) {
             if (isEphemeralEvent(view->flat_nested()->kind())) return true;
         }

src/cmd_import.cpp

@@ -39,7 +39,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
     };
 
     auto flushChanges = [&]{
-        writeEvents(txn, qdb, newEvents);
+        writeEvents(txn, qdb, newEvents, 0);
 
         uint64_t numCommits = 0;
 

src/cmd_info.cpp

@@ -18,18 +18,6 @@ void cmd_info(const std::vector<std::string> &subArgs) {
 
     auto txn = env.txn_ro();
 
-    uint64_t dbVersion;
+    std::cout << "DB version: " << getDBVersion(txn) << "\n";
-
-    {
-        auto s = env.lookup_Meta(txn, 1);
-
-        if (s) {
-            dbVersion = s->dbVersion();
-        } else {
-            dbVersion = 0;
-        }
-    }
-
-    std::cout << "DB version: " << dbVersion << "\n";
     std::cout << "merkle root: " << to_hex(qdb.root(txn)) << "\n";
 }

src/events.cpp

@@ -230,7 +230,7 @@ void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defau
 
 
 
-void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs) {
+void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs, uint64_t logLevel) {
     std::sort(evs.begin(), evs.end(), [](auto &a, auto &b) { return a.quadKey < b.quadKey; });
 
     auto changes = qdb.change();
@@ -260,7 +260,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
                     if (parsedKey.n2 < flat->created_at()) {
                         auto otherEv = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));
                         if (!otherEv) throw herr("missing event from index, corrupt DB?");
-                        LI << "Deleting event (replaceable). id=" << to_hex(sv(otherEv->flat_nested()->pubkey()));
+                        if (logLevel >= 1) LI << "Deleting event (replaceable). id=" << to_hex(sv(otherEv->flat_nested()->id()));
                         deleteEvent(txn, changes, *otherEv);
                     } else {
                         ev.status = EventWriteStatus::Replaced;
@@ -281,7 +281,6 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
             if (replace.size()) {
                 auto searchStr = std::string(sv(flat->pubkey())) + replace;
                 auto searchKey = makeKey_StringUint64(searchStr, flat->kind());
-                LI << to_hex(searchKey);
 
                 env.generic_foreachFull(txn, env.dbi_Event__replace, searchKey, lmdb::to_sv<uint64_t>(MAX_U64), [&](auto k, auto v) {
                     ParsedKey_StringUint64 parsedKey(k);
@@ -290,7 +289,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
                         if (!otherEv) throw herr("missing event from index, corrupt DB?");
 
                         if (otherEv->flat_nested()->created_at() < flat->created_at()) {
-                            LI << "Deleting event (d-tag). id=" << to_hex(sv(otherEv->flat_nested()->pubkey()));
+                            if (logLevel >= 1) LI << "Deleting event (d-tag). id=" << to_hex(sv(otherEv->flat_nested()->id()));
                             deleteEvent(txn, changes, *otherEv);
                         } else {
                             ev.status = EventWriteStatus::Replaced;
@@ -308,7 +307,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
                 if (tagPair->key() == 'e') {
                     auto otherEv = lookupEventById(txn, sv(tagPair->val()));
                     if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) {
-                        LI << "Deleting event (kind 5). id=" << to_hex(sv(tagPair->val()));
+                        if (logLevel >= 1) LI << "Deleting event (kind 5). id=" << to_hex(sv(tagPair->val()));
                         deleteEvent(txn, changes, *otherEv);
                     }
                 }

src/events.h

@@ -107,5 +107,5 @@ struct EventToWrite {
 };
 
 
-void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs);
+void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs, uint64_t logLevel = 1);
 void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev);

src/global.h

@@ -20,3 +20,4 @@ std::string renderSize(uint64_t si);
 std::string renderPercent(double p);
 uint64_t parseUint64(const std::string &s);
 std::string parseIP(const std::string &ip);
+uint64_t getDBVersion(lmdb::txn &txn);

src/misc.cpp

@@ -85,3 +85,21 @@ uint64_t parseUint64(const std::string &s) {
 
     return std::stoull(s);
 }
+
+
+
+uint64_t getDBVersion(lmdb::txn &txn) {
+    uint64_t dbVersion;
+
+    {
+        auto s = env.lookup_Meta(txn, 1);
+
+        if (s) {
+            dbVersion = s->dbVersion();
+        } else {
+            dbVersion = 0;
+        }
+    }
+
+    return dbVersion;
+}

src/onAppStartup.cpp

@@ -32,7 +32,7 @@ static void dbCheck(lmdb::txn &txn, const std::string &cmd) {
                 return false;
             });
 
-            if (cmd == "export") return;
+            if (cmd == "export" || cmd == "info") return;
             if (eventFound) dbTooOld(0);
         }
 
@@ -43,7 +43,7 @@ static void dbCheck(lmdb::txn &txn, const std::string &cmd) {
     if (s->endianness() != 1) throw herr("DB was created on a machine with different endianness");
 
     if (s->dbVersion() < CURR_DB_VERSION) {
-        if (cmd == "export") return;
+        if (cmd == "export" || cmd == "info") return;
         dbTooOld(s->dbVersion());
     }