nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-17 16:58:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

ensure you can export a version 0 DB using a new binary

Browse Source
This commit is contained in:
Doug Hoyte
2023-02-09 04:31:31 -05:00
parent cdb25aed45
commit 3d4b8853f1
11 changed files with 44 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
TODO
View File

@ -1,5 +1,4 @@
0.1 release
ensure DB upgrade flow works
when disk is full it should log warning but not crash
disable sync

6
golpe.yaml
View File

@ -31,7 +31,7 @@ tables:
type: ubytes
indices:
createdAt:
created_at:
integer: true
receivedAt:
integer: true
@ -56,8 +56,8 @@ tables:
indexPrelude: |
auto *flat = v.flat_nested();
createdAt = flat->created_at();
uint64_t indexTime = *createdAt;
created_at = flat->created_at();
uint64_t indexTime = *created_at;
receivedAt = v.receivedAt();
id = makeKey_StringUint64(sv(flat->id()), indexTime);

2
src/DBScan.h
View File

@ -187,7 +187,7 @@ struct DBScan {
} else {
scanState = CreatedAtScan{};
auto *state = std::get_if<CreatedAtScan>(&scanState);
indexDbi = env.dbi_Event__createdAt;
indexDbi = env.dbi_Event__created_at;
isComplete = [&, state]{
return state->done;

13
src/cmd_export.cpp
View File

@ -24,12 +24,23 @@ void cmd_export(const std::vector<std::string> &subArgs) {
auto txn = env.txn_ro();
env.generic_foreachFull(txn, env.dbi_Event__createdAt, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
auto dbVersion = getDBVersion(txn);
auto qdb = getQdbInstance(txn);
env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
if (lmdb::from_sv<uint64_t>(k) > until) return false;
auto view = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));
if (!view) throw herr("missing event from index, corrupt DB?");
if (dbVersion == 0) {
std::string_view raw;
bool found = qdb.dbi_nodesLeaf.get(txn, lmdb::to_sv<uint64_t>(view->primaryKeyId), raw);
if (!found) throw herr("couldn't find leaf node in quadrable, corrupted DB?");
std::cout << raw.substr(8 + 32 + 32) << "\n";
return true;
}
if (!args["--include-ephemeral"].asBool()) {
if (isEphemeralEvent(view->flat_nested()->kind())) return true;
}

2
src/cmd_import.cpp
View File

@ -39,7 +39,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
};
auto flushChanges = [&]{
writeEvents(txn, qdb, newEvents);
writeEvents(txn, qdb, newEvents, 0);
uint64_t numCommits = 0;

14
src/cmd_info.cpp
View File

@ -18,18 +18,6 @@ void cmd_info(const std::vector<std::string> &subArgs) {
auto txn = env.txn_ro();
uint64_t dbVersion;
{
auto s = env.lookup_Meta(txn, 1);
if (s) {
dbVersion = s->dbVersion();
} else {
dbVersion = 0;
}
}
std::cout << "DB version: " << dbVersion << "\n";
std::cout << "DB version: " << getDBVersion(txn) << "\n";
std::cout << "merkle root: " << to_hex(qdb.root(txn)) << "\n";
}

9
src/events.cpp
View File

@ -230,7 +230,7 @@ void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defau
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs) {
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs, uint64_t logLevel) {
std::sort(evs.begin(), evs.end(), [](auto &a, auto &b) { return a.quadKey < b.quadKey; });
auto changes = qdb.change();
@ -260,7 +260,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
if (parsedKey.n2 < flat->created_at()) {
auto otherEv = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));
if (!otherEv) throw herr("missing event from index, corrupt DB?");
LI << "Deleting event (replaceable). id=" << to_hex(sv(otherEv->flat_nested()->pubkey()));
if (logLevel >= 1) LI << "Deleting event (replaceable). id=" << to_hex(sv(otherEv->flat_nested()->id()));
deleteEvent(txn, changes, *otherEv);
} else {
ev.status = EventWriteStatus::Replaced;
@ -281,7 +281,6 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
if (replace.size()) {
auto searchStr = std::string(sv(flat->pubkey())) + replace;
auto searchKey = makeKey_StringUint64(searchStr, flat->kind());
LI << to_hex(searchKey);
env.generic_foreachFull(txn, env.dbi_Event__replace, searchKey, lmdb::to_sv<uint64_t>(MAX_U64), [&](auto k, auto v) {
ParsedKey_StringUint64 parsedKey(k);
@ -290,7 +289,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
if (!otherEv) throw herr("missing event from index, corrupt DB?");
if (otherEv->flat_nested()->created_at() < flat->created_at()) {
LI << "Deleting event (d-tag). id=" << to_hex(sv(otherEv->flat_nested()->pubkey()));
if (logLevel >= 1) LI << "Deleting event (d-tag). id=" << to_hex(sv(otherEv->flat_nested()->id()));
deleteEvent(txn, changes, *otherEv);
} else {
ev.status = EventWriteStatus::Replaced;
@ -308,7 +307,7 @@ void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToW
if (tagPair->key() == 'e') {
auto otherEv = lookupEventById(txn, sv(tagPair->val()));
if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) {
LI << "Deleting event (kind 5). id=" << to_hex(sv(tagPair->val()));
if (logLevel >= 1) LI << "Deleting event (kind 5). id=" << to_hex(sv(tagPair->val()));
deleteEvent(txn, changes, *otherEv);
}
}

2
src/events.h
View File

@ -107,5 +107,5 @@ struct EventToWrite {
};
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs);
void writeEvents(lmdb::txn &txn, quadrable::Quadrable &qdb, std::vector<EventToWrite> &evs, uint64_t logLevel = 1);
void deleteEvent(lmdb::txn &txn, quadrable::Quadrable::UpdateSet &changes, defaultDb::environment::View_Event &ev);

1
src/global.h
View File

@ -20,3 +20,4 @@ std::string renderSize(uint64_t si);
std::string renderPercent(double p);
uint64_t parseUint64(const std::string &s);
std::string parseIP(const std::string &ip);
uint64_t getDBVersion(lmdb::txn &txn);

18
src/misc.cpp
View File

@ -85,3 +85,21 @@ uint64_t parseUint64(const std::string &s) {
return std::stoull(s);
}
uint64_t getDBVersion(lmdb::txn &txn) {
uint64_t dbVersion;
{
auto s = env.lookup_Meta(txn, 1);
if (s) {
dbVersion = s->dbVersion();
} else {
dbVersion = 0;
}
}
return dbVersion;
}

4
src/onAppStartup.cpp
View File

@ -32,7 +32,7 @@ static void dbCheck(lmdb::txn &txn, const std::string &cmd) {
return false;
});
if (cmd == "export") return;
if (cmd == "export" || cmd == "info") return;
if (eventFound) dbTooOld(0);
}
@ -43,7 +43,7 @@ static void dbCheck(lmdb::txn &txn, const std::string &cmd) {
if (s->endianness() != 1) throw herr("DB was created on a machine with different endianness");
if (s->dbVersion() < CURR_DB_VERSION) {
if (cmd == "export") return;
if (cmd == "export" || cmd == "info") return;
dbTooOld(s->dbVersion());
}