nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-20 17:59:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use Bytes32 instead of std::string where possible, to reduce memory usage

Browse Source
This commit is contained in:
Doug Hoyte
2024-09-05 15:12:40 -04:00
parent 55fa4dc032
commit 169e633a08
7 changed files with 82 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/apps/dbutils/cmd_negentropy.cpp
View File

@ -3,6 +3,7 @@
#include <docopt.h>
#include "golpe.h"
#include "Bytes32.h"
#include "NegentropyFilterCache.h"
#include "events.h"
#include "DBQuery.h"
@ -74,7 +75,7 @@ void cmd_negentropy(const std::vector<std::string> &subArgs) {
struct Record {
uint64_t created_at;
uint8_t id[32];
Bytes32 id;
};
std::vector<Record> recs;
@ -100,8 +101,8 @@ void cmd_negentropy(const std::vector<std::string> &subArgs) {
bool complete = query.process(txn, [&](const auto &sub, uint64_t levId){
auto ev = lookupEventByLevId(txn, levId);
auto packed = PackedEventView(ev.buf);
recs.push_back({ packed.created_at(), });
memcpy(recs.back().id, packed.id().data(), 32);
recs.emplace_back(packed.created_at(), packed.id());
//memcpy(recs.back().id, packed.id().data(), 32);
});
if (complete) break;
@ -112,7 +113,7 @@ void cmd_negentropy(const std::vector<std::string> &subArgs) {
negentropy::storage::BTreeLMDB storage(txn, negentropyDbi, treeId);
for (const auto &r : recs) {
storage.insert(r.created_at, std::string_view((char*)r.id, 32));
storage.insert(r.created_at, r.id.sv());
}
storage.flush();

5
src/apps/mesh/cmd_stream.cpp
View File

@ -5,6 +5,7 @@
#include "golpe.h"
#include "Bytes32.h"
#include "WriterPipeline.h"
#include "Subscription.h"
#include "WSConnection.h"
@ -31,7 +32,7 @@ void cmd_stream(const std::vector<std::string> &subArgs) {
if (dir != "up" && dir != "down" && dir != "both") throw herr("invalid direction: ", dir, ". Should be one of up/down/both");
flat_hash_set<std::string> downloadedIds;
flat_hash_set<Bytes32> downloadedIds;
WriterPipeline writer;
WSConnection ws(url);
Decompressor decomp;
@ -101,7 +102,7 @@ void cmd_stream(const std::vector<std::string> &subArgs) {
env.foreach_Event(txn, [&](auto &ev){
currEventId = ev.primaryKeyId;
auto id = std::string(PackedEventView(ev.buf).id());
Bytes32 id(PackedEventView(ev.buf).id());
if (downloadedIds.find(id) != downloadedIds.end()) {
downloadedIds.erase(id);
return true;

19
src/apps/mesh/cmd_sync.cpp
View File

@ -7,6 +7,7 @@
#include "golpe.h"
#include "Bytes32.h"
#include "WriterPipeline.h"
#include "Subscription.h"
#include "WSConnection.h"
@ -150,8 +151,8 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
const uint64_t batchSizeDown = 50;
uint64_t inFlightUp = 0;
bool inFlightDown = false; // bool because we can't count on getting every EVENT we request (might've been deleted mid-query)
std::vector<std::string> have, need;
flat_hash_set<std::string> seenHave, seenNeed;
std::vector<Bytes32> have, need;
flat_hash_set<Bytes32> seenHave, seenNeed;
bool syncDone = false;
uint64_t totalHaves = 0, totalNeeds = 0;
Decompressor decomp;
@ -186,16 +187,18 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
neMsg = ne.reconcile(inputMsg, currHave, currNeed);
}
for (auto &id : currHave) {
for (auto &idStr : currHave) {
Bytes32 id(idStr);
if (seenHave.contains(id)) continue;
seenHave.insert(id);
have.push_back(std::move(id));
have.push_back(id);
}
for (auto &id : currNeed) {
for (auto &idStr : currNeed) {
Bytes32 id(idStr);
if (seenNeed.contains(id)) continue;
seenNeed.insert(id);
need.push_back(std::move(id));
need.push_back(id);
}
} catch (std::exception &e) {
LE << "Unable to parse negentropy message from relay: " << e.what();
@ -263,7 +266,7 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
auto id = std::move(have.back());
have.pop_back();
auto ev = lookupEventById(txn, id);
auto ev = lookupEventById(txn, id.sv());
if (!ev) {
LW << "Couldn't upload event because not found (deleted?)";
continue;
@ -285,7 +288,7 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
tao::json::value ids = tao::json::empty_array;
while (need.size() > 0 && ids.get_array().size() < batchSizeDown) {
ids.emplace_back(to_hex(need.back()));
ids.emplace_back(to_hex(need.back().sv()));
need.pop_back();
}