nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-20 01:40:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

work on write policy plugin

Browse Source
This commit is contained in:
Doug Hoyte
2023-02-08 16:44:53 -05:00
parent 51243ce62f
commit 61f2638f88
5 changed files with 60 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/DBScan.h
View File

@ -187,7 +187,7 @@ struct DBScan {
} else {
scanState = CreatedAtScan{};
auto *state = std::get_if<CreatedAtScan>(&scanState);
indexDbi = env.dbi_Event__created_at;
indexDbi = env.dbi_Event__createdAt;
isComplete = [&, state]{
return state->done;

52
src/PluginWritePolicy.h
View File

@ -73,14 +73,15 @@ struct PluginWritePolicy {
}
}
if (!running) setupPlugin();
auto json = tao::json::from_string(jsonStr);
if (!running) {
setupPlugin();
sendLookbackEvents();
}
auto request = tao::json::value({
{ "type", "new" },
{ "event", json },
{ "receivedAt", receivedAt },
{ "event", tao::json::from_string(jsonStr) },
{ "receivedAt", receivedAt / 1000000 },
{ "sourceType", eventSourceTypeToStr(sourceType) },
{ "sourceInfo", sourceType == EventSourceType::IP4 || sourceType == EventSourceType::IP6 ? renderIP(sourceInfo) : sourceInfo },
});
@ -88,7 +89,7 @@ struct PluginWritePolicy {
std::string output = tao::json::to_string(request);
output += "\n";
::fwrite(output.data(), output.size(), 1, running->w);
if (::fwrite(output.data(), 1, output.size(), running->w) != output.size()) throw herr("error writing to plugin");
tao::json::value response;
@ -102,7 +103,8 @@ struct PluginWritePolicy {
LW << "Got unparseable line from write policy plugin: " << buf;
continue;
}
// FIXME: verify id
if (response.at("id").get_string() != request.at("event").at("id").get_string()) throw herr("id mismatch");
break;
}
@ -175,4 +177,40 @@ struct PluginWritePolicy {
running = make_unique<RunningPlugin>(pid, inPipe.saveFd(0), outPipe.saveFd(1), path);
}
void sendLookbackEvents() {
if (cfg().relay__writePolicy__lookbackSeconds == 0) return;
Decompressor decomp;
auto now = hoytech::curr_time_us();
uint64_t start = now - (cfg().relay__writePolicy__lookbackSeconds * 1'000'000);
auto txn = env.txn_ro();
env.generic_foreachFull(txn, env.dbi_Event__receivedAt, lmdb::to_sv<uint64_t>(start), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
if (lmdb::from_sv<uint64_t>(k) > now) return false;
auto ev = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));
if (!ev) throw herr("unable to look up event, corrupt DB?");
auto sourceType = (EventSourceType)ev->sourceType();
std::string_view sourceInfo = ev->sourceInfo();
auto request = tao::json::value({
{ "type", "lookback" },
{ "event", tao::json::from_string(getEventJson(txn, decomp, ev->primaryKeyId)) },
{ "receivedAt", ev->receivedAt() / 1000000 },
{ "sourceType", eventSourceTypeToStr(sourceType) },
{ "sourceInfo", sourceType == EventSourceType::IP4 || sourceType == EventSourceType::IP6 ? renderIP(sourceInfo) : sourceInfo },
});
std::string output = tao::json::to_string(request);
output += "\n";
if (::fwrite(output.data(), 1, output.size(), running->w) != output.size()) throw herr("error writing to plugin");
return true;
});
}
};

2
src/cmd_export.cpp
View File

@ -24,7 +24,7 @@ void cmd_export(const std::vector<std::string> &subArgs) {
auto txn = env.txn_ro();
env.generic_foreachFull(txn, env.dbi_Event__created_at, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
env.generic_foreachFull(txn, env.dbi_Event__createdAt, lmdb::to_sv<uint64_t>(since), lmdb::to_sv<uint64_t>(0), [&](auto k, auto v) {
if (lmdb::from_sv<uint64_t>(k) > until) return false;
auto view = env.lookup_Event(txn, lmdb::from_sv<uint64_t>(v));