nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-18 17:27:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

use custom packing for indexable data: PackedEvent

Browse Source
This commit is contained in:
Doug Hoyte
2024-08-29 20:18:05 -04:00
parent 32a367738c
commit 61781f351f
18 changed files with 212 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/events.cpp
View File

@ -3,8 +3,8 @@
#include "events.h"
std::string nostrJsonToFlat(const tao::json::value &v) {
flatbuffers::FlatBufferBuilder builder; // FIXME: pre-allocate size approximately the same as orig JSON?
std::string nostrJsonToPackedEvent(const tao::json::value &v) {
PackedEventTagBuilder tagBuilder;
// Extract values from JSON, add strings to builder
@ -16,17 +16,11 @@ std::string nostrJsonToFlat(const tao::json::value &v) {
if (id.size() != 32) throw herr("unexpected id size");
if (pubkey.size() != 32) throw herr("unexpected pubkey size");
std::vector<flatbuffers::Offset<NostrIndex::TagGeneral>> tagsGeneral;
std::vector<flatbuffers::Offset<NostrIndex::TagFixed32>> tagsFixed32;
uint64_t expiration = 0;
if (isReplaceableKind(kind)) {
// Prepend virtual d-tag
tagsGeneral.emplace_back(NostrIndex::CreateTagGeneral(builder,
'd',
builder.CreateVector((uint8_t*)"", 0)
));
tagBuilder.add('d', "");
}
if (v.at("tags").get_array().size() > cfg().events__maxNumTags) throw herr("too many tags: ", v.at("tags").get_array().size());
@ -41,10 +35,7 @@ std::string nostrJsonToFlat(const tao::json::value &v) {
tagVal = from_hex(tagVal, false);
if (tagVal.size() != 32) throw herr("unexpected size for fixed-size tag");
tagsFixed32.emplace_back(NostrIndex::CreateTagFixed32(builder,
(uint8_t)tagName[0],
(NostrIndex::Fixed32Bytes*)tagVal.data()
));
tagBuilder.add(tagName[0], tagVal);
} else if (tagName == "expiration") {
if (expiration == 0) {
expiration = parseUint64(tagVal);
@ -54,41 +45,23 @@ std::string nostrJsonToFlat(const tao::json::value &v) {
if (tagVal.size() > cfg().events__maxTagValSize) throw herr("tag val too large: ", tagVal.size());
if (tagVal.size() <= MAX_INDEXED_TAG_VAL_SIZE) {
tagsGeneral.emplace_back(NostrIndex::CreateTagGeneral(builder,
(uint8_t)tagName[0],
builder.CreateVector((uint8_t*)tagVal.data(), tagVal.size())
));
tagBuilder.add(tagName[0], tagVal);
}
}
}
if (isParamReplaceableKind(kind)) {
// Append virtual d-tag
tagsGeneral.emplace_back(NostrIndex::CreateTagGeneral(builder,
'd',
builder.CreateVector((uint8_t*)"", 0)
));
tagBuilder.add('d', "");
}
if (isEphemeralKind(kind)) {
expiration = 1;
}
// Create flatbuffer
PackedEventBuilder builder(id, pubkey, created_at, kind, expiration, tagBuilder);
auto eventPtr = NostrIndex::CreateEvent(builder,
(NostrIndex::Fixed32Bytes*)id.data(),
(NostrIndex::Fixed32Bytes*)pubkey.data(),
created_at,
kind,
builder.CreateVector<flatbuffers::Offset<NostrIndex::TagGeneral>>(tagsGeneral),
builder.CreateVector<flatbuffers::Offset<NostrIndex::TagFixed32>>(tagsFixed32),
expiration
);
builder.Finish(eventPtr);
return std::string(reinterpret_cast<char*>(builder.GetBufferPointer()), builder.GetSize());
return std::move(builder.buf);
}
std::string nostrHash(const tao::json::value &origJson) {
@ -127,11 +100,11 @@ bool verifySig(secp256k1_context* ctx, std::string_view sig, std::string_view ha
);
}
void verifyNostrEvent(secp256k1_context *secpCtx, const NostrIndex::Event *flat, const tao::json::value &origJson) {
void verifyNostrEvent(secp256k1_context *secpCtx, PackedEventView packed, const tao::json::value &origJson) {
auto hash = nostrHash(origJson);
if (hash != sv(flat->id())) throw herr("bad event id");
if (hash != packed.id()) throw herr("bad event id");
bool valid = verifySig(secpCtx, from_hex(origJson.at("sig").get_string(), false), sv(flat->id()), sv(flat->pubkey()));
bool valid = verifySig(secpCtx, from_hex(origJson.at("sig").get_string(), false), packed.id(), packed.pubkey());
if (!valid) throw herr("bad signature");
}
@ -139,11 +112,11 @@ void verifyNostrEventJsonSize(std::string_view jsonStr) {
if (jsonStr.size() > cfg().events__maxEventSize) throw herr("event too large: ", jsonStr.size());
}
void verifyEventTimestamp(const NostrIndex::Event *flat) {
void verifyEventTimestamp(PackedEventView packed) {
auto now = hoytech::curr_time_s();
auto ts = flat->created_at();
auto ts = packed.created_at();
uint64_t earliest = now - (flat->expiration() == 1 ? cfg().events__rejectEphemeralEventsOlderThanSeconds : cfg().events__rejectEventsOlderThanSeconds);
uint64_t earliest = now - (packed.expiration() == 1 ? cfg().events__rejectEphemeralEventsOlderThanSeconds : cfg().events__rejectEventsOlderThanSeconds);
uint64_t latest = now + cfg().events__rejectEventsNewerThanSeconds;
// overflows
@ -153,14 +126,14 @@ void verifyEventTimestamp(const NostrIndex::Event *flat) {
if (ts < earliest) throw herr("created_at too early");
if (ts > latest) throw herr("created_at too late");
if (flat->expiration() > 1 && flat->expiration() <= now) throw herr("event expired");
if (packed.expiration() > 1 && packed.expiration() <= now) throw herr("event expired");
}
void parseAndVerifyEvent(const tao::json::value &origJson, secp256k1_context *secpCtx, bool verifyMsg, bool verifyTime, std::string &flatStr, std::string &jsonStr) {
flatStr = nostrJsonToFlat(origJson);
auto *flat = flatbuffers::GetRoot<NostrIndex::Event>(flatStr.data());
if (verifyTime) verifyEventTimestamp(flat);
if (verifyMsg) verifyNostrEvent(secpCtx, flat, origJson);
void parseAndVerifyEvent(const tao::json::value &origJson, secp256k1_context *secpCtx, bool verifyMsg, bool verifyTime, std::string &packedStr, std::string &jsonStr) {
packedStr = nostrJsonToPackedEvent(origJson);
PackedEventView packed(packedStr);
if (verifyTime) verifyEventTimestamp(packed);
if (verifyMsg) verifyNostrEvent(secpCtx, packed, origJson);
// Build new object to remove unknown top-level fields from json
jsonStr = tao::json::to_string(tao::json::value({
@ -276,14 +249,14 @@ void writeEvents(lmdb::txn &txn, std::vector<EventToWrite> &evs, uint64_t logLev
for (size_t i = 0; i < evs.size(); i++) {
auto &ev = evs[i];
const NostrIndex::Event *flat = flatbuffers::GetRoot<NostrIndex::Event>(ev.flatStr.data());
PackedEventView packed(ev.packedStr);
if (lookupEventById(txn, sv(flat->id())) || (i != 0 && ev.id() == evs[i-1].id())) {
if (lookupEventById(txn, packed.id()) || (i != 0 && ev.id() == evs[i-1].id())) {
ev.status = EventWriteStatus::Duplicate;
continue;
}
if (env.lookup_Event__deletion(txn, std::string(sv(flat->id())) + std::string(sv(flat->pubkey())))) {
if (env.lookup_Event__deletion(txn, std::string(packed.id()) + std::string(packed.pubkey()))) {
ev.status = EventWriteStatus::Deleted;
continue;
}
@ -291,30 +264,30 @@ void writeEvents(lmdb::txn &txn, std::vector<EventToWrite> &evs, uint64_t logLev
{
std::optional<std::string> replace;
if (isReplaceableKind(flat->kind()) || isParamReplaceableKind(flat->kind())) {
for (const auto &tagPair : *(flat->tagsGeneral())) {
auto tagName = (char)tagPair->key();
if (tagName != 'd') continue;
replace = std::string(sv(tagPair->val()));
break;
}
if (isReplaceableKind(packed.kind()) || isParamReplaceableKind(packed.kind())) {
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName != 'd') return true;
replace = std::string(tagVal);
return false;
});
}
if (replace) {
auto searchStr = std::string(sv(flat->pubkey())) + *replace;
auto searchKey = makeKey_StringUint64(searchStr, flat->kind());
auto searchStr = std::string(packed.pubkey()) + *replace;
auto searchKey = makeKey_StringUint64(searchStr, packed.kind());
env.generic_foreachFull(txn, env.dbi_Event__replace, searchKey, lmdb::to_sv<uint64_t>(MAX_U64), [&](auto k, auto v) {
ParsedKey_StringUint64 parsedKey(k);
if (parsedKey.s == searchStr && parsedKey.n == flat->kind()) {
if (parsedKey.s == searchStr && parsedKey.n == packed.kind()) {
auto otherEv = lookupEventByLevId(txn, lmdb::from_sv<uint64_t>(v));
auto thisTimestamp = flat->created_at();
auto otherTimestamp = otherEv.flat_nested()->created_at();
auto thisTimestamp = packed.created_at();
auto otherPacked = PackedEventView(otherEv.packed());
auto otherTimestamp = otherPacked.created_at();
if (otherTimestamp < thisTimestamp ||
(otherTimestamp == thisTimestamp && sv(flat->id()) < sv(otherEv.flat_nested()->id()))) {
if (logLevel >= 1) LI << "Deleting event (d-tag). id=" << to_hex(sv(otherEv.flat_nested()->id()));
(otherTimestamp == thisTimestamp && packed.id() < otherPacked.id())) {
if (logLevel >= 1) LI << "Deleting event (d-tag). id=" << to_hex(otherPacked.id());
levIdsToDelete.push_back(otherEv.primaryKeyId);
} else {
ev.status = EventWriteStatus::Replaced;
@ -326,21 +299,22 @@ void writeEvents(lmdb::txn &txn, std::vector<EventToWrite> &evs, uint64_t logLev
}
}
if (flat->kind() == 5) {
if (packed.kind() == 5) {
// Deletion event, delete all referenced events
for (const auto &tagPair : *(flat->tagsFixed32())) {
if (tagPair->key() == 'e') {
auto otherEv = lookupEventById(txn, sv(tagPair->val()));
if (otherEv && sv(otherEv->flat_nested()->pubkey()) == sv(flat->pubkey())) {
if (logLevel >= 1) LI << "Deleting event (kind 5). id=" << to_hex(sv(tagPair->val()));
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName == 'e') {
auto otherEv = lookupEventById(txn, tagVal);
if (otherEv && PackedEventView(otherEv->packed()).pubkey() == packed.pubkey()) {
if (logLevel >= 1) LI << "Deleting event (kind 5). id=" << to_hex(tagVal);
levIdsToDelete.push_back(otherEv->primaryKeyId);
}
}
}
return true;
});
}
if (ev.status == EventWriteStatus::Pending) {
ev.levId = env.insert_Event(txn, ev.receivedAt, ev.flatStr, (uint64_t)ev.sourceType, ev.sourceInfo);
ev.levId = env.insert_Event(txn, ev.receivedAt, ev.packedStr, (uint64_t)ev.sourceType, ev.sourceInfo);
tmpBuf.clear();
tmpBuf += '\x00';