nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-16 16:28:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

update for new DB format

Browse Source
This commit is contained in:
Doug Hoyte
2024-09-12 16:52:11 -04:00
parent b1781d6576
commit dda300a11e
4 changed files with 53 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/apps/web/AlgoParser.h
View File

@ -157,11 +157,13 @@ struct AlgoParseState {
if (parsedKey.s == pubkey && parsedKey.n1 == kind) {
auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);
for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() != 'p') continue;
output.insert(std::string(sv(tagPair->val())));
}
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName != 'p') return true;
output.insert(std::string(tagVal));
return true;
});
}
return false;

48
src/apps/web/AlgoScanner.h
View File

@ -36,21 +36,24 @@ struct AlgoScanner {
if (output.size() > limit) return false;
auto ev = lookupEventByLevId(txn, lmdb::from_sv<uint64_t>(v));
auto kind = ev.flat_nested()->kind();
auto id = sv(ev.flat_nested()->id());
PackedEventView packed(ev.buf);
auto kind = packed.kind();
auto id = packed.id();
if (kind == 1) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
auto pubkey = std::string(packed.pubkey());
bool foundETag = false;
for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() == 'e') {
auto tagEventId = std::string(sv(tagPair->val()));
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName == 'e') {
auto tagEventId = std::string(tagVal);
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].comments++;
foundETag = true;
}
}
return true;
});
if (foundETag) return true; // not root event
eventInfoCache.emplace(id, EventInfo{});
@ -62,18 +65,19 @@ struct AlgoScanner {
output.emplace_back(FilteredEvent{ev.primaryKeyId, std::string(id), eventInfo});
} else if (kind == 7) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
auto pubkey = std::string(packed.pubkey());
//if (a.voters && !a.voters->contains(pubkey)) return true;
const auto &tagsArr = *(ev.flat_nested()->tagsFixed32());
for (auto it = tagsArr.rbegin(); it != tagsArr.rend(); ++it) {
auto tagPair = *it;
if ((char)tagPair->key() == 'e') {
auto tagEventId = std::string(sv(tagPair->val()));
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].score++;
break;
}
std::optional<std::string_view> lastETag;
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName == 'e') lastETag = tagVal;
return true;
});
if (lastETag) {
auto tagEventId = std::string(*lastETag);
eventInfoCache.emplace(tagEventId, EventInfo{});
eventInfoCache[tagEventId].score++;
}
}
@ -97,11 +101,13 @@ struct AlgoScanner {
if (parsedKey.s == pubkey && parsedKey.n1 == kind) {
auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);
for (const auto &tagPair : *(ev.flat_nested()->tagsFixed32())) {
if ((char)tagPair->key() != 'p') continue;
output.insert(std::string(sv(tagPair->val())));
}
packed.foreachTag([&](char tagName, std::string_view tagVal){
if (tagName != 'p') return true;
output.insert(std::string(tagVal));
return true;
});
}
return false;

17
src/apps/web/WebData.h
View File

@ -142,9 +142,10 @@ struct User {
auto levId = lmdb::from_sv<uint64_t>(v);
auto ev = lookupEventByLevId(txn, levId);
PackedEventView packed(ev.buf);
if (ev.flat_nested()->kind() == 3) {
auto pubkey = std::string(sv(ev.flat_nested()->pubkey()));
if (packed.kind() == 3) {
auto pubkey = std::string(packed.pubkey());
if (!alreadySeen.contains(pubkey)) {
alreadySeen.insert(pubkey);
@ -206,19 +207,23 @@ struct Event {
std::string getId() const {
return std::string(sv(ev.flat_nested()->id()));
PackedEventView packed(ev.buf);
return std::string(packed.id());
}
uint64_t getKind() const {
return ev.flat_nested()->kind();
PackedEventView packed(ev.buf);
return packed.kind();
}
uint64_t getCreatedAt() const {
return ev.flat_nested()->created_at();
PackedEventView packed(ev.buf);
return packed.created_at();
}
std::string getPubkey() const {
return std::string(sv(ev.flat_nested()->pubkey()));
PackedEventView packed(ev.buf);
return std::string(packed.pubkey());
}
std::string getNoteId() const {

18
src/apps/web/WebWriter.cpp
View File

@ -10,16 +10,16 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
secp256k1_context *secpCtx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY);
PluginEventSifter writePolicy;
NegentropyFilterCache neFilterCache;
while(1) {
auto newMsgs = thr.inbox.pop_all();
auto now = hoytech::curr_time_us();
std::vector<EventToWrite> newEvents;
for (auto &newMsg : newMsgs) {
if (auto msg = std::get_if<MsgWebWriter::Request>(&newMsg.msg)) {
auto &req = msg->req;
EventSourceType sourceType = req.ipAddr.size() == 4 ? EventSourceType::IP4 : EventSourceType::IP6;
Url u(req.url);
if (u.path.size() != 1 || u.path[0] != "submit-post") {
@ -27,23 +27,23 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
continue;
}
std::string flatStr, jsonStr;
std::string packedStr, jsonStr;
try {
tao::json::value json = tao::json::from_string(req.body);
parseAndVerifyEvent(json, secpCtx, true, true, flatStr, jsonStr);
parseAndVerifyEvent(json, secpCtx, true, true, packedStr, jsonStr);
} catch(std::exception &e) {
sendHttpResponse(req, tao::json::to_string(tao::json::value({{ "message", e.what() }})), "404 Not Found", "application/json; charset=utf-8");
continue;
}
newEvents.emplace_back(std::move(flatStr), std::move(jsonStr), now, sourceType, req.ipAddr, &req);
newEvents.emplace_back(std::move(packedStr), std::move(jsonStr), &req);
}
}
try {
auto txn = env.txn_rw();
writeEvents(txn, newEvents);
writeEvents(txn, neFilterCache, newEvents);
txn.commit();
} catch (std::exception &e) {
LE << "Error writing " << newEvents.size() << " events: " << e.what();
@ -61,8 +61,8 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
for (auto &newEvent : newEvents) {
auto *flat = flatbuffers::GetRoot<NostrIndex::Event>(newEvent.flatStr.data());
auto eventIdHex = to_hex(sv(flat->id()));
PackedEventView packed(newEvent.packedStr);
auto eventIdHex = to_hex(packed.id());
tao::json::value output = tao::json::empty_object;
std::string message;
@ -71,7 +71,7 @@ void WebServer::runWriter(ThreadPool<MsgWebWriter>::Thread &thr) {
LI << "Inserted event. id=" << eventIdHex << " levId=" << newEvent.levId;
output["message"] = message = "ok";
output["written"] = true;
output["event"] = encodeBech32Simple("note", sv(flat->id()));
output["event"] = encodeBech32Simple("note", packed.id());
} else if (newEvent.status == EventWriteStatus::Duplicate) {
output["message"] = message = "duplicate: have this event";
output["written"] = true;