nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-16 16:28:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

support empty arrays in REQ filters

Browse Source 
- These never match, so clients shouldn't really ever send them, but there is no reason to fail if they do
This commit is contained in:
Doug Hoyte
2023-01-16 14:04:44 -05:00
parent a9c69bd7c1
commit 18a7cc556a
5 changed files with 70 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
golpe.yaml
View File

@ -131,4 +131,4 @@ config:
default: 250 default: 250
- name: events__maxTagValSize - name: events__maxTagValSize
desc: "Maximum size for tag values, in bytes" desc: "Maximum size for tag values, in bytes"
default: 128 default: 255

42
src/ActiveMonitors.h
View File

@ -161,14 +161,14 @@ struct ActiveMonitors : NonCopyable {
void installLookups(Monitor *m, uint64_t currEventId) { void installLookups(Monitor *m, uint64_t currEventId) {
for (auto &f : m->sub.filterGroup.filters) { for (auto &f : m->sub.filterGroup.filters) {
if (f.ids.size()) { if (f.ids) {
for (size_t i = 0; i < f.ids.size(); i++) { for (size_t i = 0; i < f.ids->size(); i++) {
auto res = allIds.try_emplace(f.ids.at(i)); auto res = allIds.try_emplace(f.ids->at(i));
res.first->second.try_emplace(&f, MonitorItem{m, currEventId}); res.first->second.try_emplace(&f, MonitorItem{m, currEventId});
} }
} else if (f.authors.size()) { } else if (f.authors) {
for (size_t i = 0; i < f.authors.size(); i++) { for (size_t i = 0; i < f.authors->size(); i++) {
auto res = allAuthors.try_emplace(f.authors.at(i)); auto res = allAuthors.try_emplace(f.authors->at(i));
res.first->second.try_emplace(&f, MonitorItem{m, currEventId}); res.first->second.try_emplace(&f, MonitorItem{m, currEventId});
} }
} else if (f.tags.size()) { } else if (f.tags.size()) {
@ -179,9 +179,9 @@ struct ActiveMonitors : NonCopyable {
res.first->second.try_emplace(&f, MonitorItem{m, currEventId}); res.first->second.try_emplace(&f, MonitorItem{m, currEventId});
} }
} }
} else if (f.kinds.size()) { } else if (f.kinds) {
for (size_t i = 0; i < f.kinds.size(); i++) { for (size_t i = 0; i < f.kinds->size(); i++) {
auto res = allKinds.try_emplace(f.kinds.at(i)); auto res = allKinds.try_emplace(f.kinds->at(i));
res.first->second.try_emplace(&f, MonitorItem{m, currEventId}); res.first->second.try_emplace(&f, MonitorItem{m, currEventId});
} }
} else { } else {
@ -192,17 +192,17 @@ struct ActiveMonitors : NonCopyable {
void uninstallLookups(Monitor *m) { void uninstallLookups(Monitor *m) {
for (auto &f : m->sub.filterGroup.filters) { for (auto &f : m->sub.filterGroup.filters) {
if (f.ids.size()) { if (f.ids) {
for (size_t i = 0; i < f.ids.size(); i++) { for (size_t i = 0; i < f.ids->size(); i++) {
auto &monSet = allIds.at(f.ids.at(i)); auto &monSet = allIds.at(f.ids->at(i));
monSet.erase(&f); monSet.erase(&f);
if (monSet.empty()) allIds.erase(f.ids.at(i)); if (monSet.empty()) allIds.erase(f.ids->at(i));
} }
} else if (f.authors.size()) { } else if (f.authors) {
for (size_t i = 0; i < f.authors.size(); i++) { for (size_t i = 0; i < f.authors->size(); i++) {
auto &monSet = allAuthors.at(f.authors.at(i)); auto &monSet = allAuthors.at(f.authors->at(i));
monSet.erase(&f); monSet.erase(&f);
if (monSet.empty()) allAuthors.erase(f.authors.at(i)); if (monSet.empty()) allAuthors.erase(f.authors->at(i));
} }
} else if (f.tags.size()) { } else if (f.tags.size()) {
for (const auto &[tagName, filterSet] : f.tags) { for (const auto &[tagName, filterSet] : f.tags) {
@ -213,11 +213,11 @@ struct ActiveMonitors : NonCopyable {
if (monSet.empty()) allTags.erase(tagSpec); if (monSet.empty()) allTags.erase(tagSpec);
} }
} }
} else if (f.kinds.size()) { } else if (f.kinds) {
for (size_t i = 0; i < f.kinds.size(); i++) { for (size_t i = 0; i < f.kinds->size(); i++) {
auto &monSet = allKinds.at(f.kinds.at(i)); auto &monSet = allKinds.at(f.kinds->at(i));
monSet.erase(&f); monSet.erase(&f);
if (monSet.empty()) allKinds.erase(f.kinds.at(i)); if (monSet.empty()) allKinds.erase(f.kinds->at(i));
} }
} else { } else {
allOthers.erase(&f); allOthers.erase(&f);

32
src/DBScan.h
View File

@ -57,7 +57,7 @@ struct DBScan {
DBScan(const NostrFilter &f_) : f(f_) { DBScan(const NostrFilter &f_) : f(f_) {
remainingLimit = f.limit; remainingLimit = f.limit;
if (f.ids.size()) { if (f.ids) {
LI << "ID Scan"; LI << "ID Scan";
scanState = IdScan{}; scanState = IdScan{};
@ -65,20 +65,20 @@ struct DBScan {
indexDbi = env.dbi_Event__id; indexDbi = env.dbi_Event__id;
isComplete = [&, state]{ isComplete = [&, state]{
return state->index >= f.ids.size(); return state->index >= f.ids->size();
}; };
nextFilterItem = [&, state]{ nextFilterItem = [&, state]{
state->index++; state->index++;
}; };
resetResume = [&, state]{ resetResume = [&, state]{
state->prefix = f.ids.at(state->index); state->prefix = f.ids->at(state->index);
resumeKey = padBytes(state->prefix, 32 + 8, '\xFF'); resumeKey = padBytes(state->prefix, 32 + 8, '\xFF');
resumeVal = MAX_U64; resumeVal = MAX_U64;
}; };
keyMatch = [&, state](std::string_view k, bool&){ keyMatch = [&, state](std::string_view k, bool&){
return k.starts_with(state->prefix); return k.starts_with(state->prefix);
}; };
} else if (f.authors.size() && f.kinds.size()) { } else if (f.authors && f.kinds) {
LI << "PubkeyKind Scan"; LI << "PubkeyKind Scan";
scanState = PubkeyKindScan{}; scanState = PubkeyKindScan{};
@ -86,18 +86,18 @@ struct DBScan {
indexDbi = env.dbi_Event__pubkeyKind; indexDbi = env.dbi_Event__pubkeyKind;
isComplete = [&, state]{ isComplete = [&, state]{
return state->indexAuthor >= f.authors.size(); return state->indexAuthor >= f.authors->size();
}; };
nextFilterItem = [&, state]{ nextFilterItem = [&, state]{
state->indexKind++; state->indexKind++;
if (state->indexKind >= f.kinds.size()) { if (state->indexKind >= f.kinds->size()) {
state->indexAuthor++; state->indexAuthor++;
state->indexKind = 0; state->indexKind = 0;
} }
}; };
resetResume = [&, state]{ resetResume = [&, state]{
state->prefix = f.authors.at(state->indexAuthor); state->prefix = f.authors->at(state->indexAuthor);
if (state->prefix.size() == 32) state->prefix += lmdb::to_sv<uint64_t>(f.kinds.at(state->indexKind)); if (state->prefix.size() == 32) state->prefix += lmdb::to_sv<uint64_t>(f.kinds->at(state->indexKind));
resumeKey = padBytes(state->prefix, 32 + 8 + 8, '\xFF'); resumeKey = padBytes(state->prefix, 32 + 8 + 8, '\xFF');
resumeVal = MAX_U64; resumeVal = MAX_U64;
}; };
@ -106,14 +106,14 @@ struct DBScan {
if (state->prefix.size() == 32 + 8) return true; if (state->prefix.size() == 32 + 8) return true;
ParsedKey_StringUint64Uint64 parsedKey(k); ParsedKey_StringUint64Uint64 parsedKey(k);
if (parsedKey.n1 <= f.kinds.at(state->indexKind)) return true; if (parsedKey.n1 <= f.kinds->at(state->indexKind)) return true;
resumeKey = makeKey_StringUint64Uint64(parsedKey.s, f.kinds.at(state->indexKind), MAX_U64); resumeKey = makeKey_StringUint64Uint64(parsedKey.s, f.kinds->at(state->indexKind), MAX_U64);
resumeVal = MAX_U64; resumeVal = MAX_U64;
skipBack = true; skipBack = true;
return false; return false;
}; };
} else if (f.authors.size()) { } else if (f.authors) {
LI << "Pubkey Scan"; LI << "Pubkey Scan";
scanState = PubkeyScan{}; scanState = PubkeyScan{};
@ -121,13 +121,13 @@ struct DBScan {
indexDbi = env.dbi_Event__pubkey; indexDbi = env.dbi_Event__pubkey;
isComplete = [&, state]{ isComplete = [&, state]{
return state->index >= f.authors.size(); return state->index >= f.authors->size();
}; };
nextFilterItem = [&, state]{ nextFilterItem = [&, state]{
state->index++; state->index++;
}; };
resetResume = [&, state]{ resetResume = [&, state]{
state->prefix = f.authors.at(state->index); state->prefix = f.authors->at(state->index);
resumeKey = padBytes(state->prefix, 32 + 8, '\xFF'); resumeKey = padBytes(state->prefix, 32 + 8, '\xFF');
resumeVal = MAX_U64; resumeVal = MAX_U64;
}; };
@ -160,7 +160,7 @@ struct DBScan {
keyMatch = [&, state](std::string_view k, bool&){ keyMatch = [&, state](std::string_view k, bool&){
return k.substr(0, state->search.size()) == state->search; return k.substr(0, state->search.size()) == state->search;
}; };
} else if (f.kinds.size()) { } else if (f.kinds) {
LI << "Kind Scan"; LI << "Kind Scan";
scanState = KindScan{}; scanState = KindScan{};
@ -168,13 +168,13 @@ struct DBScan {
indexDbi = env.dbi_Event__kind; indexDbi = env.dbi_Event__kind;
isComplete = [&, state]{ isComplete = [&, state]{
return state->index >= f.kinds.size(); return state->index >= f.kinds->size();
}; };
nextFilterItem = [&, state]{ nextFilterItem = [&, state]{
state->index++; state->index++;
}; };
resetResume = [&, state]{ resetResume = [&, state]{
state->kind = f.kinds.at(state->index); state->kind = f.kinds->at(state->index);
resumeKey = std::string(lmdb::to_sv<uint64_t>(state->kind)) + std::string(8, '\xFF'); resumeKey = std::string(lmdb::to_sv<uint64_t>(state->kind)) + std::string(8, '\xFF');
resumeVal = MAX_U64; resumeVal = MAX_U64;
}; };

44
src/filters.h
View File

@ -15,11 +15,9 @@ struct FilterSetBytes {
std::vector<Item> items; std::vector<Item> items;
std::string buf; std::string buf;
FilterSetBytes() {}
// Sizes are post-hex decode // Sizes are post-hex decode
void init(const tao::json::value &arrHex, bool hexDecode, size_t minSize, size_t maxSize) { FilterSetBytes(const tao::json::value &arrHex, bool hexDecode, size_t minSize, size_t maxSize) {
std::vector<std::string> arr; std::vector<std::string> arr;
uint64_t totalSize = 0; uint64_t totalSize = 0;
@ -32,8 +30,6 @@ struct FilterSetBytes {
totalSize += itemSize; totalSize += itemSize;
} }
if (arr.size() == 0) throw herr("empty filter item");
std::sort(arr.begin(), arr.end()); std::sort(arr.begin(), arr.end());
for (const auto &item : arr) { for (const auto &item : arr) {
@ -90,15 +86,11 @@ struct FilterSetBytes {
struct FilterSetUint { struct FilterSetUint {
std::vector<uint64_t> items; std::vector<uint64_t> items;
FilterSetUint() {} FilterSetUint(const tao::json::value &arr) {
void init(const tao::json::value &arr) {
for (const auto &i : arr.get_array()) { for (const auto &i : arr.get_array()) {
items.push_back(i.get_unsigned()); items.push_back(i.get_unsigned());
} }
if (items.size() == 0) throw herr("empty filter item");
std::sort(items.begin(), items.end()); std::sort(items.begin(), items.end());
items.erase(std::unique(items.begin(), items.end()), items.end()); // remove duplicates items.erase(std::unique(items.begin(), items.end()), items.end()); // remove duplicates
@ -119,39 +111,44 @@ struct FilterSetUint {
}; };
struct NostrFilter { struct NostrFilter {
FilterSetBytes ids; std::optional<FilterSetBytes> ids;
FilterSetBytes authors; std::optional<FilterSetBytes> authors;
FilterSetUint kinds; std::optional<FilterSetUint> kinds;
std::map<char, FilterSetBytes> tags; std::map<char, FilterSetBytes> tags;
uint64_t since = 0; uint64_t since = 0;
uint64_t until = MAX_U64; uint64_t until = MAX_U64;
uint64_t limit = MAX_U64; uint64_t limit = MAX_U64;
bool neverMatch = false;
bool indexOnlyScans = false; bool indexOnlyScans = false;
explicit NostrFilter(const tao::json::value &filterObj) { explicit NostrFilter(const tao::json::value &filterObj) {
uint64_t numMajorFields = 0; uint64_t numMajorFields = 0;
for (const auto &[k, v] : filterObj.get_object()) { for (const auto &[k, v] : filterObj.get_object()) {
if (v.is_array() && v.get_array().size() == 0) {
neverMatch = true;
break;
}
if (k == "ids") { if (k == "ids") {
ids.init(v, true, 1, 32); ids.emplace(v, true, 1, 32);
numMajorFields++; numMajorFields++;
} else if (k == "authors") { } else if (k == "authors") {
authors.init(v, true, 1, 32); authors.emplace(v, true, 1, 32);
numMajorFields++; numMajorFields++;
} else if (k == "kinds") { } else if (k == "kinds") {
kinds.init(v); kinds.emplace(v);
numMajorFields++; numMajorFields++;
} else if (k.starts_with('#')) { } else if (k.starts_with('#')) {
numMajorFields++; numMajorFields++;
if (k.size() == 2) { if (k.size() == 2) {
char tag = k[1]; char tag = k[1];
auto [it, _] = tags.emplace(tag, FilterSetBytes{});
if (tag == 'p' || tag == 'e') { if (tag == 'p' || tag == 'e') {
it->second.init(v, true, 32, 32); tags.emplace(tag, FilterSetBytes(v, true, 32, 32));
} else { } else {
it->second.init(v, false, 1, cfg().events__maxTagValSize); tags.emplace(tag, FilterSetBytes(v, false, 1, cfg().events__maxTagValSize));
} }
} else { } else {
throw herr("unindexed tag filter"); throw herr("unindexed tag filter");
@ -182,11 +179,13 @@ struct NostrFilter {
} }
bool doesMatch(const NostrIndex::Event *ev) const { bool doesMatch(const NostrIndex::Event *ev) const {
if (neverMatch) return false;
if (!doesMatchTimes(ev->created_at())) return false; if (!doesMatchTimes(ev->created_at())) return false;
if (ids.size() && !ids.doesMatch(sv(ev->id()))) return false; if (ids && !ids->doesMatch(sv(ev->id()))) return false;
if (authors.size() && !authors.doesMatch(sv(ev->pubkey()))) return false; if (authors && !authors->doesMatch(sv(ev->pubkey()))) return false;
if (kinds.size() && !kinds.doesMatch(ev->kind())) return false; if (kinds && !kinds->doesMatch(ev->kind())) return false;
for (const auto &[tag, filt] : tags) { for (const auto &[tag, filt] : tags) {
bool foundMatch = false; bool foundMatch = false;
@ -216,6 +215,7 @@ struct NostrFilterGroup {
for (size_t i = 2; i < arr.size(); i++) { for (size_t i = 2; i < arr.size(); i++) {
filters.emplace_back(arr[i]); filters.emplace_back(arr[i]);
if (filters.back().neverMatch) filters.pop_back();
} }
} }

15
test/filterFuzzTest.pl
View File

@ -96,31 +96,36 @@ sub genRandomFilterGroup {
while (!keys %$f) { while (!keys %$f) {
if (rand() < .15) { if (rand() < .15) {
for (1..(rand()*10) + 1) { $f->{ids} = [];
for (1..(rand()*10)) {
push @{$f->{ids}}, randPrefix($ids->[int(rand() * @$ids)]); push @{$f->{ids}}, randPrefix($ids->[int(rand() * @$ids)]);
} }
} }
if (rand() < .3) { if (rand() < .3) {
for (1..(rand()*5) + 1) { $f->{authors} = [];
for (1..(rand()*5)) {
push @{$f->{authors}}, randPrefix($pubkeys->[int(rand() * @$pubkeys)]); push @{$f->{authors}}, randPrefix($pubkeys->[int(rand() * @$pubkeys)]);
} }
} }
if (rand() < .2) { if (rand() < .2) {
for (1..(rand()*5) + 1) { $f->{kinds} = [];
for (1..(rand()*5)) {
push @{$f->{kinds}}, 0+$kinds->[int(rand() * @$kinds)]; push @{$f->{kinds}}, 0+$kinds->[int(rand() * @$kinds)];
} }
} }
if (rand() < .2) { if (rand() < .2) {
for (1..(rand()*10) + 1) { $f->{'#e'} = [];
for (1..(rand()*10)) {
push @{$f->{'#e'}}, $ids->[int(rand() * @$ids)]; push @{$f->{'#e'}}, $ids->[int(rand() * @$ids)];
} }
} }
if (rand() < .2) { if (rand() < .2) {
for (1..(rand()*5) + 1) { $f->{'#p'} = [];
for (1..(rand()*5)) {
push @{$f->{'#p'}}, $pubkeys->[int(rand() * @$pubkeys)]; push @{$f->{'#p'}}, $pubkeys->[int(rand() * @$pubkeys)];
} }
} }