nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-16 08:18:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

index-only scans for pubkey+kind

Browse Source
This commit is contained in:
Doug Hoyte
2023-01-29 04:22:30 -05:00
parent 9b83093be7
commit ee612416e0
3 changed files with 28 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
TODO
View File

@ -15,7 +15,6 @@ features
less verbose default logging
nice new config "units" feature, ie 1d instead of 86400
make it easier for a thread to setup a quadrable env
opt: PubkeyKind scans could be done index-only
rate limits
slow-reader detection and back-pressure

39
src/DBScan.h
View File

@ -49,10 +49,16 @@ struct DBScan {
std::string resumeKey;
uint64_t resumeVal;
enum class KeyMatchResult {
Yes,
No,
NoButContinue,
};
std::function<bool()> isComplete;
std::function<void()> nextFilterItem;
std::function<void()> resetResume;
std::function<bool(std::string_view, bool&)> keyMatch;
std::function<KeyMatchResult(std::string_view, bool&)> keyMatch;
DBScan(const NostrFilter &f_) : f(f_) {
remainingLimit = f.limit;
@ -74,7 +80,7 @@ struct DBScan {
resumeVal = MAX_U64;
};
keyMatch = [&, state](std::string_view k, bool&){
return k.starts_with(state->prefix);
return k.starts_with(state->prefix) ? KeyMatchResult::Yes : KeyMatchResult::No;
};
} else if (f.authors && f.kinds) {
scanState = PubkeyKindScan{};
@ -98,16 +104,22 @@ struct DBScan {
resumeVal = MAX_U64;
};
keyMatch = [&, state](std::string_view k, bool &skipBack){
if (!k.starts_with(state->prefix)) return false;
if (state->prefix.size() == 32 + 8) return true;
if (!k.starts_with(state->prefix)) return KeyMatchResult::No;
if (state->prefix.size() == 32 + 8) return KeyMatchResult::Yes;
ParsedKey_StringUint64Uint64 parsedKey(k);
if (parsedKey.n1 <= f.kinds->at(state->indexKind)) return true;
if (parsedKey.n1 == f.kinds->at(state->indexKind)) {
return KeyMatchResult::Yes;
} else if (parsedKey.n1 < f.kinds->at(state->indexKind)) {
// With a prefix pubkey, continue scanning (pubkey,kind) backwards because with this index
// we don't know the next pubkey to jump back to
return KeyMatchResult::NoButContinue;
}
resumeKey = makeKey_StringUint64Uint64(parsedKey.s, f.kinds->at(state->indexKind), MAX_U64);
resumeVal = MAX_U64;
skipBack = true;
return false;
return KeyMatchResult::No;
};
} else if (f.authors) {
scanState = PubkeyScan{};
@ -126,7 +138,7 @@ struct DBScan {
resumeVal = MAX_U64;
};
keyMatch = [&, state](std::string_view k, bool&){
return k.starts_with(state->prefix);
return k.starts_with(state->prefix) ? KeyMatchResult::Yes : KeyMatchResult::No;
};
} else if (f.tags.size()) {
scanState = TagScan{f.tags.begin()};
@ -150,7 +162,7 @@ struct DBScan {
resumeVal = MAX_U64;
};
keyMatch = [&, state](std::string_view k, bool&){
return k.substr(0, state->search.size()) == state->search;
return k.substr(0, state->search.size()) == state->search ? KeyMatchResult::Yes : KeyMatchResult::No;
};
} else if (f.kinds) {
scanState = KindScan{};
@ -170,7 +182,7 @@ struct DBScan {
};
keyMatch = [&, state](std::string_view k, bool&){
ParsedKey_Uint64Uint64 parsedKey(k);
return parsedKey.n1 == state->kind;
return parsedKey.n1 == state->kind ? KeyMatchResult::Yes : KeyMatchResult::No;
};
} else {
scanState = CreatedAtScan{};
@ -188,7 +200,7 @@ struct DBScan {
resumeVal = MAX_U64;
};
keyMatch = [&, state](std::string_view k, bool&){
return true;
return KeyMatchResult::Yes;
};
}
}
@ -208,7 +220,8 @@ struct DBScan {
return false;
}
if (!keyMatch(k, skipBack)) return false;
auto matched = keyMatch(k, skipBack);
if (matched == KeyMatchResult::No) return false;
uint64_t created;
@ -234,7 +247,9 @@ struct DBScan {
bool sent = false;
uint64_t levId = lmdb::from_sv<uint64_t>(v);
if (f.indexOnlyScans) {
if (matched == KeyMatchResult::NoButContinue) {
// Don't attempt to match filter
} else if (f.indexOnlyScans) {
if (f.doesMatchTimes(created)) {
handleEvent(levId);
sent = true;

3
src/filters.h
View File

@ -168,8 +168,7 @@ struct NostrFilter {
if (limit > maxFilterLimit) limit = maxFilterLimit;
indexOnlyScans = numMajorFields <= 1;
// FIXME: pubkeyKind scan could be serviced index-only too
indexOnlyScans = (numMajorFields <= 1) || (numMajorFields == 2 && authors && kinds);
}
bool doesMatchTimes(uint64_t created) const {