From ee612416e08b20048a159932f98180ea1bb906d6 Mon Sep 17 00:00:00 2001
From: Doug Hoyte <doug@hcsw.org>
Date: Sun, 29 Jan 2023 04:22:30 -0500
Subject: [PATCH] index-only scans for pubkey+kind

---
 TODO          |  1 -
 src/DBScan.h  | 39 +++++++++++++++++++++++++++------------
 src/filters.h |  3 +--
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/TODO b/TODO
index 5a58bbf..e062d76 100644
--- a/TODO
+++ b/TODO
@@ -15,7 +15,6 @@ features
   less verbose default logging
   nice new config "units" feature, ie 1d instead of 86400
   make it easier for a thread to setup a quadrable env
-  opt: PubkeyKind scans could be done index-only
 
 rate limits
   slow-reader detection and back-pressure
diff --git a/src/DBScan.h b/src/DBScan.h
index a6ebd86..dddc178 100644
--- a/src/DBScan.h
+++ b/src/DBScan.h
@@ -49,10 +49,16 @@ struct DBScan {
     std::string resumeKey;
     uint64_t resumeVal;
 
+    enum class KeyMatchResult {
+        Yes,
+        No,
+        NoButContinue,
+    };
+
     std::function<bool()> isComplete;
     std::function<void()> nextFilterItem;
     std::function<void()> resetResume;
-    std::function<bool(std::string_view, bool&)> keyMatch;
+    std::function<KeyMatchResult(std::string_view, bool&)> keyMatch;
 
     DBScan(const NostrFilter &f_) : f(f_) {
         remainingLimit = f.limit;
@@ -74,7 +80,7 @@ struct DBScan {
                 resumeVal = MAX_U64;
             };
             keyMatch = [&, state](std::string_view k, bool&){
-                return k.starts_with(state->prefix);
+                return k.starts_with(state->prefix) ? KeyMatchResult::Yes : KeyMatchResult::No;
             };
         } else if (f.authors && f.kinds) {
             scanState = PubkeyKindScan{};
@@ -98,16 +104,22 @@ struct DBScan {
                 resumeVal = MAX_U64;
             };
             keyMatch = [&, state](std::string_view k, bool &skipBack){
-                if (!k.starts_with(state->prefix)) return false;
-                if (state->prefix.size() == 32 + 8) return true;
+                if (!k.starts_with(state->prefix)) return KeyMatchResult::No;
+                if (state->prefix.size() == 32 + 8) return KeyMatchResult::Yes;
 
                 ParsedKey_StringUint64Uint64 parsedKey(k);
-                if (parsedKey.n1 <= f.kinds->at(state->indexKind)) return true;
+                if (parsedKey.n1 == f.kinds->at(state->indexKind)) {
+                    return KeyMatchResult::Yes;
+                } else if (parsedKey.n1 < f.kinds->at(state->indexKind)) {
+                    // With a prefix pubkey, continue scanning (pubkey,kind) backwards because with this index
+                    // we don't know the next pubkey to jump back to
+                    return KeyMatchResult::NoButContinue;
+                }
 
                 resumeKey = makeKey_StringUint64Uint64(parsedKey.s, f.kinds->at(state->indexKind), MAX_U64);
                 resumeVal = MAX_U64;
                 skipBack = true;
-                return false;
+                return KeyMatchResult::No;
             };
         } else if (f.authors) {
             scanState = PubkeyScan{};
@@ -126,7 +138,7 @@ struct DBScan {
                 resumeVal = MAX_U64;
             };
             keyMatch = [&, state](std::string_view k, bool&){
-                return k.starts_with(state->prefix);
+                return k.starts_with(state->prefix) ? KeyMatchResult::Yes : KeyMatchResult::No;
             };
         } else if (f.tags.size()) {
             scanState = TagScan{f.tags.begin()};
@@ -150,7 +162,7 @@ struct DBScan {
                 resumeVal = MAX_U64;
             };
             keyMatch = [&, state](std::string_view k, bool&){
-                return k.substr(0, state->search.size()) == state->search;
+                return k.substr(0, state->search.size()) == state->search ? KeyMatchResult::Yes : KeyMatchResult::No;
             };
         } else if (f.kinds) {
             scanState = KindScan{};
@@ -170,7 +182,7 @@ struct DBScan {
             };
             keyMatch = [&, state](std::string_view k, bool&){
                 ParsedKey_Uint64Uint64 parsedKey(k);
-                return parsedKey.n1 == state->kind;
+                return parsedKey.n1 == state->kind ? KeyMatchResult::Yes : KeyMatchResult::No;
             };
         } else {
             scanState = CreatedAtScan{};
@@ -188,7 +200,7 @@ struct DBScan {
                 resumeVal = MAX_U64;
             };
             keyMatch = [&, state](std::string_view k, bool&){
-                return true;
+                return KeyMatchResult::Yes;
             };
         }
     }
@@ -208,7 +220,8 @@ struct DBScan {
                     return false;
                 }
 
-                if (!keyMatch(k, skipBack)) return false;
+                auto matched = keyMatch(k, skipBack);
+                if (matched == KeyMatchResult::No) return false;
 
                 uint64_t created;
 
@@ -234,7 +247,9 @@ struct DBScan {
                 bool sent = false;
                 uint64_t levId = lmdb::from_sv<uint64_t>(v);
 
-                if (f.indexOnlyScans) {
+                if (matched == KeyMatchResult::NoButContinue) {
+                    // Don't attempt to match filter
+                } else if (f.indexOnlyScans) {
                     if (f.doesMatchTimes(created)) {
                         handleEvent(levId);
                         sent = true;
diff --git a/src/filters.h b/src/filters.h
index ab1d054..cdabe44 100644
--- a/src/filters.h
+++ b/src/filters.h
@@ -168,8 +168,7 @@ struct NostrFilter {
 
         if (limit > maxFilterLimit) limit = maxFilterLimit;
 
-        indexOnlyScans = numMajorFields <= 1;
-        // FIXME: pubkeyKind scan could be serviced index-only too
+        indexOnlyScans = (numMajorFields <= 1) || (numMajorFields == 2 && authors && kinds);
     }
 
     bool doesMatchTimes(uint64_t created) const {