nostrdb/search: allow searching from newest-to-oldest and oldest-to-newest

Signed-off-by: William Casarin <jb55@jb55.com>

nostrdb/nostrdb.c

@@ -177,7 +177,7 @@ static int ndb_make_text_search_key(unsigned char *buf, int bufsize,
 	// TODO: need update this to uint64_t
 	// we push this first because our query function can pull this off
 	// quicky to check matches
-	if (!push_varint(&cur, (int)note_id))
+	if (!push_varint(&cur, (int32_t)note_id))
 		return 0;
 
 	// string length
@@ -238,12 +238,14 @@ static int ndb_make_text_search_key_high(unsigned char *buf, int bufsize,
 					 int *keysize)
 {
 	uint64_t timestamp, note_id;
-	timestamp = UINT64_MAX;
+	timestamp = INT32_MAX;
-	note_id = UINT64_MAX;
+	note_id = INT32_MAX;
 	return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word,
 					timestamp, note_id, keysize);
 }
 
+typedef int (*ndb_text_search_key_order_fn)(unsigned char *buf, int bufsize, int wordlen, const char *word, int *keysize);
+
 /** From LMDB: Compare two items lexically */
 static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) {
 	int diff;
@@ -2469,7 +2471,8 @@ static int prefix_count(const char *str1, int len1, const char *str2, int len2)
 static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
 	MDB_val *k, struct ndb_word *search_word,
 	struct ndb_text_search_result *last_result,
-	struct ndb_text_search_result *result)
+	struct ndb_text_search_result *result,
+	MDB_cursor_op order_op)
 {
 	struct cursor key_cursor;
 	MDB_val v;
@@ -2481,8 +2484,18 @@ static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
 	// key.
 	//
 	// Subsequent searches should use MDB_NEXT
-	if (mdb_cursor_get(cursor, k, &v, op))
+	if (mdb_cursor_get(cursor, k, &v, op)) {
-		return 0;
+		// we should only do this if we're going in reverse
+		if (op == MDB_SET_RANGE && order_op == MDB_PREV) {
+			// if set range worked and our key exists, it should be
+			// the one right before this one
+			if (mdb_cursor_get(cursor, k, &v, MDB_PREV))
+				return 0;
+		} else {
+			return 0;
+		}
+	}
+
 
 	make_cursor(k->mv_data, k->mv_data + k->mv_size, &key_cursor);
 
@@ -2566,28 +2579,68 @@ static void ndb_text_search_results_init(
 	results->num_results = 0;
 }
 
+static void ndb_print_text_search_key(struct ndb_text_search_key *key)
+{
+	printf("K<'%.*s' %d %" PRIu64 " note_id:%" PRIu64 ">", key->str_len, key->str,
+						    key->word_index,
+						    key->timestamp,
+						    key->note_id);
+}
+
+void ndb_default_text_search_config(struct ndb_text_search_config *cfg)
+{
+	cfg->order = NDB_ORDER_DESCENDING;
+	cfg->limit = MAX_TEXT_SEARCH_RESULTS;
+}
+
+void ndb_text_search_config_set_order(struct ndb_text_search_config *cfg,
+				     enum ndb_search_order order)
+{
+	cfg->order = order;
+}
+
+void ndb_text_search_config_set_limit(struct ndb_text_search_config *cfg, int limit)
+{
+	cfg->limit = limit;
+}
+
 int ndb_text_search(struct ndb_txn *txn, const char *query,
-		    struct ndb_text_search_results *results, int limit)
+		    struct ndb_text_search_results *results,
+		    struct ndb_text_search_config *config)
 {
 	unsigned char buffer[1024], *buf;
 	unsigned char saved_buf[1024], *saved;
 	struct ndb_text_search_result *result, *last_result;
 	struct ndb_text_search_result candidate, last_candidate;
 	struct ndb_search_words search_words;
+	//struct ndb_text_search_key search_key;
 	struct ndb_word *search_word;
 	struct cursor cur;
+	ndb_text_search_key_order_fn key_order_fn;
 	MDB_dbi text_db;
 	MDB_cursor *cursor;
 	MDB_val k, v;
-	int i, j, keysize, saved_size;
+	int i, j, keysize, saved_size, limit;
-	MDB_cursor_op op;
+	MDB_cursor_op op, order_op;
 	//int num_note_ids;
 
 	saved = NULL;
 	ndb_text_search_results_init(results);
 	ndb_search_words_init(&search_words);
+
+	// search config
+	limit = MAX_TEXT_SEARCH_RESULTS;
+	order_op = MDB_PREV;
+	key_order_fn = ndb_make_text_search_key_high;
+	if (config) {
+		if (config->order == NDB_ORDER_ASCENDING) {
+			order_op = MDB_NEXT;
+			key_order_fn = ndb_make_text_search_key_low;
+		}
+		limit = min(limit, config->limit);
+	}
+	// end search config
 	
-	//num_note_ids = 0;
 	text_db = txn->lmdb->dbs[NDB_DB_NOTE_TEXT];
 	make_cursor((unsigned char *)query, (unsigned char *)query + strlen(query), &cur);
 
@@ -2600,8 +2653,6 @@ int ndb_text_search(struct ndb_txn *txn, const char *query,
 		return 0;
 	}
 
-	limit = min(MAX_TEXT_SEARCH_RESULTS, limit);
-
 	// for each word, we recursively find all of the submatches
 	while (results->num_results < limit) {
 		last_result = NULL;
@@ -2619,18 +2670,17 @@ int ndb_text_search(struct ndb_txn *txn, const char *query,
 
 			// reposition the cursor so we can continue
 			if (mdb_cursor_get(cursor, &k, &v, MDB_SET_RANGE))
-				break;
+				return 0;
 
-			op = MDB_NEXT;
+			op = order_op;
 		} else {
 			// construct a packed fulltext search key using this
 			// word this key doesn't contain any timestamp or index
 			// info, so it should range match instead of exact
 			// match
-			if (!ndb_make_text_search_key_low(
+			if (!key_order_fn(buffer, sizeof(buffer),
-				buffer, sizeof(buffer),
+					  search_words.words[0].word_len,
-				search_words.words[0].word_len,
+					  search_words.words[0].word, &keysize))
-				search_words.words[0].word, &keysize))
 			{
 				// word is too big to fit in 1024-sized key
 				continue;
@@ -2677,7 +2727,8 @@ int ndb_text_search(struct ndb_txn *txn, const char *query,
 			if (!ndb_text_search_next_word(cursor, op, &k,
 						       search_word,
 						       last_result,
-						       &candidate)) {
+						       &candidate,
+						       order_op)) {
 				break;
 			}
 

nostrdb/nostrdb.h

@@ -277,6 +277,11 @@ enum ndb_generic_element_type {
 	NDB_ELEMENT_ID      = 2,
 };
 
+enum ndb_search_order {
+	NDB_ORDER_DESCENDING,
+	NDB_ORDER_ASCENDING,
+};
+
 union ndb_filter_element {
 	const char *string;
 	const unsigned char *id;
@@ -311,6 +316,11 @@ struct ndb_config {
 	ndb_ingest_filter_fn ingest_filter;
 };
 
+struct ndb_text_search_config {
+	enum ndb_search_order order;
+	int limit;
+};
+
 // CONFIG
 void ndb_default_config(struct ndb_config *);
 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads);
@@ -377,7 +387,10 @@ void ndb_filter_end_field(struct ndb_filter *);
 void ndb_filter_free(struct ndb_filter *filter);
 
 // FULLTEXT SEARCH
-int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, int limit);
+int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, struct ndb_text_search_config *);
+void ndb_default_text_search_config(struct ndb_text_search_config *);
+void ndb_text_search_config_set_order(struct ndb_text_search_config *, enum ndb_search_order);
+void ndb_text_search_config_set_limit(struct ndb_text_search_config *, int limit);
 
 // stats
 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat);