nostr/damus
1
0
Fork 0
mirror of git://jb55.com/damus synced 2024-10-04 19:00:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

nostrdb/search: fix subtle bug with some newest-first text search

Browse Source 
Due to the way the range queries work for newest-first searches, we can
have a situation where the MDB_SET_RANGE gets placed on either the
correct place or just after the correct place. To position the cursor
correctly, we jump back one if the search result prefix doesn't match.

Signed-off-by: William Casarin <jb55@jb55.com>
This commit is contained in:
William Casarin 2023-12-02 13:09:30 -08:00
parent 9bbeffe320
commit 8e361a9586
1 changed files with 82 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
nostrdb/nostrdb.c
View File

@ -2473,6 +2473,38 @@ static void ndb_print_text_search_key(struct ndb_text_search_key *key)
key->note_id); key->note_id);
} }
static int ndb_prefix_matches(struct ndb_text_search_result *result,
struct ndb_word *search_word)
{
// Empty strings shouldn't happen but let's
if (result->key.str_len < 2 || search_word->word_len < 2)
return 0;
// make sure we at least have two matching prefix characters. exact
// matches are nice but range searches allow us to match prefixes as
// well. A double-char prefix is suffient, but maybe we could up this
// in the future.
//
// TODO: How are we handling utf-8 prefix matches like
// japanese?
//
if ( result->key.str[0] != tolower(search_word->word[0])
&& result->key.str[1] != tolower(search_word->word[1])
)
return 0;
// count the number of prefix-matched characters. This will be used
// for ranking search results
result->prefix_chars = prefix_count(result->key.str,
result->key.str_len,
search_word->word,
search_word->word_len);
if (result->prefix_chars <= (int)((double)search_word->word_len / 1.5))
return 0;
return 1;
}
// This is called when scanning the full text search index. Scanning stops // This is called when scanning the full text search index. Scanning stops
// when we no longer have a prefix match for the word // when we no longer have a prefix match for the word
@ -2483,7 +2515,10 @@ static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
MDB_cursor_op order_op) MDB_cursor_op order_op)
{ {
struct cursor key_cursor; struct cursor key_cursor;
//struct ndb_text_search_key search_key;
MDB_val v; MDB_val v;
int retries;
retries = -1;
make_cursor(k->mv_data, k->mv_data + k->mv_size, &key_cursor); make_cursor(k->mv_data, k->mv_data + k->mv_size, &key_cursor);
@ -2504,6 +2539,15 @@ static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
} }
} }
retry:
retries++;
/*
printf("continuing from ");
if (ndb_unpack_text_search_key(k->mv_data, k->mv_size, &search_key)) {
ndb_print_text_search_key(&search_key);
} else { printf("??"); }
printf("\n");
*/
make_cursor(k->mv_data, k->mv_data + k->mv_size, &key_cursor); make_cursor(k->mv_data, k->mv_data + k->mv_size, &key_cursor);
@ -2533,32 +2577,17 @@ static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
return 0; return 0;
} }
// Empty strings shouldn't happen but let's if (!ndb_prefix_matches(result, search_word)) {
if (result->key.str_len < 2 || search_word->word_len < 2) // we should only do this if we're going in reverse
return 0; if (retries == 0 && op == MDB_SET_RANGE && order_op == MDB_PREV) {
// if set range worked and our key exists, it should be
// make sure we at least have two matching prefix characters. exact // the one right before this one
// matches are nice but range searches allow us to match prefixes as if (mdb_cursor_get(cursor, k, &v, MDB_PREV))
// well. A double-char prefix is suffient, but maybe we could up this goto retry;
// in the future. } else {
// return 0;
// TODO: How are we handling utf-8 prefix matches like }
// japanese? }
//
if ( result->key.str[0] != tolower(search_word->word[0])
&& result->key.str[1] != tolower(search_word->word[1])
)
return 0;
// count the number of prefix-matched characters. This will be used
// for ranking search results
result->prefix_chars = prefix_count(result->key.str,
result->key.str_len,
search_word->word,
search_word->word_len);
if (result->prefix_chars <= (int)((double)search_word->word_len / 1.5))
return 0;
// Unpack the remaining text search key, we will need this information // Unpack the remaining text search key, we will need this information
// when building up our search results. // when building up our search results.
@ -4439,3 +4468,30 @@ void ndb_config_set_ingest_filter(struct ndb_config *config,
config->ingest_filter = fn; config->ingest_filter = fn;
config->filter_context = filter_ctx; config->filter_context = filter_ctx;
} }
// used by ndb.c
int ndb_print_search_keys(struct ndb_txn *txn)
{
MDB_cursor *cur;
MDB_val k, v;
int i;
struct ndb_text_search_key search_key;
if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_TEXT], &cur))
return 0;
i = 1;
while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
if (!ndb_unpack_text_search_key(k.mv_data, k.mv_size, &search_key)) {
fprintf(stderr, "error decoding key %d\n", i);
continue;
}
ndb_print_text_search_key(&search_key);
printf("\n");
i++;
}
return 1;
}