search: switch to nostrdb profile searching

Changelog-Changed: Switch to nostrdb for @'s and user search

damus/ContentView.swift

@@ -629,7 +629,7 @@ struct ContentView: View {
                                       likes: EventCounter(our_pubkey: pubkey),
                                       boosts: EventCounter(our_pubkey: pubkey),
                                       contacts: Contacts(our_pubkey: pubkey),
-                                      profiles: Profiles(user_search_cache: user_search_cache, ndb: ndb),
+                                      profiles: Profiles(ndb: ndb),
                                       dms: home.dms,
                                       previews: PreviewCache(),
                                       zaps: Zaps(our_pubkey: pubkey),
@@ -646,7 +646,6 @@ struct ContentView: View {
                                       muted_threads: MutedThreadsManager(keypair: keypair),
                                       wallet: WalletModel(settings: settings),
                                       nav: self.navigationCoordinator,
-                                      user_search_cache: user_search_cache,
                                       music: MusicController(onChange: music_changed),
                                       video: VideoController(),
                                       ndb: ndb
@@ -919,7 +918,6 @@ func handle_unfollow(state: DamusState, unfollow: FollowRef) -> Bool {
     switch unfollow {
     case .pubkey(let pk):
         state.contacts.remove_friend(pk)
-        state.user_search_cache.updateOwnContactsPetnames(id: state.pubkey, oldEvent: old_contacts, newEvent: ev)
     case .hashtag:
         // nothing to handle here really
         break

damus/Models/DamusState.swift

@@ -31,7 +31,6 @@ struct DamusState {
     let muted_threads: MutedThreadsManager
     let wallet: WalletModel
     let nav: NavigationCoordinator
-    let user_search_cache: UserSearchCache
     let music: MusicController?
     let video: VideoController
     let ndb: Ndb
@@ -62,7 +61,6 @@ struct DamusState {
     }
     
     static var empty: DamusState {
-        let user_search_cache = UserSearchCache()
         let empty_pub: Pubkey = .empty
         let empty_sec: Privkey = .empty
         let kp = Keypair(pubkey: empty_pub, privkey: nil)
@@ -73,7 +71,7 @@ struct DamusState {
             likes: EventCounter(our_pubkey: empty_pub),
             boosts: EventCounter(our_pubkey: empty_pub),
             contacts: Contacts(our_pubkey: empty_pub),
-            profiles: Profiles(user_search_cache: user_search_cache, ndb: .empty),
+            profiles: Profiles(ndb: .empty),
             dms: DirectMessagesModel(our_pubkey: empty_pub),
             previews: PreviewCache(),
             zaps: Zaps(our_pubkey: empty_pub),
@@ -90,7 +88,6 @@ struct DamusState {
             muted_threads: MutedThreadsManager(keypair: kp),
             wallet: WalletModel(settings: UserSettingsStore()),
             nav: NavigationCoordinator(),
-            user_search_cache: user_search_cache,
             music: nil,
             video: VideoController(),
             ndb: .empty

damus/Models/HomeModel.swift

@@ -734,8 +734,6 @@ func load_our_contacts(state: DamusState, m_old_ev: NostrEvent?, ev: NostrEvent)
             }
         }
     }
-
-    state.user_search_cache.updateOwnContactsPetnames(id: contacts.our_pubkey, oldEvent: m_old_ev, newEvent: ev)
 }
 
 

damus/Nostr/Profiles.swift

@@ -38,10 +38,7 @@ class Profiles {
     @MainActor
     var nip05_pubkey: [String: Pubkey] = [:]
 
-    let user_search_cache: UserSearchCache
+    init(ndb: Ndb) {
-
-    init(user_search_cache: UserSearchCache, ndb: Ndb) {
-        self.user_search_cache = user_search_cache
         self.ndb = ndb
     }
 
@@ -84,6 +81,10 @@ class Profiles {
         return ndb.lookup_profile_by_key(key: key)
     }
 
+    func search<Y>(_ query: String, limit: Int, txn: NdbTxn<Y>) -> [Pubkey] {
+        return ndb.search_profile(query, limit: limit, txn: txn)
+    }
+
     func lookup(id: Pubkey) -> NdbTxn<Profile?> {
         return ndb.lookup_profile(id).map({ pr in pr?.profile })
     }

damus/TestData.swift

@@ -66,7 +66,6 @@ var test_damus_state: DamusState = ({
     print("opening \(tempDir!)")
     let ndb = Ndb(path: tempDir)!
     let our_pubkey = test_pubkey
-    let user_search_cache = UserSearchCache()
     let pool = RelayPool(ndb: ndb)
     let settings = UserSettingsStore()
     let damus = DamusState(pool: pool,
@@ -74,7 +73,7 @@ var test_damus_state: DamusState = ({
                            likes: .init(our_pubkey: our_pubkey),
                            boosts: .init(our_pubkey: our_pubkey),
                            contacts: .init(our_pubkey: our_pubkey),
-                           profiles: .init(user_search_cache: user_search_cache, ndb: ndb),
+                           profiles: .init(ndb: ndb),
                            dms: .init(our_pubkey: our_pubkey),
                            previews: .init(),
                            zaps: .init(our_pubkey: our_pubkey),
@@ -91,7 +90,6 @@ var test_damus_state: DamusState = ({
                            muted_threads: .init(keypair: test_keypair),
                            wallet: .init(settings: settings),
                            nav: .init(),
-                           user_search_cache: user_search_cache,
                            music: .init(onChange: {_ in }),
                            video: .init(),
                            ndb: ndb)

damus/Views/Posting/UserSearch.swift

@@ -17,7 +17,8 @@ struct UserSearch: View {
     @EnvironmentObject var tagModel: TagModel
     
     var users: [Pubkey] {
-        return search_profiles(profiles: damus_state.profiles, search: search)
+        let txn = NdbTxn(ndb: damus_state.ndb)
+        return search_profiles(profiles: damus_state.profiles, search: search, txn: txn)
     }
     
     func on_user_tapped(pk: Pubkey) {

damus/Views/Profile/ProfilePicView.swift

@@ -113,8 +113,7 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR
 }
 
 func make_preview_profiles(_ pubkey: Pubkey) -> Profiles {
-    let user_search_cache = UserSearchCache()
+    let profiles = Profiles(ndb: test_damus_state.ndb)
-    let profiles = Profiles(user_search_cache: user_search_cache, ndb: test_damus_state.ndb)
     let picture = "http://cdn.jb55.com/img/red-me.jpg"
     let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil)
     //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note)

damus/Views/SearchResultsView.swift

@@ -108,10 +108,12 @@ struct SearchResultsView: View {
         }
         .frame(maxHeight: .infinity)
         .onAppear {
-            self.result = search_for_string(profiles: damus_state.profiles, search)
+            let txn = NdbTxn.init(ndb: damus_state.ndb)
+            self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn)
         }
         .onChange(of: search) { new in
-            self.result = search_for_string(profiles: damus_state.profiles, new)
+            let txn = NdbTxn.init(ndb: damus_state.ndb)
+            self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn)
         }
     }
 }
@@ -125,7 +127,7 @@ struct SearchResultsView_Previews: PreviewProvider {
  */
 
 
-func search_for_string(profiles: Profiles, _ new: String) -> Search? {
+func search_for_string<Y>(profiles: Profiles, search new: String, txn: NdbTxn<Y>) -> Search? {
     guard new.count != 0 else {
         return nil
     }
@@ -154,7 +156,7 @@ func search_for_string(profiles: Profiles, _ new: String) -> Search? {
         return .note(NoteId(decoded.data))
     }
     
-    let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new))
+    let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new, txn: txn))
     return .multi(multisearch)
 }
 
@@ -171,7 +173,7 @@ func make_hashtagable(_ str: String) -> String {
     return String(new.filter{$0 != " "})
 }
 
-func search_profiles(profiles: Profiles, search: String) -> [Pubkey] {
+func search_profiles<Y>(profiles: Profiles, search: String, txn: NdbTxn<Y>) -> [Pubkey] {
     // Search by hex pubkey.
     if let pubkey = hex_decode_pubkey(search),
        profiles.lookup_key_by_pubkey(pubkey) != nil
@@ -189,5 +191,7 @@ func search_profiles(profiles: Profiles, search: String) -> [Pubkey] {
     }
 
     let new = search.lowercased()
-    return profiles.user_search_cache.search(key: new)
+
+    return profiles.search(search, limit: 10, txn: txn)
 }
+

nostrdb/Ndb.swift

@@ -188,6 +188,32 @@ class Ndb {
         }
     }
 
+    func search_profile<Y>(_ search: String, limit: Int, txn: NdbTxn<Y>) -> [Pubkey] {
+        var pks = Array<Pubkey>()
+
+        return search.withCString { q in
+            var s = ndb_search()
+            guard ndb_search_profile(&txn.txn, &s, q) != 0 else {
+                return pks
+            }
+
+            defer { ndb_search_profile_end(&s) }
+            pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32)))
+
+            var n = limit
+            while n > 0 {
+                guard ndb_search_profile_next(&s) != 0 else {
+                    return pks
+                }
+                pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32)))
+
+                n -= 1
+            }
+
+            return pks
+        }
+    }
+
     deinit {
         ndb_destroy(ndb.ndb)
     }

nostrdb/nostrdb.c

@@ -69,6 +69,7 @@ enum ndb_dbs {
 	NDB_DB_NOTE_ID,
 	NDB_DB_PROFILE_PK,
 	NDB_DB_NDB_META,
+	NDB_DB_PROFILE_SEARCH,
 	NDB_DBS,
 };
 
@@ -123,16 +124,149 @@ struct ndb_tsid {
 	uint64_t timestamp;
 };
 
+static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
+			        uint64_t timestamp, const char *search)
+{
+	memcpy(key->id, id, 32);
+	key->timestamp = timestamp;
+	strncpy(key->search, search, sizeof(key->search) - 1);
+	key->search[sizeof(key->search) - 1] = '\0';
+}
+
+static int ndb_write_profile_search_index(struct ndb_lmdb *lmdb,
+					  MDB_txn *txn,
+					  struct ndb_search_key *index_key,
+					  uint64_t profile_key)
+{
+	int rc;
+	MDB_val key, val;
+	
+	key.mv_data = index_key;
+	key.mv_size = sizeof(*index_key);
+	val.mv_data = &profile_key;
+	val.mv_size = sizeof(profile_key);
+
+	if ((rc = mdb_put(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], &key, &val, 0))) {
+		ndb_debug("ndb_write_profile_search_index failed: %s\n",
+			  mdb_strerror(rc));
+		return 0;
+	}
+
+	return 1;
+}
+
+
+// map usernames and display names to profile keys for user searching
+static int ndb_write_profile_search_indices(struct ndb_lmdb *lmdb,
+					    MDB_txn *txn,
+					    struct ndb_note *note,
+					    uint64_t profile_key,
+					    void *profile_root)
+{
+	struct ndb_search_key index;
+	NdbProfileRecord_table_t profile_record;
+	NdbProfile_table_t profile;
+
+	profile_record = NdbProfileRecord_as_root(profile_root);
+	profile = NdbProfileRecord_profile_get(profile_record);
+
+	const char *name = NdbProfile_name_get(profile);
+	const char *display_name = NdbProfile_display_name_get(profile);
+
+	// words + pubkey + created
+	if (name) {
+		ndb_make_search_key(&index, note->pubkey, note->created_at,
+				    name);
+		if (!ndb_write_profile_search_index(lmdb, txn, &index,
+						    profile_key))
+			return 0;
+	}
+
+	if (display_name) {
+		// don't write the same name/display_name twice
+		if (name && !strcmp(display_name, name)) {
+			return 1;
+		}
+		ndb_make_search_key(&index, note->pubkey, note->created_at,
+				    display_name);
+		if (!ndb_write_profile_search_index(lmdb, txn, &index,
+						    profile_key))
+			return 0;
+	}
+
+	return 1;
+}
+
+int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn)
+{
+	txn->ndb = ndb;
+	MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn;
+	return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0;
+}
+
+
 // Migrations
 //
 
 static int ndb_migrate_user_search_indices(struct ndb *ndb)
 {
+	int rc;
+	MDB_cursor *cur;
+	MDB_val k, v;
+	void *profile_root;
+	NdbProfileRecord_table_t record;
+	struct ndb_txn txn;
+	struct ndb_note *note;
+	uint64_t note_key, profile_key;
+	size_t len;
+	int count;
+
+	if (!ndb_begin_query(ndb, &txn)) {
+		fprintf(stderr, "ndb_migrate_user_search_indices: ndb_begin_query failed\n");
+		return 0;
+	}
+
+	if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) {
+		fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc);
+		return 0;
+	}
+
+	count = 0;
+
+	// loop through all profiles and write search indices
+	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
+		profile_root = v.mv_data;
+		profile_key = *((uint64_t*)k.mv_data);
+		record = NdbProfileRecord_as_root(profile_root);
+		note_key = NdbProfileRecord_note_key(record);
+		note = ndb_get_note_by_key(&txn, note_key, &len);
+
+		if (note == NULL) {
+			fprintf(stderr, "ndb_migrate_user_search_indices: note lookup failed\n");
+			return 0;
+		}
+
+		if (!ndb_write_profile_search_indices(&ndb->lmdb, txn.mdb_txn,
+						      note, profile_key,
+						      profile_root)) {
+
+			fprintf(stderr, "ndb_migrate_user_search_indices: ndb_write_profile_search_indices failed\n");
+			return 0;
+		}
+
+		count++;
+	}
+
+	fprintf(stderr, "migrated %d profiles to include search indices\n", count);
+
+	mdb_cursor_close(cur);
+	mdb_txn_commit(txn.mdb_txn);
+
 	return 1;
 }
 
 static struct ndb_migration MIGRATIONS[] = {
-	//{ .fn = ndb_migrate_user_search_indices }
+	{ .fn = ndb_migrate_user_search_indices }
 };
 
 
@@ -187,6 +321,7 @@ static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b)
 {
 	struct ndb_tsid *tsa, *tsb;
 	MDB_val a2 = *a, b2 = *b;
+
 	a2.mv_size = sizeof(tsa->id);
 	b2.mv_size = sizeof(tsb->id);
 
@@ -271,13 +406,6 @@ struct ndb_writer_msg {
 	};
 };
 
-int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn)
-{
-	txn->ndb = ndb;
-	MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn;
-	return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0;
-}
-
 void ndb_end_query(struct ndb_txn *txn)
 {
 	mdb_txn_abort(txn->mdb_txn);
@@ -674,6 +802,113 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db)
         return *((uint64_t*)key.mv_data);
 }
 
+// make a search key meant for user queries without any other note info
+static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search)
+{
+	memset(key->id, 0, sizeof(key->id));
+	key->timestamp = 0;
+	strncpy(key->search, search, sizeof(key->search) - 1);
+	key->search[sizeof(key->search) - 1] = '\0';
+}
+
+int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query)
+{
+	int rc;
+	struct ndb_search_key s;
+	MDB_val k, v;
+	search->cursor = NULL;
+
+	MDB_cursor **cursor = (MDB_cursor **)&search->cursor;
+
+	ndb_make_search_key_low(&s, query);
+
+	k.mv_data = &s;
+	k.mv_size = sizeof(s);
+
+	if ((rc = mdb_cursor_open(txn->mdb_txn,
+			    txn->ndb->lmdb.dbs[NDB_DB_PROFILE_SEARCH],
+			    cursor))) {
+		printf("search_profile: cursor opened failed: %s\n",
+				mdb_strerror(rc));
+		return 0;
+	}
+
+	// Position cursor at the next key greater than or equal to the specified key
+	if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) {
+		printf("search_profile: cursor get failed\n");
+		goto cleanup;
+	} else {
+		search->key = k.mv_data;
+		assert(v.mv_size == 8);
+		search->profile_key = *((uint64_t*)v.mv_data);
+		return 1;
+	}
+
+cleanup:
+	mdb_cursor_close(search->cursor);
+	search->cursor = NULL;
+	return 0;
+}
+
+void ndb_search_profile_end(struct ndb_search *search)
+{
+	if (search->cursor)
+		mdb_cursor_close(search->cursor);
+}
+
+int ndb_search_profile_next(struct ndb_search *search)
+{
+	int rc;
+	MDB_val k, v;
+	unsigned char *init_id;
+
+	init_id = search->key->id;
+	k.mv_data = search->key;
+	k.mv_size = sizeof(*search->key);
+
+retry:
+	if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) {
+		ndb_debug("ndb_search_profile_next: %s\n",
+				mdb_strerror(rc));
+		return 0;
+	} else {
+		search->key = k.mv_data;
+		assert(v.mv_size == 8);
+		search->profile_key = *((uint64_t*)v.mv_data);
+
+		// skip duplicate pubkeys
+		if (!memcmp(init_id, search->key->id, 32))
+			goto retry;
+	}
+
+	return 1;
+}
+
+static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b)
+{
+	int cmp;
+	struct ndb_search_key *ska, *skb;
+
+	ska = a->mv_data;
+	skb = b->mv_data;
+
+	MDB_val a2 = *a;
+	MDB_val b2 = *b;
+
+	a2.mv_data = ska->search;
+	a2.mv_size = sizeof(ska->search) + sizeof(ska->id);
+
+	cmp = mdb_cmp_memn(&a2, &b2);
+	if (cmp) return cmp;
+
+	if (ska->timestamp < skb->timestamp)
+		return -1;
+	else if (ska->timestamp > skb->timestamp)
+		return 1;
+
+	return 0;
+}
+
 static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
 			     struct ndb_writer_profile *profile,
 			     uint64_t note_key)
@@ -737,6 +972,13 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
 		return 0;
 	}
 
+	// write name, display_name profile search indices
+	if (!ndb_write_profile_search_indices(lmdb, txn, note, profile_key,
+					      flatbuf)) {
+		ndb_debug("failed to write profile search indices\n");
+		return 0;
+	}
+
 	return 1;
 }
 
@@ -803,6 +1045,8 @@ static void ndb_write_version(struct ndb_lmdb *lmdb, MDB_txn *txn, uint64_t vers
 				mdb_strerror(rc));
 		return;
 	}
+
+	fprintf(stderr, "writing version %" PRIu64 "\n", version);
 }
 
 static void *ndb_writer_thread(void *data)
@@ -1090,6 +1334,13 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map
 		return 0;
 	}
 
+	// profile search db
+	if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) {
+		fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc);
+		return 0;
+	}
+	mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp);
+
 	// ndb metadata (db version, etc)
 	if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) {
 		fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc);
@@ -1137,6 +1388,7 @@ static int ndb_run_migrations(struct ndb *ndb)
 	latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]);
 
 	if ((version = ndb_db_version(ndb)) == -1) {
+		fprintf(stderr, "run_migrations: no version found, assuming new db\n");
 		version = latest_version;
 
 		// no version found. fresh db?
@@ -1146,6 +1398,8 @@ static int ndb_run_migrations(struct ndb *ndb)
 		}
 
 		return 1;
+	} else {
+		fprintf(stderr, "ndb: version %" PRIu64 " found\n", version);
 	}
 
 	if (version < latest_version)

nostrdb/nostrdb.h

@@ -25,6 +25,19 @@ struct ndb_t {
 	struct ndb *ndb;
 };
 
+struct ndb_search_key
+{
+	char search[24];
+	unsigned char id[32];
+	uint64_t timestamp;
+};
+
+struct ndb_search {
+	struct ndb_search_key *key;
+	uint64_t profile_key;
+	void *cursor; // MDB_cursor *
+};
+
 // required to keep a read 
 struct ndb_txn {
 	struct ndb *ndb;
@@ -165,6 +178,9 @@ int ndb_db_version(struct ndb *ndb);
 int ndb_process_event(struct ndb *, const char *json, int len);
 int ndb_process_events(struct ndb *, const char *ldjson, size_t len);
 int ndb_begin_query(struct ndb *, struct ndb_txn *);
+int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query);
+int ndb_search_profile_next(struct ndb_search *search);
+void ndb_search_profile_end(struct ndb_search *search);
 void ndb_end_query(struct ndb_txn *);
 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey);
 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);