mirror of
git://jb55.com/damus
synced 2024-09-07 05:43:42 +00:00
nostrdb: add migration to fix local japanese profile names
This commit is contained in:
parent
2895c374c0
commit
cc385d3c3f
@ -64,6 +64,14 @@ struct ndb_ingest_controller
|
|||||||
struct ndb_lmdb *lmdb;
|
struct ndb_lmdb *lmdb;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum ndb_writer_msgtype {
|
||||||
|
NDB_WRITER_QUIT, // kill thread immediately
|
||||||
|
NDB_WRITER_NOTE, // write a note to the db
|
||||||
|
NDB_WRITER_PROFILE, // write a profile to the db
|
||||||
|
NDB_WRITER_DBMETA, // write ndb metadata
|
||||||
|
NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched
|
||||||
|
};
|
||||||
|
|
||||||
enum ndb_dbs {
|
enum ndb_dbs {
|
||||||
NDB_DB_NOTE,
|
NDB_DB_NOTE,
|
||||||
NDB_DB_META,
|
NDB_DB_META,
|
||||||
@ -121,6 +129,7 @@ struct ndb {
|
|||||||
// lmdb environ handles, etc
|
// lmdb environ handles, etc
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
// A clustered key with an id and a timestamp
|
// A clustered key with an id and a timestamp
|
||||||
struct ndb_tsid {
|
struct ndb_tsid {
|
||||||
unsigned char id[32];
|
unsigned char id[32];
|
||||||
@ -147,7 +156,6 @@ static void lowercase_strncpy(char *dst, const char *src, int n) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
|
static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
|
||||||
uint64_t timestamp, const char *search)
|
uint64_t timestamp, const char *search)
|
||||||
{
|
{
|
||||||
@ -319,10 +327,7 @@ static int ndb_migrate_lower_user_search_indices(struct ndb *ndb)
|
|||||||
return ndb_migrate_user_search_indices(ndb);
|
return ndb_migrate_user_search_indices(ndb);
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct ndb_migration MIGRATIONS[] = {
|
int ndb_process_profile_note(struct ndb_note *note, struct ndb_profile_record_builder *profile);
|
||||||
{ .fn = ndb_migrate_user_search_indices },
|
|
||||||
{ .fn = ndb_migrate_lower_user_search_indices }
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
int ndb_db_version(struct ndb *ndb)
|
int ndb_db_version(struct ndb *ndb)
|
||||||
@ -418,14 +423,6 @@ enum ndb_ingester_msgtype {
|
|||||||
NDB_INGEST_QUIT, // kill ingester thread immediately
|
NDB_INGEST_QUIT, // kill ingester thread immediately
|
||||||
};
|
};
|
||||||
|
|
||||||
enum ndb_writer_msgtype {
|
|
||||||
NDB_WRITER_QUIT, // kill thread immediately
|
|
||||||
NDB_WRITER_NOTE, // write a note to the db
|
|
||||||
NDB_WRITER_PROFILE, // write a profile to the db
|
|
||||||
NDB_WRITER_DBMETA, // write ndb metadata
|
|
||||||
NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched
|
|
||||||
};
|
|
||||||
|
|
||||||
struct ndb_ingester_event {
|
struct ndb_ingester_event {
|
||||||
char *json;
|
char *json;
|
||||||
unsigned client : 1; // ["EVENT", {...}] messages
|
unsigned client : 1; // ["EVENT", {...}] messages
|
||||||
@ -473,6 +470,92 @@ struct ndb_writer_msg {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static inline int ndb_writer_queue_msg(struct ndb_writer *writer,
|
||||||
|
struct ndb_writer_msg *msg)
|
||||||
|
{
|
||||||
|
return prot_queue_push(&writer->inbox, msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int ndb_migrate_utf8_profile_names(struct ndb *ndb)
|
||||||
|
{
|
||||||
|
int rc;
|
||||||
|
MDB_cursor *cur;
|
||||||
|
MDB_val k, v;
|
||||||
|
void *profile_root;
|
||||||
|
NdbProfileRecord_table_t record;
|
||||||
|
struct ndb_txn txn;
|
||||||
|
struct ndb_note *note, *copied_note;
|
||||||
|
uint64_t note_key;
|
||||||
|
size_t len;
|
||||||
|
int count, failed;
|
||||||
|
struct ndb_writer_msg out;
|
||||||
|
|
||||||
|
if (!ndb_begin_rw_query(ndb, &txn)) {
|
||||||
|
fprintf(stderr, "ndb_migrate_utf8_profile_names: ndb_begin_rw_query failed\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) {
|
||||||
|
fprintf(stderr, "ndb_migrate_utf8_profile_names: mdb_cursor_open failed, error %d\n", rc);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
count = 0;
|
||||||
|
failed = 0;
|
||||||
|
|
||||||
|
// loop through all profiles and write search indices
|
||||||
|
while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
|
||||||
|
profile_root = v.mv_data;
|
||||||
|
record = NdbProfileRecord_as_root(profile_root);
|
||||||
|
note_key = NdbProfileRecord_note_key(record);
|
||||||
|
note = ndb_get_note_by_key(&txn, note_key, &len);
|
||||||
|
|
||||||
|
if (note == NULL) {
|
||||||
|
fprintf(stderr, "ndb_migrate_utf8_profile_names: note lookup failed\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ndb_profile_record_builder *b = &out.profile.record;
|
||||||
|
|
||||||
|
// reprocess profile
|
||||||
|
if (!ndb_process_profile_note(note, b)) {
|
||||||
|
failed++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// the writer needs to own this note, and its expected to free it
|
||||||
|
copied_note = malloc(len);
|
||||||
|
memcpy(copied_note, note, len);
|
||||||
|
|
||||||
|
out.type = NDB_WRITER_PROFILE;
|
||||||
|
out.profile.note.note = copied_note;
|
||||||
|
out.profile.note.note_len = len;
|
||||||
|
|
||||||
|
ndb_writer_queue_msg(&ndb->writer, &out);
|
||||||
|
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "migrated %d profiles to fix utf8 profile names\n", count);
|
||||||
|
|
||||||
|
if (failed != 0) {
|
||||||
|
fprintf(stderr, "failed to migrate %d profiles to fix utf8 profile names\n", failed);
|
||||||
|
}
|
||||||
|
|
||||||
|
mdb_cursor_close(cur);
|
||||||
|
|
||||||
|
ndb_end_query(&txn);
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static struct ndb_migration MIGRATIONS[] = {
|
||||||
|
{ .fn = ndb_migrate_user_search_indices },
|
||||||
|
{ .fn = ndb_migrate_lower_user_search_indices },
|
||||||
|
{ .fn = ndb_migrate_utf8_profile_names }
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
int ndb_end_query(struct ndb_txn *txn)
|
int ndb_end_query(struct ndb_txn *txn)
|
||||||
{
|
{
|
||||||
// this works on read or write queries.
|
// this works on read or write queries.
|
||||||
@ -496,12 +579,6 @@ int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32],
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline int ndb_writer_queue_msg(struct ndb_writer *writer,
|
|
||||||
struct ndb_writer_msg *msg)
|
|
||||||
{
|
|
||||||
return prot_queue_push(&writer->inbox, msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline int ndb_writer_queue_msgs(struct ndb_writer *writer,
|
static inline int ndb_writer_queue_msgs(struct ndb_writer *writer,
|
||||||
struct ndb_writer_msg *msgs,
|
struct ndb_writer_msg *msgs,
|
||||||
int num_msgs)
|
int num_msgs)
|
||||||
@ -814,8 +891,8 @@ void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b)
|
|||||||
b->flatbuf = NULL;
|
b->flatbuf = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int ndb_process_profile_note(struct ndb_note *note,
|
int ndb_process_profile_note(struct ndb_note *note,
|
||||||
struct ndb_profile_record_builder *profile)
|
struct ndb_profile_record_builder *profile)
|
||||||
{
|
{
|
||||||
int res;
|
int res;
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user