nostr/damus
1
0
Fork 0
mirror of git://jb55.com/damus synced 2024-10-04 19:00:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

nostrdb/flag: add ndb config flag for skipping note verification

Browse Source 
makes some large imports a bit faster

Signed-off-by: William Casarin <jb55@jb55.com>
This commit is contained in:
William Casarin 2023-11-30 09:44:10 -08:00
parent b74bde5cc4
commit 213a26cd01
2 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
nostrdb/nostrdb.c
View File

@ -112,6 +112,7 @@ struct ndb_writer {
}; };
struct ndb_ingester { struct ndb_ingester {
uint32_t flags;
struct threadpool tp; struct threadpool tp;
struct ndb_writer *writer; struct ndb_writer *writer;
}; };
@ -122,6 +123,7 @@ struct ndb {
struct ndb_ingester ingester; struct ndb_ingester ingester;
struct ndb_writer writer; struct ndb_writer writer;
int version; int version;
uint32_t flags; // setting flags
// lmdb environ handles, etc // lmdb environ handles, etc
}; };
@ -1540,17 +1542,22 @@ int ndb_process_profile_note(struct ndb_note *note,
static int ndb_ingester_process_note(secp256k1_context *ctx, static int ndb_ingester_process_note(secp256k1_context *ctx,
struct ndb_note *note, struct ndb_note *note,
size_t note_size, size_t note_size,
struct ndb_writer_msg *out) struct ndb_writer_msg *out,
uint32_t flags)
{ {
//printf("ndb_ingester_process_note "); //printf("ndb_ingester_process_note ");
//print_hex(note->id, 32); //print_hex(note->id, 32);
//printf("\n"); //printf("\n");
// Verify! If it's an invalid note we don't need to // some special situations we might want to skip sig validation,
// bother writing it to the database // like during large imports
if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) { if (!(flags & NDB_FLAG_SKIP_NOTE_VERIFY)) {
ndb_debug("signature verification failed\n"); // Verify! If it's an invalid note we don't need to
return 0; // bother writing it to the database
if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) {
ndb_debug("signature verification failed\n");
return 0;
}
} }
// we didn't find anything. let's send it // we didn't find anything. let's send it
@ -1636,9 +1643,10 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
goto cleanup; goto cleanup;
} }
if (!ndb_ingester_process_note(ctx, note, note_size, out)) if (!ndb_ingester_process_note(ctx, note, note_size,
out, ingester->flags)) {
goto cleanup; goto cleanup;
else { } else {
// we're done with the original json, free it // we're done with the original json, free it
free(ev->json); free(ev->json);
return 1; return 1;
@ -1656,9 +1664,10 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
goto cleanup; goto cleanup;
} }
if (!ndb_ingester_process_note(ctx, note, note_size, out)) if (!ndb_ingester_process_note(ctx, note, note_size,
out, ingester->flags)) {
goto cleanup; goto cleanup;
else { } else {
// we're done with the original json, free it // we're done with the original json, free it
free(ev->json); free(ev->json);
return 1; return 1;
@ -2902,7 +2911,8 @@ static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb)
// initialize the ingester queue and then spawn the thread // initialize the ingester queue and then spawn the thread
static int ndb_ingester_init(struct ndb_ingester *ingester, static int ndb_ingester_init(struct ndb_ingester *ingester,
struct ndb_writer *writer, int num_threads) struct ndb_writer *writer, int num_threads,
int flags)
{ {
int elem_size, num_elems; int elem_size, num_elems;
static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT }; static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT };
@ -2912,6 +2922,7 @@ static int ndb_ingester_init(struct ndb_ingester *ingester,
num_elems = DEFAULT_QUEUE_SIZE; num_elems = DEFAULT_QUEUE_SIZE;
ingester->writer = writer; ingester->writer = writer;
ingester->flags = flags;
if (!threadpool_init(&ingester->tp, num_threads, elem_size, num_elems, if (!threadpool_init(&ingester->tp, num_threads, elem_size, num_elems,
&quit_msg, ingester, ndb_ingester_thread)) &quit_msg, ingester, ndb_ingester_thread))
@ -3127,6 +3138,8 @@ int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingest
//MDB_dbi ind_id; // TODO: ind_pk, etc //MDB_dbi ind_id; // TODO: ind_pk, etc
ndb = *pndb = calloc(1, sizeof(struct ndb)); ndb = *pndb = calloc(1, sizeof(struct ndb));
ndb->flags = flags;
if (ndb == NULL) { if (ndb == NULL) {
fprintf(stderr, "ndb_init: malloc failed\n"); fprintf(stderr, "ndb_init: malloc failed\n");
return 0; return 0;
@ -3140,7 +3153,8 @@ int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingest
return 0; return 0;
} }
if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads)) { if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads,
ndb->flags)) {
fprintf(stderr, "failed to initialize %d ingester thread(s)\n", fprintf(stderr, "failed to initialize %d ingester thread(s)\n",
ingester_threads); ingester_threads);
return 0; return 0;

1
nostrdb/nostrdb.h
View File

@ -8,6 +8,7 @@
#define NDB_PACKED_ID 0x2 #define NDB_PACKED_ID 0x2
#define NDB_FLAG_NOMIGRATE (1 << 0) #define NDB_FLAG_NOMIGRATE (1 << 0)
#define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1)
//#define DEBUG 1 //#define DEBUG 1