nostr/gossip
1
0
Fork 0
mirror of https://github.com/mikedilger/gossip.git synced 2024-09-29 08:21:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

storage: migration 11: Remove event_references_person

Browse Source
This commit is contained in:
Mike Dilger 2023-10-07 08:38:11 +13:00
parent 3edbb73f81
commit 82b6ca53a6
7 changed files with 31 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
gossip-bin/src/ui/help/stats.rs
View File

@ -82,15 +82,6 @@ pub(super) fn update(app: &mut GossipUi, _ctx: &Context, _frame: &mut eframe::Fr
));
ui.add_space(6.0);
ui.label(format!(
"Event Index (References Person): {} records",
GLOBALS
.storage
.get_event_references_person_len()
.unwrap_or(0)
));
ui.add_space(6.0);
ui.label(format!(
"Event Relationships: {} records",
GLOBALS.storage.get_relationships_len().unwrap_or(0)

19
gossip-lib/src/feed.rs
View File

@ -367,7 +367,7 @@ impl Feed {
if e.created_at < since || e.created_at > now {
return false;
}
if ! kinds_with_dms.contains(&e.kind) {
if !kinds_with_dms.contains(&e.kind) {
return false;
}
if dismissed.contains(&e.id) {
@ -426,7 +426,7 @@ impl Feed {
if dismissed.contains(&e.id) {
return false;
}
if ! kinds_without_dms.contains(&e.kind) {
if !kinds_without_dms.contains(&e.kind) {
return false;
}
true
@ -439,26 +439,19 @@ impl Feed {
&[person_pubkey],
Some(since),
filter,
false
false,
)?
.iter()
.chain(
GLOBALS
.storage
.find_tagged_events(
"delegation",
Some(pphex.as_str()),
filter,
false
)?
.iter()
.find_tagged_events("delegation", Some(pphex.as_str()), filter, false)?
.iter(),
)
.map(|e| e.to_owned())
.collect();
events.sort_by(|a,b| b.created_at.cmp(&a.created_at).then(
b.id.cmp(&a.id)
));
events.sort_by(|a, b| b.created_at.cmp(&a.created_at).then(b.id.cmp(&a.id)));
let events: Vec<Id> = events.iter().map(|e| e.id).collect();

163
gossip-lib/src/storage/event_references_person1.rs
View File

@ -1,163 +0,0 @@
use crate::error::{Error, ErrorKind};
use crate::globals::GLOBALS;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags, RwTxn};
use nostr_types::{Event, EventKind, Id, PublicKey, Unixtime};
use speedy::Readable;
use std::cmp::Ordering;
use std::collections::HashSet;
use std::ops::Bound;
use std::sync::Mutex;
// PublicKey:ReverseUnixtime -> Id
// (pubkey is referenced by the event somehow)
// (only feed-displayable events are included)
// (dup keys, so multiple Ids per key)
// NOTE: this may be far too much data. Maybe we should only build this for the
// user's pubkey as their inbox.
static EVENT_REFERENCES_PERSON1_DB_CREATE_LOCK: Mutex<()> = Mutex::new(());
static mut EVENT_REFERENCES_PERSON1_DB: Option<RawDatabase> = None;
impl Storage {
pub(super) fn db_event_references_person1(&self) -> Result<RawDatabase, Error> {
unsafe {
if let Some(db) = EVENT_REFERENCES_PERSON1_DB {
Ok(db)
} else {
// Lock. This drops when anything returns.
let _lock = EVENT_REFERENCES_PERSON1_DB_CREATE_LOCK.lock();
// In case of a race, check again
if let Some(db) = EVENT_REFERENCES_PERSON1_DB {
return Ok(db);
}
// Create it. We know that nobody else is doing this and that
// it cannot happen twice.
let mut txn = self.env.write_txn()?;
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_references_person")
.create(&mut txn)?;
txn.commit()?;
EVENT_REFERENCES_PERSON1_DB = Some(db);
Ok(db)
}
}
}
pub(crate) fn write_event_references_person1<'a>(
&'a self,
event: &Event,
rw_txn: Option<&mut RwTxn<'a>>,
) -> Result<(), Error> {
let f = |txn: &mut RwTxn<'a>| -> Result<(), Error> {
let mut event = event;
// If giftwrap, index the inner rumor instead
let mut rumor_event: Event;
if event.kind == EventKind::GiftWrap {
match GLOBALS.signer.unwrap_giftwrap(event) {
Ok(rumor) => {
rumor_event = rumor.into_event_with_bad_signature();
rumor_event.id = event.id; // lie, so it indexes it under the giftwrap
event = &rumor_event;
}
Err(e) => {
if matches!(e.kind, ErrorKind::NoPrivateKey) {
// Store as unindexed for later indexing
let bytes = vec![];
self.db_unindexed_giftwraps()?
.put(txn, event.id.as_slice(), &bytes)?;
}
}
}
}
if !event.kind.is_feed_displayable() {
return Ok(());
}
let bytes = event.id.as_slice();
let mut pubkeys: HashSet<PublicKey> = HashSet::new();
for (pubkeyhex, _, _) in event.people() {
let pubkey = match PublicKey::try_from_hex_string(pubkeyhex.as_str(), false) {
Ok(pk) => pk,
Err(_) => continue,
};
pubkeys.insert(pubkey);
}
for pubkey in event.people_referenced_in_content() {
pubkeys.insert(pubkey);
}
if !pubkeys.is_empty() {
for pubkey in pubkeys.drain() {
let mut key: Vec<u8> = pubkey.to_bytes();
key.extend((i64::MAX - event.created_at.0).to_be_bytes().as_slice()); // reverse created_at
self.db_event_references_person1()?.put(txn, &key, bytes)?;
}
}
Ok(())
};
match rw_txn {
Some(txn) => f(txn)?,
None => {
let mut txn = self.env.write_txn()?;
f(&mut txn)?;
txn.commit()?;
}
};
Ok(())
}
// Read all events referencing a given person in reverse time order
pub(crate) fn read_events_referencing_person1<F>(
&self,
pubkey: &PublicKey,
since: Unixtime,
f: F,
) -> Result<Vec<Event>, Error>
where
F: Fn(&Event) -> bool,
{
let txn = self.env.read_txn()?;
let now = Unixtime::now().unwrap();
let mut start_key: Vec<u8> = pubkey.to_bytes();
let mut end_key: Vec<u8> = start_key.clone();
start_key.extend((i64::MAX - now.0).to_be_bytes().as_slice()); // work back from now
end_key.extend((i64::MAX - since.0).to_be_bytes().as_slice()); // until since
let range = (Bound::Included(&*start_key), Bound::Excluded(&*end_key));
let iter = self.db_event_references_person1()?.range(&txn, &range)?;
let mut events: Vec<Event> = Vec::new();
for result in iter {
let (_key, val) = result?;
// Take the event
let id = Id(val[0..32].try_into()?);
// (like read_event, but we supply our on transaction)
if let Some(bytes) = self.db_events1()?.get(&txn, id.as_slice())? {
let event = Event::read_from_buffer(bytes)?;
if f(&event) {
events.push(event);
}
}
}
// We have to sort these because (pubkey/unixtime) isn't unique.
// The sort should be pretty fast given they are already nearly sorted.
events.sort_by(|a, b| match b.created_at.cmp(&a.created_at) {
Ordering::Equal => b.id.cmp(&a.id),
ordered => ordered,
});
Ok(events)
}
}

1
gossip-lib/src/storage/events1.rs
View File

@ -59,7 +59,6 @@ impl Storage {
self.write_event_ek_pk_index(event, Some(txn))?;
self.write_event_ek_c_index(event, Some(txn))?;
self.write_event_tag_index(event, Some(txn))?;
self.write_event_references_person(event, Some(txn))?;
for hashtag in event.hashtags() {
if hashtag.is_empty() {
continue;

28
gossip-lib/src/storage/migrations/mod.rs
View File

@ -2,12 +2,13 @@ use super::types::{Person2, PersonRelay1, Settings1, Settings2, Theme1, ThemeVar
use super::Storage;
use crate::error::{Error, ErrorKind};
use crate::people::PersonList;
use heed::RwTxn;
use heed::types::UnalignedSlice;
use heed::{DatabaseFlags, RwTxn};
use nostr_types::{Event, Id, RelayUrl, Signature};
use speedy::{Readable, Writable};
impl Storage {
const MAX_MIGRATION_LEVEL: u32 = 11;
const MAX_MIGRATION_LEVEL: u32 = 12;
pub(super) fn migrate(&self, mut level: u32) -> Result<(), Error> {
if level > Self::MAX_MIGRATION_LEVEL {
@ -57,7 +58,6 @@ impl Storage {
let _ = self.db_events1()?;
let _ = self.db_event_ek_pk_index1()?;
let _ = self.db_event_ek_c_index1()?;
let _ = self.db_event_references_person1()?;
let _ = self.db_hashtags1()?;
}
10 => {
@ -119,6 +119,10 @@ impl Storage {
tracing::info!("{prefix}: populating event tag index...");
self.populate_event_tag_index(txn)?;
}
11 => {
tracing::info!("{prefix}: removing now unused event_references_person index...");
self.remove_event_references_person(txn)?;
}
_ => panic!("Unreachable migration level"),
};
@ -535,4 +539,22 @@ impl Storage {
Ok(())
}
pub fn remove_event_references_person<'a>(&'a self, txn: &mut RwTxn<'a>) -> Result<(), Error> {
{
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_references_person")
.create(txn)?;
db.clear(txn)?;
}
// heed doesn't expose mdb_drop(1) yet, so we can't actually remove this database.
Ok(())
}
}

39
gossip-lib/src/storage/mod.rs
View File

@ -19,7 +19,6 @@ pub mod types;
// database implementations
mod event_ek_c_index1;
mod event_ek_pk_index1;
mod event_references_person1;
mod event_seen_on_relay1;
mod event_tag_index1;
mod event_viewed1;
@ -181,7 +180,6 @@ impl Storage {
// triggered into existence if their migration is necessary.
let _ = self.db_event_ek_c_index()?;
let _ = self.db_event_ek_pk_index()?;
let _ = self.db_event_references_person()?;
let _ = self.db_event_tag_index()?;
let _ = self.db_events()?;
let _ = self.db_event_seen_on_relay()?;
@ -234,11 +232,6 @@ impl Storage {
self.db_event_ek_pk_index1()
}
#[inline]
pub(crate) fn db_event_references_person(&self) -> Result<RawDatabase, Error> {
self.db_event_references_person1()
}
#[inline]
pub(crate) fn db_event_tag_index(&self) -> Result<RawDatabase, Error> {
self.db_event_tag_index1()
@ -344,12 +337,6 @@ impl Storage {
Ok(self.db_event_ek_c_index()?.len(&txn)?)
}
/// The number of records in the event_references_person index table
pub fn get_event_references_person_len(&self) -> Result<u64, Error> {
let txn = self.env.read_txn()?;
Ok(self.db_event_references_person()?.len(&txn)?)
}
/// The number of records in the event_tag index table
pub fn get_event_tag_index_len(&self) -> Result<u64, Error> {
let txn = self.env.read_txn()?;
@ -1731,30 +1718,6 @@ impl Storage {
Ok(events)
}
// We don't call this externally. Whenever we write an event, we do this.
#[inline]
fn write_event_references_person<'a>(
&'a self,
event: &Event,
rw_txn: Option<&mut RwTxn<'a>>,
) -> Result<(), Error> {
self.write_event_references_person1(event, rw_txn)
}
/// Read all events referencing a given person in reverse time order
#[inline]
pub fn read_events_referencing_person<F>(
&self,
pubkey: &PublicKey,
since: Unixtime,
f: F,
) -> Result<Vec<Event>, Error>
where
F: Fn(&Event) -> bool,
{
self.read_events_referencing_person1(pubkey, since, f)
}
#[inline]
pub(crate) fn index_unindexed_giftwraps(&self) -> Result<(), Error> {
self.index_unindexed_giftwraps1()
@ -2211,7 +2174,6 @@ impl Storage {
self.db_event_ek_pk_index()?.clear(txn)?;
self.db_event_ek_c_index()?.clear(txn)?;
self.db_event_tag_index()?.clear(txn)?;
self.db_event_references_person()?.clear(txn)?;
self.db_hashtags()?.clear(txn)?;
let loop_txn = self.env.read_txn()?;
@ -2221,7 +2183,6 @@ impl Storage {
self.write_event_ek_pk_index(&event, Some(txn))?;
self.write_event_ek_c_index(&event, Some(txn))?;
self.write_event_tag_index(&event, Some(txn))?;
self.write_event_references_person(&event, Some(txn))?;
for hashtag in event.hashtags() {
if hashtag.is_empty() {
continue;

1
gossip-lib/src/storage/unindexed_giftwraps1.rs
View File

@ -64,7 +64,6 @@ impl Storage {
self.write_event_ek_pk_index(&event, Some(&mut txn))?;
self.write_event_ek_c_index(&event, Some(&mut txn))?;
self.write_event_tag_index(&event, Some(&mut txn))?;
self.write_event_references_person(&event, Some(&mut txn))?;
}
self.db_unindexed_giftwraps1()?
.delete(&mut txn, id.as_slice())?;