Update to heed 0.20

This commit is contained in:
Mike Dilger 2024-05-13 10:03:52 +12:00
parent 9e3950db77
commit 694498522d
34 changed files with 88 additions and 84 deletions

26
Cargo.lock generated
View File

@ -2384,11 +2384,11 @@ dependencies = [
[[package]]
name = "heed"
version = "0.20.0-alpha.8"
source = "git+https://github.com/meilisearch/heed?rev=64fd6fec293c0dee94855b8267557ce03e7ce5d8#64fd6fec293c0dee94855b8267557ce03e7ce5d8"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a300b0deeb2957162d7752b0f063b3be1c88333af5bb4e7a57d8fb3716f50b"
dependencies = [
"bitflags 2.5.0",
"bytemuck",
"byteorder",
"heed-traits",
"heed-types",
@ -2403,16 +2403,17 @@ dependencies = [
[[package]]
name = "heed-traits"
version = "0.20.0-alpha.8"
source = "git+https://github.com/meilisearch/heed?rev=64fd6fec293c0dee94855b8267557ce03e7ce5d8#64fd6fec293c0dee94855b8267557ce03e7ce5d8"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
[[package]]
name = "heed-types"
version = "0.20.0-alpha.8"
source = "git+https://github.com/meilisearch/heed?rev=64fd6fec293c0dee94855b8267557ce03e7ce5d8#64fd6fec293c0dee94855b8267557ce03e7ce5d8"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cb0d6ba3700c9a57e83c013693e3eddb68a6d9b6781cacafc62a0d992e8ddb3"
dependencies = [
"bincode",
"bytemuck",
"byteorder",
"heed-traits",
"serde",
@ -2936,8 +2937,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
[[package]]
name = "lmdb-master-sys"
version = "0.1.0"
source = "git+https://github.com/meilisearch/heed?rev=64fd6fec293c0dee94855b8267557ce03e7ce5d8#64fd6fec293c0dee94855b8267557ce03e7ce5d8"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc9048db3a58c0732d7236abc4909058f9d2708cfb6d7d047eb895fddec6419a"
dependencies = [
"cc",
"doxygen-rs",
@ -3533,9 +3535,9 @@ dependencies = [
[[package]]
name = "page_size"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",

View File

@ -51,7 +51,7 @@ filetime = "0.2"
futures = "0.3"
futures-util = "0.3"
gossip-relay-picker = { git = "https://github.com/mikedilger/gossip-relay-picker", rev = "6e67994be02d7504069de7579b369972b2bdf318" }
heed = { git = "https://github.com/meilisearch/heed", rev = "64fd6fec293c0dee94855b8267557ce03e7ce5d8" }
heed = "0.20"
hex = "0.4"
http = "1.0"
image = { version = "0.24.6", features = [ "png", "jpeg" ] }

View File

@ -1,6 +1,6 @@
use crate::error::{Error, ErrorKind};
use crate::storage::{EmptyDatabase, Storage};
use heed::types::{UnalignedSlice, Unit};
use heed::types::{Bytes, Unit};
use nostr_types::{EventKind, Id, PublicKey, Unixtime};
use std::sync::Mutex;
@ -29,7 +29,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, Unit>()
.types::<Bytes, Unit>()
.name("event_akci_index")
.create(&mut txn)?;
txn.commit()?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags};
use heed::{types::Bytes, DatabaseFlags};
use std::sync::Mutex;
// EventKind::ReverseUnixtime -> Id
@ -30,7 +30,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_ek_c_index")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags};
use heed::{types::Bytes, DatabaseFlags};
use std::sync::Mutex;
// EventKind:PublicKey -> Id
@ -31,7 +31,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_ek_pk_index")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::{Error, ErrorKind};
use crate::storage::{EmptyDatabase, Storage};
use heed::types::{UnalignedSlice, Unit};
use heed::types::{Bytes, Unit};
use nostr_types::{EventKind, Id, Unixtime};
use std::sync::Mutex;
@ -37,7 +37,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, Unit>()
.types::<Bytes, Unit>()
.name("event_kci_index")
.create(&mut txn)?;
txn.commit()?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage, MAX_LMDB_KEY};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::{Id, RelayUrl, Unixtime};
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("event_seen_on_relay")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags, RwTxn};
use heed::{types::Bytes, DatabaseFlags, RwTxn};
use nostr_types::{EventV2, EventV3, PublicKeyHex, TagV3};
use std::sync::Mutex;
@ -34,7 +34,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_tag_index")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::Id;
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("event_viewed")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use nostr_types::{EventV1, Id};
use speedy::Readable;
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("events")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::EventV2;
use speedy::Writable;
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("events2")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::{EventV3, Id};
use speedy::{Readable, Writable};
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("events3")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::{Error, ErrorKind};
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags, RwTxn};
use heed::{types::Bytes, DatabaseFlags, RwTxn};
use nostr_types::Id;
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("hashtags")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::Storage;
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::{DatabaseFlags, RwTxn};
impl Storage {
@ -27,7 +27,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED)
.name("event_references_person")
.create(txn)?;

View File

@ -61,7 +61,7 @@ use crate::person_relay::PersonRelay;
use crate::profile::Profile;
use crate::relationship::{RelationshipByAddr, RelationshipById};
use crate::relay::Relay;
use heed::types::{UnalignedSlice, Unit};
use heed::types::{Bytes, Unit};
use heed::{Database, Env, EnvFlags, EnvOpenOptions, RoTxn, RwTxn};
use nostr_types::{
EncryptedPrivateKey, Event, EventAddr, EventKind, EventReference, Filter, Id, MilliSatoshi,
@ -181,8 +181,8 @@ macro_rules! def_flag {
};
}
type RawDatabase = Database<UnalignedSlice<u8>, UnalignedSlice<u8>>;
type EmptyDatabase = Database<UnalignedSlice<u8>, Unit>;
type RawDatabase = Database<Bytes, Bytes>;
type EmptyDatabase = Database<Bytes, Unit>;
/// The LMDB storage engine.
///
@ -213,11 +213,13 @@ impl Storage {
builder.map_size(1048576 * 1024 * 24); // 24 GB
let dir = Profile::current()?.lmdb_dir;
let env = match builder.open(&dir) {
Ok(env) => env,
Err(e) => {
tracing::error!("Unable to open LMDB at {}", dir.display());
return Err(e.into());
let env = unsafe {
match builder.open(&dir) {
Ok(env) => env,
Err(e) => {
tracing::error!("Unable to open LMDB at {}", dir.display());
return Err(e.into());
}
}
};
@ -225,7 +227,7 @@ impl Storage {
let general = env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.create(&mut txn)?;
txn.commit()?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::nip46::Nip46Server;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use speedy::Writable;
use std::sync::Mutex;
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("nip46server1")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::nip46::Nip46Server;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::PublicKey;
use speedy::{Readable, Writable};
@ -34,7 +34,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("nip46server2")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::storage::types::Person1;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::PublicKey;
use std::sync::Mutex;
@ -34,7 +34,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("people")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::storage::types::Person2;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::PublicKey;
use std::sync::Mutex;
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("people2")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::people::PersonList;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::PublicKey;
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_lists")
.create(&mut txn)?;

View File

@ -2,7 +2,7 @@ use super::types::PersonList1;
use crate::error::Error;
use crate::misc::Private;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::PublicKey;
use speedy::{Readable, Writable};
@ -36,7 +36,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_lists_2")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use super::types::{PersonList1, PersonListMetadata1};
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use speedy::{Readable, Writable};
use std::sync::Mutex;
@ -31,7 +31,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_lists_metadata1")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use super::types::{PersonList1, PersonListMetadata2};
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use speedy::{Readable, Writable};
use std::sync::Mutex;
@ -31,7 +31,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_lists_metadata2")
.create(&mut txn)?;

View File

@ -2,7 +2,7 @@ use super::types::{PersonList1, PersonListMetadata3};
use crate::error::{Error, ErrorKind};
use crate::misc::Private;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use speedy::{Readable, Writable};
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_lists_metadata3")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::storage::types::PersonRelay1;
use crate::storage::{RawDatabase, Storage, MAX_LMDB_KEY};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::{PublicKey, RelayUrl};
use speedy::{Readable, Writable};
@ -34,7 +34,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("person_relays")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::storage::types::Relationship1;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::Id;
use speedy::Writable;
@ -40,7 +40,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed?
.name("relationships")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags};
use heed::{types::Bytes, DatabaseFlags};
use std::sync::Mutex;
// Kind:Pubkey:d-tag -> RelationshipByAddr1:Id
@ -29,7 +29,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT) // NOT FIXED, RelationshipByAddr1 serialized isn't.
.name("relationships_by_addr1")
.create(&mut txn)?;

View File

@ -2,7 +2,7 @@ use crate::error::Error;
use crate::storage::types::RelationshipByAddr2;
use crate::storage::{RawDatabase, Storage};
use heed::RwTxn;
use heed::{types::UnalignedSlice, DatabaseFlags};
use heed::{types::Bytes, DatabaseFlags};
use nostr_types::{EventAddr, Id};
use speedy::{Readable, Writable};
use std::sync::Mutex;
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT) // NOT FIXED, RelationshipByAddr2 serialized isn't.
.name("relationships_by_addr2")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use std::sync::Mutex;
// Id:Id -> RelationshipById1
@ -36,7 +36,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed?
.name("relationships_by_id1")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::Error;
use crate::storage::types::RelationshipById2;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::Id;
use speedy::{Readable, Writable};
@ -40,7 +40,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed?
.name("relationships_by_id2")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::{Error, ErrorKind};
use crate::storage::types::Relay1;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::RwTxn;
use nostr_types::RelayUrl;
use std::sync::Mutex;
@ -34,7 +34,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("relays")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::{Error, ErrorKind};
use crate::storage::types::Relay2;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use heed::{RoTxn, RwTxn};
use nostr_types::RelayUrl;
use std::sync::Mutex;
@ -33,7 +33,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("relays2")
.create(&mut txn)?;

View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::storage::{RawDatabase, Storage};
use heed::{types::UnalignedSlice, DatabaseFlags};
use heed::{types::Bytes, DatabaseFlags};
use std::sync::Mutex;
// Kind:Pubkey:d-tag -> Relationship1:Id
@ -29,7 +29,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
.flags(DatabaseFlags::DUP_SORT) // NOT FIXED, Relationship1 serialized isn't.
.name("reprel1")
.create(&mut txn)?;

View File

@ -1,7 +1,7 @@
use crate::error::{Error, ErrorKind};
use crate::globals::GLOBALS;
use crate::storage::{RawDatabase, Storage};
use heed::types::UnalignedSlice;
use heed::types::Bytes;
use nostr_types::Id;
use std::sync::Mutex;
@ -32,7 +32,7 @@ impl Storage {
let db = self
.env
.database_options()
.types::<UnalignedSlice<u8>, UnalignedSlice<u8>>()
.types::<Bytes, Bytes>()
// no .flags needed
.name("unindexed_giftwraps")
.create(&mut txn)?;