Settings: migrate to a new structure

src/settings.rs

@@ -5,73 +5,171 @@ use speedy::{Readable, Writable};
 
 #[derive(Clone, Debug, Serialize, Deserialize, Readable, Writable)]
 pub struct Settings {
-    pub feed_chunk: u64,
-    pub replies_chunk: u64,
-    pub overlap: u64,
-    pub num_relays_per_person: u8,
-    pub max_relays: u8,
+    // ID settings
     pub public_key: Option<PublicKey>,
-    pub max_fps: u32,
-    pub recompute_feed_periodically: bool,
-    pub feed_recompute_interval_ms: u32,
-    pub pow: u8,
+    pub log_n: u8,
+
+    // Network settings
     pub offline: bool,
-    pub theme: Theme,
-    pub set_client_tag: bool,
-    pub set_user_agent: bool,
-    pub override_dpi: Option<u32>,
-    pub reactions: bool,
-    pub reposts: bool,
-    pub show_long_form: bool,
-    pub show_mentions: bool,
-    pub show_media: bool,
     pub load_avatars: bool,
     pub load_media: bool,
     pub check_nip05: bool,
-    pub direct_messages: bool,
     pub automatically_fetch_metadata: bool,
+
+    // Relay settings
+    pub num_relays_per_person: u8,
+    pub max_relays: u8,
+
+    // Feed Settings
+    pub feed_chunk: u64,
+    pub replies_chunk: u64,
+    pub person_feed_chunk: u64,
+    pub overlap: u64,
+
+    // Event Selection
+    pub reposts: bool,
+    pub show_long_form: bool,
+    pub show_mentions: bool,
+    pub direct_messages: bool,
+    pub future_allowance_secs: u64,
+
+    // Event Content Settings
+    pub reactions: bool,
+    pub enable_zap_receipts: bool,
+    pub show_media: bool,
+
+    // Posting Settings
+    pub pow: u8,
+    pub set_client_tag: bool,
+    pub set_user_agent: bool,
     pub delegatee_tag: String,
+
+    // UI settings
+    pub max_fps: u32,
+    pub recompute_feed_periodically: bool,
+    pub feed_recompute_interval_ms: u32,
+    pub theme: Theme,
+    pub override_dpi: Option<u32>,
     pub highlight_unread_events: bool,
     pub posting_area_at_top: bool,
-    pub enable_zap_receipts: bool,
+    pub status_bar: bool,
+    pub image_resize_algorithm: String,
+
+    // Staletime settings
+    pub relay_list_becomes_stale_hours: u64,
+    pub metadata_becomes_stale_hours: u64,
+    pub nip05_becomes_stale_if_valid_hours: u64,
+    pub nip05_becomes_stale_if_invalid_minutes: u64,
+    pub avatar_becomes_stale_hours: u64,
+    pub media_becomes_stale_hours: u64,
+
+    // Websocket settings
+    pub max_websocket_message_size_kb: usize,
+    pub max_websocket_frame_size_kb: usize,
+    pub websocket_accept_unmasked_frames: bool,
+    pub websocket_connect_timeout_sec: u64,
+    pub websocket_ping_frequency_sec: u64,
+
+    // HTTP settings
+    pub fetcher_metadata_looptime_ms: u64,
+    pub fetcher_looptime_ms: u64,
+    pub fetcher_connect_timeout_sec: u64,
+    pub fetcher_timeout_sec: u64,
+    pub fetcher_max_requests_per_host: usize,
+    pub fetcher_host_exclusion_on_low_error_secs: u64,
+    pub fetcher_host_exclusion_on_med_error_secs: u64,
+    pub fetcher_host_exclusion_on_high_error_secs: u64,
+    pub nip11_lines_to_output_on_error: usize,
+
+    // Database settings
+    pub prune_period_days: u64,
 }
 
 impl Default for Settings {
     fn default() -> Settings {
         Settings {
-            feed_chunk: 60 * 60 * 12,        // 12 hours
-            replies_chunk: 60 * 60 * 24 * 7, // 1 week
-            overlap: 300,                    // 5 minutes
+            // ID settings
+            public_key: None,
+            log_n: 18,
+
+            // Network settings
+            offline: false,
+            load_avatars: true,
+            load_media: true,
+            check_nip05: true,
+            automatically_fetch_metadata: true,
+
+            // Relay settings
             num_relays_per_person: 2,
             max_relays: 50,
-            public_key: None,
+
+            // Feed settings
+            feed_chunk: 60 * 60 * 12,             // 12 hours
+            replies_chunk: 60 * 60 * 24 * 7,      // 1 week
+            person_feed_chunk: 60 * 60 * 24 * 30, // 1 month
+            overlap: 300,                         // 5 minutes
+
+            // Event Selection
+            reposts: true,
+            show_long_form: false,
+            show_mentions: true,
+            direct_messages: true,
+            future_allowance_secs: 60 * 15, // 15 minutes
+
+            // Event Content Settings
+            reactions: true,
+            enable_zap_receipts: true,
+            show_media: true,
+
+            // Posting settings
+            pow: 0,
+            set_client_tag: false,
+            set_user_agent: false,
+            delegatee_tag: String::new(),
+
+            // UI settings
             max_fps: 12,
             recompute_feed_periodically: true,
             feed_recompute_interval_ms: 8000,
-            pow: 0,
-            offline: false,
             theme: Theme {
                 variant: ThemeVariant::Default,
                 dark_mode: false,
                 follow_os_dark_mode: false,
             },
-            set_client_tag: false,
-            set_user_agent: false,
             override_dpi: None,
-            reactions: true,
-            reposts: true,
-            show_long_form: false,
-            show_mentions: true,
-            show_media: true,
-            load_avatars: true,
-            load_media: true,
-            check_nip05: true,
-            direct_messages: true,
-            automatically_fetch_metadata: true,
-            delegatee_tag: String::new(),
             highlight_unread_events: true,
             posting_area_at_top: true,
-            enable_zap_receipts: true,
+            status_bar: false,
+            image_resize_algorithm: "CatmullRom".to_owned(),
+
+            // Staletime settings
+            relay_list_becomes_stale_hours: 8,
+            metadata_becomes_stale_hours: 8,
+            nip05_becomes_stale_if_valid_hours: 8,
+            nip05_becomes_stale_if_invalid_minutes: 30, // 30 minutes
+            avatar_becomes_stale_hours: 8,
+            media_becomes_stale_hours: 8,
+
+            // Websocket settings
+            max_websocket_message_size_kb: 1024, // 1 MB
+            max_websocket_frame_size_kb: 1024,   // 1 MB
+            websocket_accept_unmasked_frames: false,
+            websocket_connect_timeout_sec: 15,
+            websocket_ping_frequency_sec: 55,
+
+            // HTTP settings
+            fetcher_metadata_looptime_ms: 3000,
+            fetcher_looptime_ms: 1800,
+            fetcher_connect_timeout_sec: 15,
+            fetcher_timeout_sec: 30,
+            fetcher_max_requests_per_host: 3,
+            fetcher_host_exclusion_on_low_error_secs: 30,
+            fetcher_host_exclusion_on_med_error_secs: 60,
+            fetcher_host_exclusion_on_high_error_secs: 600,
+            nip11_lines_to_output_on_error: 10,
+
+            // Database settings
+            prune_period_days: 30,
         }
     }
 }

src/storage/migrations/mod.rs

@@ -4,11 +4,13 @@ use lmdb::{Cursor, RwTransaction, Transaction};
 use nostr_types::Event;
 use speedy::Readable;
 
+mod settings;
+
 impl Storage {
-    const MIGRATION_LEVEL: u32 = 1;
+    const MAX_MIGRATION_LEVEL: u32 = 2;
 
     pub(super) fn migrate(&self, mut level: u32) -> Result<(), Error> {
-        if level > Self::MIGRATION_LEVEL {
+        if level > Self::MAX_MIGRATION_LEVEL {
             return Err(ErrorKind::General(format!(
                 "Migration level {} unknown: This client is older than your data.",
                 level
@@ -17,47 +19,48 @@ impl Storage {
         }
 
         let mut txn = self.env.begin_rw_txn()?;
-
-        while level < Self::MIGRATION_LEVEL {
+        while level < Self::MAX_MIGRATION_LEVEL {
+            self.migrate_inner(level, &mut txn)?;
             level += 1;
-            tracing::info!("LMDB Migration to level {}...", level);
-            self.migrate_inner(level, Some(&mut txn))?;
             self.write_migration_level(level, Some(&mut txn))?;
         }
-
         txn.commit()?;
 
         Ok(())
     }
 
-    fn migrate_inner<'a>(
-        &'a self,
-        level: u32,
-        rw_txn: Option<&mut RwTransaction<'a>>,
-    ) -> Result<(), Error> {
+    fn migrate_inner<'a>(&'a self, level: u32, txn: &mut RwTransaction<'a>) -> Result<(), Error> {
+        let prefix = format!("LMDB Migration {} -> {}", level, level + 1);
         match level {
-            0 => Ok(()),
-            1 => self.compute_relationships(rw_txn),
-            n => panic!("Unknown migration level {}", n),
-        }
+            0 => {
+                let total = self.get_event_stats()?.entries();
+                tracing::info!(
+                    "{prefix}: Computing and storing event relationships for {total} events..."
+                );
+                self.compute_relationships(total, Some(txn))?;
+            }
+            1 => {
+                tracing::info!("{prefix}: Updating Settings...");
+                self.try_migrate_settings1_settings2(Some(txn))?;
+            }
+            _ => panic!("Unreachable migration level"),
+        };
+
+        tracing::info!("done.");
+
+        Ok(())
     }
 
     // Load and process every event in order to generate the relationships data
     fn compute_relationships<'a>(
         &'a self,
+        total: usize,
         rw_txn: Option<&mut RwTransaction<'a>>,
     ) -> Result<(), Error> {
         self.disable_sync()?;
 
         let f = |txn: &mut RwTransaction<'a>| -> Result<(), Error> {
             // track progress
-            let total = self.get_event_stats()?.entries();
-
-            tracing::info!(
-                "LMDB Migration 1: Computing and storing event relationships for {} events...",
-                total
-            );
-
             let mut count = 0;
 
             let event_txn = self.env.begin_ro_txn()?;

src/storage/migrations/settings/mod.rs

@@ -0,0 +1,52 @@
+// In order for this migration to work in the distant future after all kinds of
+// other code has changed, it has to have it's own version of what Settings used
+// to look like.
+
+mod settings1;
+use settings1::Settings1;
+
+mod settings2;
+use settings2::Settings2;
+
+mod theme1;
+
+use crate::error::Error;
+use crate::storage::Storage;
+use lmdb::{RwTransaction, Transaction, WriteFlags};
+use speedy::{Readable, Writable};
+
+impl Storage {
+    pub(in crate::storage) fn try_migrate_settings1_settings2<'a>(
+        &'a self,
+        rw_txn: Option<&mut RwTransaction<'a>>,
+    ) -> Result<(), Error> {
+        let f = |txn: &mut RwTransaction<'a>| -> Result<(), Error> {
+            // If something is under the old "settings" key
+            if let Ok(bytes) = txn.get(self.general, b"settings") {
+                let settings1 = Settings1::read_from_buffer(bytes)?;
+
+                // Convert it to the new Settings2 structure
+                let settings2: Settings2 = settings1.into();
+                let bytes = settings2.write_to_vec()?;
+
+                // And store it under the new "settings2" key
+                txn.put(self.general, b"settings2", &bytes, WriteFlags::empty())?;
+
+                // Then delete the old "settings" key
+                txn.del(self.general, b"settings", None)?;
+            }
+            Ok(())
+        };
+
+        match rw_txn {
+            Some(txn) => f(txn)?,
+            None => {
+                let mut txn = self.env.begin_rw_txn()?;
+                f(&mut txn)?;
+                txn.commit()?;
+            }
+        };
+
+        Ok(())
+    }
+}

src/storage/migrations/settings/settings1.rs

@@ -0,0 +1,77 @@
+use super::theme1::{Theme1, ThemeVariant1};
+use nostr_types::PublicKey;
+use serde::{Deserialize, Serialize};
+use speedy::{Readable, Writable};
+
+#[derive(Clone, Debug, Serialize, Deserialize, Readable, Writable)]
+pub struct Settings1 {
+    pub feed_chunk: u64,
+    pub replies_chunk: u64,
+    pub overlap: u64,
+    pub num_relays_per_person: u8,
+    pub max_relays: u8,
+    pub public_key: Option<PublicKey>,
+    pub max_fps: u32,
+    pub recompute_feed_periodically: bool,
+    pub feed_recompute_interval_ms: u32,
+    pub pow: u8,
+    pub offline: bool,
+    pub theme: Theme1,
+    pub set_client_tag: bool,
+    pub set_user_agent: bool,
+    pub override_dpi: Option<u32>,
+    pub reactions: bool,
+    pub reposts: bool,
+    pub show_long_form: bool,
+    pub show_mentions: bool,
+    pub show_media: bool,
+    pub load_avatars: bool,
+    pub load_media: bool,
+    pub check_nip05: bool,
+    pub direct_messages: bool,
+    pub automatically_fetch_metadata: bool,
+    pub delegatee_tag: String,
+    pub highlight_unread_events: bool,
+    pub posting_area_at_top: bool,
+    pub enable_zap_receipts: bool,
+}
+
+impl Default for Settings1 {
+    fn default() -> Settings1 {
+        Settings1 {
+            feed_chunk: 60 * 60 * 12,        // 12 hours
+            replies_chunk: 60 * 60 * 24 * 7, // 1 week
+            overlap: 300,                    // 5 minutes
+            num_relays_per_person: 2,
+            max_relays: 50,
+            public_key: None,
+            max_fps: 12,
+            recompute_feed_periodically: true,
+            feed_recompute_interval_ms: 8000,
+            pow: 0,
+            offline: false,
+            theme: Theme1 {
+                variant: ThemeVariant1::Default,
+                dark_mode: false,
+                follow_os_dark_mode: false,
+            },
+            set_client_tag: false,
+            set_user_agent: false,
+            override_dpi: None,
+            reactions: true,
+            reposts: true,
+            show_long_form: false,
+            show_mentions: true,
+            show_media: true,
+            load_avatars: true,
+            load_media: true,
+            check_nip05: true,
+            direct_messages: true,
+            automatically_fetch_metadata: true,
+            delegatee_tag: String::new(),
+            highlight_unread_events: true,
+            posting_area_at_top: true,
+            enable_zap_receipts: true,
+        }
+    }
+}

src/storage/migrations/settings/settings2.rs

@@ -0,0 +1,229 @@
+use super::settings1::Settings1;
+use super::theme1::{Theme1, ThemeVariant1};
+use nostr_types::PublicKey;
+use serde::{Deserialize, Serialize};
+use speedy::{Readable, Writable};
+
+#[derive(Clone, Debug, Serialize, Deserialize, Readable, Writable)]
+pub struct Settings2 {
+    // ID settings
+    pub public_key: Option<PublicKey>,
+    pub log_n: u8,
+
+    // Network settings
+    pub offline: bool,
+    pub load_avatars: bool,
+    pub load_media: bool,
+    pub check_nip05: bool,
+    pub automatically_fetch_metadata: bool,
+
+    // Relay settings
+    pub num_relays_per_person: u8,
+    pub max_relays: u8,
+
+    // Feed Settings
+    pub feed_chunk: u64,
+    pub replies_chunk: u64,
+    pub person_feed_chunk: u64,
+    pub overlap: u64,
+
+    // Event Selection
+    pub reposts: bool,
+    pub show_long_form: bool,
+    pub show_mentions: bool,
+    pub direct_messages: bool,
+    pub future_allowance_secs: u64,
+
+    // Event Content Settings
+    pub reactions: bool,
+    pub enable_zap_receipts: bool,
+    pub show_media: bool,
+
+    // Posting Settings
+    pub pow: u8,
+    pub set_client_tag: bool,
+    pub set_user_agent: bool,
+    pub delegatee_tag: String,
+
+    // UI settings
+    pub max_fps: u32,
+    pub recompute_feed_periodically: bool,
+    pub feed_recompute_interval_ms: u32,
+    pub theme: Theme1,
+    pub override_dpi: Option<u32>,
+    pub highlight_unread_events: bool,
+    pub posting_area_at_top: bool,
+    pub status_bar: bool,
+    pub image_resize_algorithm: String,
+
+    // Staletime settings
+    pub relay_list_becomes_stale_hours: u64,
+    pub metadata_becomes_stale_hours: u64,
+    pub nip05_becomes_stale_if_valid_hours: u64,
+    pub nip05_becomes_stale_if_invalid_minutes: u64,
+    pub avatar_becomes_stale_hours: u64,
+    pub media_becomes_stale_hours: u64,
+
+    // Websocket settings
+    pub max_websocket_message_size_kb: usize,
+    pub max_websocket_frame_size_kb: usize,
+    pub websocket_accept_unmasked_frames: bool,
+    pub websocket_connect_timeout_sec: u64,
+    pub websocket_ping_frequency_sec: u64,
+
+    // HTTP settings
+    pub fetcher_metadata_looptime_ms: u64,
+    pub fetcher_looptime_ms: u64,
+    pub fetcher_connect_timeout_sec: u64,
+    pub fetcher_timeout_sec: u64,
+    pub fetcher_max_requests_per_host: usize,
+    pub fetcher_host_exclusion_on_low_error_secs: u64,
+    pub fetcher_host_exclusion_on_med_error_secs: u64,
+    pub fetcher_host_exclusion_on_high_error_secs: u64,
+    pub nip11_lines_to_output_on_error: usize,
+
+    // Database settings
+    pub prune_period_days: u64,
+}
+
+impl Default for Settings2 {
+    fn default() -> Settings2 {
+        Settings2 {
+            // ID settings
+            public_key: None,
+            log_n: 18,
+
+            // Network settings
+            offline: false,
+            load_avatars: true,
+            load_media: true,
+            check_nip05: true,
+            automatically_fetch_metadata: true,
+
+            // Relay settings
+            num_relays_per_person: 2,
+            max_relays: 50,
+
+            // Feed settings
+            feed_chunk: 60 * 60 * 12,             // 12 hours
+            replies_chunk: 60 * 60 * 24 * 7,      // 1 week
+            person_feed_chunk: 60 * 60 * 24 * 30, // 1 month
+            overlap: 300,                         // 5 minutes
+
+            // Event Selection
+            reposts: true,
+            show_long_form: false,
+            show_mentions: true,
+            direct_messages: true,
+            future_allowance_secs: 60 * 15, // 15 minutes
+
+            // Event Content Settings
+            reactions: true,
+            enable_zap_receipts: true,
+            show_media: true,
+
+            // Posting settings
+            pow: 0,
+            set_client_tag: false,
+            set_user_agent: false,
+            delegatee_tag: String::new(),
+
+            // UI settings
+            max_fps: 12,
+            recompute_feed_periodically: true,
+            feed_recompute_interval_ms: 8000,
+            theme: Theme1 {
+                variant: ThemeVariant1::Default,
+                dark_mode: false,
+                follow_os_dark_mode: false,
+            },
+            override_dpi: None,
+            highlight_unread_events: true,
+            posting_area_at_top: true,
+            status_bar: false,
+            image_resize_algorithm: "CatmullRom".to_owned(),
+
+            // Staletime settings
+            relay_list_becomes_stale_hours: 8,
+            metadata_becomes_stale_hours: 8,
+            nip05_becomes_stale_if_valid_hours: 8,
+            nip05_becomes_stale_if_invalid_minutes: 30, // 30 minutes
+            avatar_becomes_stale_hours: 8,
+            media_becomes_stale_hours: 8,
+
+            // Websocket settings
+            max_websocket_message_size_kb: 1024, // 1 MB
+            max_websocket_frame_size_kb: 1024,   // 1 MB
+            websocket_accept_unmasked_frames: false,
+            websocket_connect_timeout_sec: 15,
+            websocket_ping_frequency_sec: 55,
+
+            // HTTP settings
+            fetcher_metadata_looptime_ms: 3000,
+            fetcher_looptime_ms: 1800,
+            fetcher_connect_timeout_sec: 15,
+            fetcher_timeout_sec: 30,
+            fetcher_max_requests_per_host: 3,
+            fetcher_host_exclusion_on_low_error_secs: 30,
+            fetcher_host_exclusion_on_med_error_secs: 60,
+            fetcher_host_exclusion_on_high_error_secs: 600,
+            nip11_lines_to_output_on_error: 10,
+
+            // Database settings
+            prune_period_days: 30,
+        }
+    }
+}
+
+impl From<Settings1> for Settings2 {
+    fn from(old: Settings1) -> Settings2 {
+        Settings2 {
+            // ID settings
+            public_key: old.public_key,
+
+            // Network settings
+            offline: old.offline,
+            load_avatars: old.load_avatars,
+            load_media: old.load_media,
+            check_nip05: old.check_nip05,
+            automatically_fetch_metadata: old.automatically_fetch_metadata,
+
+            // Relay settings
+            num_relays_per_person: old.num_relays_per_person,
+            max_relays: old.max_relays,
+
+            // Feed settings
+            feed_chunk: old.feed_chunk,
+            replies_chunk: old.replies_chunk,
+            overlap: old.overlap,
+
+            // Event Selection
+            reposts: old.reposts,
+            show_long_form: old.show_long_form,
+            show_mentions: old.show_mentions,
+            direct_messages: old.direct_messages,
+
+            // Event Content Settings
+            reactions: old.reactions,
+            enable_zap_receipts: old.enable_zap_receipts,
+            show_media: old.show_media,
+
+            // Posting settings
+            pow: old.pow,
+            set_client_tag: old.set_client_tag,
+            set_user_agent: old.set_user_agent,
+            delegatee_tag: old.delegatee_tag,
+
+            // UI settings
+            max_fps: old.max_fps,
+            recompute_feed_periodically: old.recompute_feed_periodically,
+            feed_recompute_interval_ms: old.feed_recompute_interval_ms,
+            theme: old.theme,
+            override_dpi: old.override_dpi,
+            highlight_unread_events: old.highlight_unread_events,
+            posting_area_at_top: old.posting_area_at_top,
+
+            ..Default::default()
+        }
+    }
+}

src/storage/migrations/settings/theme1.rs

@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+use speedy::{Readable, Writable};
+
+// note: if we store anything inside the variants, we can't use macro_rules.
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, Readable, Writable)]
+pub enum ThemeVariant1 {
+    Classic,
+    Default,
+    Roundy,
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, Readable, Writable)]
+pub struct Theme1 {
+    pub variant: ThemeVariant1,
+    pub dark_mode: bool,
+    pub follow_os_dark_mode: bool,
+}

src/storage/mod.rs

@@ -497,7 +497,7 @@ impl Storage {
         let bytes = settings.write_to_vec()?;
 
         let f = |txn: &mut RwTransaction<'a>| -> Result<(), Error> {
-            txn.put(self.general, b"settings", &bytes, WriteFlags::empty())?;
+            txn.put(self.general, b"settings2", &bytes, WriteFlags::empty())?;
             Ok(())
         };
 
@@ -515,7 +515,7 @@ impl Storage {
 
     pub fn read_settings(&self) -> Result<Option<Settings>, Error> {
         let txn = self.env.begin_ro_txn()?;
-        match txn.get(self.general, b"settings") {
+        match txn.get(self.general, b"settings2") {
             Ok(bytes) => Ok(Some(Settings::read_from_buffer(bytes)?)),
             Err(lmdb::Error::NotFound) => Ok(None),
             Err(e) => Err(e.into()),