migration 36: remove and don't save records for known relays that proliferate under many paths

gossip-lib/src/storage/migrations/m36.rs

@@ -0,0 +1,56 @@
+use crate::error::Error;
+use crate::storage::types::{PersonRelay2, Relay2};
+use crate::storage::Storage;
+use heed::RwTxn;
+use speedy::Readable;
+
+impl Storage {
+    pub(super) fn m36_trigger(&self) -> Result<(), Error> {
+        let _ = self.db_relays2()?;
+        let _ = self.db_person_relays2()?;
+        Ok(())
+    }
+
+    pub(super) fn m36_migrate(&self, prefix: &str, txn: &mut RwTxn<'_>) -> Result<(), Error> {
+        // Info message
+        tracing::info!("{prefix}: Cleaning out relays...");
+
+        // Migrate
+        self.m36_clean_out_relays(txn)?;
+
+        Ok(())
+    }
+
+    fn m36_clean_out_relays(&self, txn: &mut RwTxn<'_>) -> Result<(), Error> {
+        // Delete any person_relay with this relay
+        let mut deletions: Vec<Vec<u8>> = Vec::new();
+        for result in self.db_person_relays2()?.iter(txn)? {
+            let (key, val) = result?;
+            if let Ok(person_relay) = PersonRelay2::read_from_buffer(val) {
+                if Self::url_is_banned(&person_relay.url) {
+                    deletions.push(key.to_owned());
+                }
+            }
+        }
+        for deletion in deletions.drain(..) {
+            self.db_person_relays2()?.delete(txn, &deletion)?;
+        }
+
+        // Delete any relay that matches
+        let mut deletions: Vec<Vec<u8>> = Vec::new();
+        {
+            for result in self.db_relays2()?.iter(txn)? {
+                let (key, val) = result?;
+                let relay: Relay2 = serde_json::from_slice(val)?;
+                if Self::url_is_banned(&relay.url) {
+                    deletions.push(key.to_owned());
+                }
+            }
+        }
+        for deletion in deletions.drain(..) {
+            self.db_relays2()?.delete(txn, &deletion)?;
+        }
+
+        Ok(())
+    }
+}

gossip-lib/src/storage/migrations/mod.rs

@@ -29,6 +29,7 @@ mod m32;
 mod m33;
 mod m34;
 mod m35;
+mod m36;
 mod m4;
 mod m5;
 mod m6;
@@ -41,7 +42,7 @@ use crate::error::{Error, ErrorKind};
 use heed::RwTxn;
 
 impl Storage {
-    const MAX_MIGRATION_LEVEL: u32 = 35;
+    const MAX_MIGRATION_LEVEL: u32 = 36;
 
     /// Initialize the database from empty
     pub(super) fn init_from_empty(&self) -> Result<(), Error> {
@@ -131,6 +132,7 @@ impl Storage {
             33 => self.m33_trigger()?,
             34 => self.m34_trigger()?,
             35 => self.m35_trigger()?,
+            36 => self.m36_trigger()?,
             _ => panic!("Unreachable migration level"),
         }
 
@@ -175,6 +177,7 @@ impl Storage {
             33 => self.m33_migrate(&prefix, txn)?,
             34 => self.m34_migrate(&prefix, txn)?,
             35 => self.m35_migrate(&prefix, txn)?,
+            36 => self.m36_migrate(&prefix, txn)?,
             _ => panic!("Unreachable migration level"),
         };
 

gossip-lib/src/storage/mod.rs

@@ -858,6 +858,11 @@ impl Storage {
         when: Unixtime,
         rw_txn: Option<&mut RwTxn<'a>>,
     ) -> Result<(), Error> {
+        // Don't save banned relay URLs
+        if Self::url_is_banned(url) {
+            return Ok(());
+        }
+
         self.add_event_seen_on_relay1(id, url, when, rw_txn)
     }
 
@@ -931,6 +936,11 @@ impl Storage {
         url: &RelayUrl,
         rw_txn: Option<&mut RwTxn<'a>>,
     ) -> Result<(), Error> {
+        // Don't save banned relay URLs
+        if Self::url_is_banned(url) {
+            return Ok(());
+        }
+
         let f = |txn: &mut RwTxn<'a>| -> Result<(), Error> {
             let rtxn = &**txn;
             if self.read_relay(url, Some(rtxn))?.is_none() {
@@ -986,6 +996,11 @@ impl Storage {
         url: &RelayUrl,
         rw_txn: Option<&mut RwTxn<'a>>,
     ) -> Result<Relay, Error> {
+        // Don't save banned relay URLs
+        if Self::url_is_banned(url) {
+            return Ok(Relay::new(url.to_owned()));
+        }
+
         let f = |txn: &mut RwTxn<'a>| -> Result<Relay, Error> {
             let rtxn = &**txn;
             match self.read_relay(url, Some(rtxn))? {
@@ -1084,7 +1099,10 @@ impl Storage {
         for tag in event.tags.iter() {
             if tag.tagname() == "relay" {
                 if let Ok(relay_url) = RelayUrl::try_from_str(tag.value()) {
-                    relays.push(relay_url);
+                    // Don't use banned relay URLs
+                    if !Self::url_is_banned(&relay_url) {
+                        relays.push(relay_url);
+                    }
                 }
             }
         }
@@ -1427,7 +1445,12 @@ impl Storage {
                     }
                 }
             }
-        } else if !filter.tags.is_empty() && filter.tags.iter().all(|t| INDEXED_TAGS.contains(&&*t.0.to_string())) {
+        } else if !filter.tags.is_empty()
+            && filter
+                .tags
+                .iter()
+                .all(|t| INDEXED_TAGS.contains(&&*t.0.to_string()))
+        {
             // event_tag_index
             for tag in &filter.tags {
                 let mut start_key: Vec<u8> = tag.0.to_string().as_bytes().to_owned();
@@ -1956,6 +1979,11 @@ impl Storage {
         person_relay: &PersonRelay,
         rw_txn: Option<&mut RwTxn<'a>>,
     ) -> Result<(), Error> {
+        // Don't save banned relay URLs
+        if Self::url_is_banned(&person_relay.url) {
+            return Ok(());
+        }
+
         self.write_person_relay2(person_relay, rw_txn)
     }
 
@@ -1981,6 +2009,11 @@ impl Storage {
         url: &RelayUrl,
         rw_txn: Option<&mut RwTxn<'a>>,
     ) -> Result<PersonRelay, Error> {
+        // Don't save banned relay URLs
+        if Self::url_is_banned(&url) {
+            return Ok(PersonRelay::new(pubkey.to_owned(), url.to_owned()));
+        }
+
         match self.read_person_relay(pubkey, url)? {
             Some(pr) => Ok(pr),
             None => {
@@ -2577,4 +2610,8 @@ impl Storage {
     ) -> Result<(), Error> {
         self.delete_nip46server2(pubkey, rw_txn)
     }
+
+    fn url_is_banned(url: &RelayUrl) -> bool {
+        url.as_str().contains("relay.nostr.band") || url.as_str().contains("filter.nostr.wine")
+    }
 }