Handle gzip/deflate/brotli on all reqwest clients

Cargo.lock

@@ -139,6 +139,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -195,6 +210,20 @@ dependencies = [
  "parking_lot",
 ]
 
+[[package]]
+name = "async-compression"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a"
+dependencies = [
+ "brotli",
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-executor"
 version = "1.5.0"
@@ -370,6 +399,27 @@ dependencies = [
  "objc2-encode",
 ]
 
+[[package]]
+name = "brotli"
+version = "3.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.12.0"
@@ -3027,6 +3077,7 @@ version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
 dependencies = [
+ "async-compression",
  "base64 0.21.0",
  "bytes",
  "encoding_rs",
@@ -3050,6 +3101,7 @@ dependencies = [
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",

Cargo.toml

@@ -30,7 +30,7 @@ nostr-types = { git = "https://github.com/mikedilger/nostr-types" }
 parking_lot = "0.12"
 rand = "0.8"
 regex = "1.7"
-reqwest = { version = "0.11", features = ["json"] }
+reqwest = { version = "0.11", features = ["brotli", "deflate", "gzip", "json"] }
 rusqlite = { version = "0.28", features = ["bundled", "chrono", "serde_json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

src/fetcher.rs

@@ -11,6 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::RwLock;
 use tokio::task;
 
+#[derive(Debug, Default)]
 pub struct Fetcher {
     // we don't want new() to fail in lazy_static init, so we just mark it dead if there was an error
     // on creation
@@ -28,13 +29,27 @@ pub struct Fetcher {
 
 impl Fetcher {
     pub fn new() -> Fetcher {
-        let mut f = Fetcher {
-            dead: None,
-            cache_dir: PathBuf::new(),
-            client: Client::new(),
-            pending: RwLock::new(HashSet::new()),
-            failed: RwLock::new(HashMap::new()),
-            requests_in_flight: AtomicUsize::new(0),
+        let client = match Client::builder()
+            // .user_agent...
+            // .timeout(std::time::Duration::new(60, 0)) ?
+            // .redirect(reqwest::redirect::Policy::none()) ?
+            .gzip(true)
+            .brotli(true)
+            .deflate(true)
+            .build()
+        {
+            Ok(c) => c,
+            Err(e) => {
+                return Fetcher {
+                    dead: Some(format!("{}", e)),
+                    ..Default::default()
+                }
+            }
+        };
+
+        let mut f: Fetcher = Fetcher {
+            client,
+            ..Default::default()
         };
 
         // Setup the cache directory

src/nip05.rs

@@ -172,6 +172,9 @@ async fn fetch_nip05(user: &str, domain: &str) -> Result<Nip05, Error> {
     let nip05_future = reqwest::Client::builder()
         .timeout(std::time::Duration::new(60, 0))
         .redirect(reqwest::redirect::Policy::none()) // see NIP-05
+        .gzip(true)
+        .brotli(true)
+        .deflate(true)
         .build()?
         .get(format!(
             "https://{}/.well-known/nostr.json?name={}",

src/overlord/minion/mod.rs

@@ -95,24 +95,30 @@ impl Minion {
                 return Err(Error::UrlHasEmptyHostname);
             }
 
-            let request_nip11_future = reqwest::Client::new()
+            let request_nip11_future = reqwest::Client::builder()
+                .timeout(std::time::Duration::new(30, 0))
+                .redirect(reqwest::redirect::Policy::none())
+                .gzip(true)
+                .brotli(true)
+                .deflate(true)
+                .build()?
                 .get(format!("https://{}", host))
                 .header("Host", host)
                 .header("Accept", "application/nostr+json")
                 .send();
 
             // Read NIP-11 information
-            if let Ok(response) =
-                tokio::time::timeout(std::time::Duration::new(15, 0), request_nip11_future).await?
+            match request_nip11_future
+                .await?
+                .json::<RelayInformationDocument>()
+                .await
             {
-                match response.json::<RelayInformationDocument>().await {
-                    Ok(nip11) => {
-                        tracing::info!("{}: {}", &self.url, nip11);
-                        self.nip11 = Some(nip11);
-                    }
-                    Err(e) => {
-                        tracing::warn!("{}: Unable to parse response as NIP-11: {}", &self.url, e);
-                    }
+                Ok(nip11) => {
+                    tracing::info!("{}: {}", &self.url, nip11);
+                    self.nip11 = Some(nip11);
+                }
+                Err(e) => {
+                    tracing::warn!("{}: Unable to parse response as NIP-11: {}", &self.url, e);
                 }
             }