Track how many bytes are read and display on stats page

Cargo.lock

@@ -1613,14 +1613,17 @@ dependencies = [
  "dirs",
  "eframe",
  "egui_extras",
+ "encoding_rs",
  "futures",
  "futures-util",
  "hex",
  "http",
+ "humansize",
  "image",
  "lazy_static",
  "linkify",
  "memoize",
+ "mime",
  "nostr-types",
  "parking_lot",
  "qrcode",
@@ -1756,6 +1759,15 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
+[[package]]
+name = "humansize"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
+dependencies = [
+ "libm",
+]
+
 [[package]]
 name = "hyper"
 version = "0.14.24"
@@ -2000,6 +2012,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "libm"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
+
 [[package]]
 name = "libsqlite3-sys"
 version = "0.25.2"

Cargo.toml

@@ -18,14 +18,17 @@ dashmap = "5.4"
 dirs = "4.0"
 eframe = { version = "0.21", features = [ "dark-light", "persistence" ] }
 egui_extras = { version = "0.21", features = [ "image", "svg", "tracing" ] }
+encoding_rs = "0.8"
 futures = "0.3"
 futures-util = "0.3"
 hex = "0.4"
 http = "0.2"
+humansize = "2.1"
 image = { version = "0.24", features = [ "png", "jpeg" ] }
 lazy_static = "1.4"
 linkify = "0.9"
 memoize = "0.4"
+mime = "0.3"
 nostr-types = { git = "https://github.com/mikedilger/nostr-types" }
 parking_lot = "0.12"
 qrcode = { git = "https://github.com/mikedilger/qrcode-rust" }

src/fetcher.rs

@@ -186,6 +186,8 @@ impl Fetcher {
 
         let bytes = maybe_bytes?;
 
+        GLOBALS.bytes_read.fetch_add(bytes.len(), Ordering::Relaxed);
+
         let cache_file = GLOBALS.fetcher.cache_file(&url);
 
         // Write to the file

src/globals.rs

@@ -11,7 +11,7 @@ use crate::signer::Signer;
 use nostr_types::{Event, Id, Profile, PublicKeyHex, RelayUrl};
 use rusqlite::Connection;
 use std::collections::{HashMap, HashSet};
-use std::sync::atomic::{AtomicBool, AtomicU32};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize};
 use tokio::sync::{broadcast, mpsc, Mutex, RwLock};
 
 /// Only one of these is ever created, via lazy_static!, and represents
@@ -80,6 +80,8 @@ pub struct Globals {
     pub status_message: RwLock<String>,
 
     pub pull_following_merge: AtomicBool,
+
+    pub bytes_read: AtomicUsize,
 }
 
 lazy_static! {
@@ -112,6 +114,7 @@ lazy_static! {
             pixels_per_point_times_100: AtomicU32::new(139), // 100 dpi, 1/72th inch => 1.38888
             status_message: RwLock::new("Welcome to Gossip. Status messages will appear here. Click them to dismiss them.".to_owned()),
             pull_following_merge: AtomicBool::new(true),
+            bytes_read: AtomicUsize::new(0),
         }
     };
 }

src/nip05.rs

@@ -4,6 +4,7 @@ use crate::globals::GLOBALS;
 use crate::people::DbPerson;
 use dashmap::mapref::entry::Entry;
 use nostr_types::{Metadata, Nip05, PublicKeyHex, RelayUrl, Unixtime};
+use std::sync::atomic::Ordering;
 
 // This updates the people map and the database with the result
 pub async fn validate_nip05(person: DbPerson) -> Result<(), Error> {
@@ -183,5 +184,7 @@ async fn fetch_nip05(user: &str, domain: &str) -> Result<Nip05, Error> {
         .header("Host", domain)
         .send();
     let response = nip05_future.await?;
-    Ok(response.json::<Nip05>().await?)
+    let bytes = response.bytes().await?;
+    GLOBALS.bytes_read.fetch_add(bytes.len(), Ordering::Relaxed);
+    Ok(serde_json::from_slice(&bytes)?)
 }

src/overlord/minion/mod.rs

@@ -6,13 +6,18 @@ use crate::db::DbRelay;
 use crate::error::Error;
 use crate::globals::GLOBALS;
 use base64::Engine;
+use encoding_rs::{Encoding, UTF_8};
 use futures::{SinkExt, StreamExt};
 use futures_util::stream::{SplitSink, SplitStream};
 use http::Uri;
+use mime::Mime;
 use nostr_types::{
     ClientMessage, EventKind, Filter, IdHex, IdHexPrefix, PublicKeyHex, PublicKeyHexPrefix,
     RelayInformationDocument, RelayUrl, Unixtime,
 };
+use reqwest::Response;
+use std::borrow::Cow;
+use std::sync::atomic::Ordering;
 use std::time::Duration;
 use subscription::Subscriptions;
 use tokio::net::TcpStream;
@@ -98,6 +103,7 @@ impl Minion {
                 return Err(Error::UrlHasEmptyHostname);
             }
 
+            // Read NIP-11 information
             let request_nip11_future = reqwest::Client::builder()
                 .timeout(std::time::Duration::new(30, 0))
                 .redirect(reqwest::redirect::Policy::none())
@@ -109,9 +115,8 @@ impl Minion {
                 .header("Host", host)
                 .header("Accept", "application/nostr+json")
                 .send();
-
-            // Read NIP-11 information
-            match request_nip11_future.await?.text().await {
+            let response = request_nip11_future.await?;
+            match Self::text_with_charset(response, "utf-8").await {
                 Ok(text) => match serde_json::from_str::<RelayInformationDocument>(&text) {
                     Ok(nip11) => {
                         tracing::info!("{}: {}", &self.url, nip11);
@@ -239,6 +244,8 @@ impl Minion {
                     }
                 }?;
 
+                GLOBALS.bytes_read.fetch_add(ws_message.len(), Ordering::Relaxed);
+
                 tracing::trace!("{}: Handling message", &self.url);
                 match ws_message {
                     WsMessage::Text(t) => {
@@ -918,4 +925,33 @@ impl Minion {
         }
         Ok(())
     }
+
+    // This replictes reqwest Response text_with_charset to handle decoding
+    // whatever charset they used into UTF-8, as well as counting the bytes.
+    async fn text_with_charset(
+        response: Response,
+        default_encoding: &str,
+    ) -> Result<String, Error> {
+        let content_type = response
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<Mime>().ok());
+        let encoding_name = content_type
+            .as_ref()
+            .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
+            .unwrap_or(default_encoding);
+        let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8);
+        let full = response.bytes().await?;
+        GLOBALS.bytes_read.fetch_add(full.len(), Ordering::Relaxed);
+        let (text, _, _) = encoding.decode(&full);
+        if let Cow::Owned(s) = text {
+            return Ok(s);
+        }
+        unsafe {
+            // decoding returned Cow::Borrowed, meaning these bytes
+            // are already valid utf8
+            Ok(String::from_utf8_unchecked(full.to_vec()))
+        }
+    }
 }

src/ui/help/stats.rs

@@ -1,7 +1,15 @@
 use super::GossipUi;
+use crate::globals::GLOBALS;
 use eframe::egui;
 use egui::{Context, Ui};
+use humansize::{format_size, DECIMAL};
+use std::sync::atomic::Ordering;
 
 pub(super) fn update(_app: &mut GossipUi, _ctx: &Context, _frame: &mut eframe::Frame, ui: &mut Ui) {
     ui.label("STATS PAGE - Coming Soon".to_string());
+
+    ui.label(format!(
+        "BYTES READ: {}",
+        format_size(GLOBALS.bytes_read.load(Ordering::Relaxed), DECIMAL)
+    ));
 }