From 3ce38ed9ba20cc06de9dae2ffb7a124592fd9a23 Mon Sep 17 00:00:00 2001 From: Mike Dilger Date: Wed, 15 Feb 2023 10:34:11 +1300 Subject: [PATCH] Track how many bytes are read and display on stats page --- Cargo.lock | 18 ++++++++++++++++ Cargo.toml | 3 +++ src/fetcher.rs | 2 ++ src/globals.rs | 5 ++++- src/nip05.rs | 5 ++++- src/overlord/minion/mod.rs | 42 +++++++++++++++++++++++++++++++++++--- src/ui/help/stats.rs | 8 ++++++++ 7 files changed, 78 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d3272be..ab6f32c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1613,14 +1613,17 @@ dependencies = [ "dirs", "eframe", "egui_extras", + "encoding_rs", "futures", "futures-util", "hex", "http", + "humansize", "image", "lazy_static", "linkify", "memoize", + "mime", "nostr-types", "parking_lot", "qrcode", @@ -1756,6 +1759,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "hyper" version = "0.14.24" @@ -2000,6 +2012,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + [[package]] name = "libsqlite3-sys" version = "0.25.2" diff --git a/Cargo.toml b/Cargo.toml index 9e3f0c7d..5c6cf48b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,17 @@ dashmap = "5.4" dirs = "4.0" eframe = { version = "0.21", features = [ "dark-light", "persistence" ] } egui_extras = { version = "0.21", features = [ "image", "svg", "tracing" ] } +encoding_rs = "0.8" futures = "0.3" futures-util = "0.3" hex = "0.4" http = "0.2" +humansize = "2.1" image = { version = "0.24", features = [ "png", "jpeg" ] } lazy_static = "1.4" linkify = "0.9" memoize = "0.4" +mime = "0.3" nostr-types = { git = "https://github.com/mikedilger/nostr-types" } parking_lot = "0.12" qrcode = { git = "https://github.com/mikedilger/qrcode-rust" } diff --git a/src/fetcher.rs b/src/fetcher.rs index b303fa8e..693b283c 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -186,6 +186,8 @@ impl Fetcher { let bytes = maybe_bytes?; + GLOBALS.bytes_read.fetch_add(bytes.len(), Ordering::Relaxed); + let cache_file = GLOBALS.fetcher.cache_file(&url); // Write to the file diff --git a/src/globals.rs b/src/globals.rs index 2763d188..33197244 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -11,7 +11,7 @@ use crate::signer::Signer; use nostr_types::{Event, Id, Profile, PublicKeyHex, RelayUrl}; use rusqlite::Connection; use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize}; use tokio::sync::{broadcast, mpsc, Mutex, RwLock}; /// Only one of these is ever created, via lazy_static!, and represents @@ -80,6 +80,8 @@ pub struct Globals { pub status_message: RwLock, pub pull_following_merge: AtomicBool, + + pub bytes_read: AtomicUsize, } lazy_static! { @@ -112,6 +114,7 @@ lazy_static! { pixels_per_point_times_100: AtomicU32::new(139), // 100 dpi, 1/72th inch => 1.38888 status_message: RwLock::new("Welcome to Gossip. Status messages will appear here. Click them to dismiss them.".to_owned()), pull_following_merge: AtomicBool::new(true), + bytes_read: AtomicUsize::new(0), } }; } diff --git a/src/nip05.rs b/src/nip05.rs index 76ba42fe..ca99f10e 100644 --- a/src/nip05.rs +++ b/src/nip05.rs @@ -4,6 +4,7 @@ use crate::globals::GLOBALS; use crate::people::DbPerson; use dashmap::mapref::entry::Entry; use nostr_types::{Metadata, Nip05, PublicKeyHex, RelayUrl, Unixtime}; +use std::sync::atomic::Ordering; // This updates the people map and the database with the result pub async fn validate_nip05(person: DbPerson) -> Result<(), Error> { @@ -183,5 +184,7 @@ async fn fetch_nip05(user: &str, domain: &str) -> Result { .header("Host", domain) .send(); let response = nip05_future.await?; - Ok(response.json::().await?) + let bytes = response.bytes().await?; + GLOBALS.bytes_read.fetch_add(bytes.len(), Ordering::Relaxed); + Ok(serde_json::from_slice(&bytes)?) } diff --git a/src/overlord/minion/mod.rs b/src/overlord/minion/mod.rs index 1400404c..d9109403 100644 --- a/src/overlord/minion/mod.rs +++ b/src/overlord/minion/mod.rs @@ -6,13 +6,18 @@ use crate::db::DbRelay; use crate::error::Error; use crate::globals::GLOBALS; use base64::Engine; +use encoding_rs::{Encoding, UTF_8}; use futures::{SinkExt, StreamExt}; use futures_util::stream::{SplitSink, SplitStream}; use http::Uri; +use mime::Mime; use nostr_types::{ ClientMessage, EventKind, Filter, IdHex, IdHexPrefix, PublicKeyHex, PublicKeyHexPrefix, RelayInformationDocument, RelayUrl, Unixtime, }; +use reqwest::Response; +use std::borrow::Cow; +use std::sync::atomic::Ordering; use std::time::Duration; use subscription::Subscriptions; use tokio::net::TcpStream; @@ -98,6 +103,7 @@ impl Minion { return Err(Error::UrlHasEmptyHostname); } + // Read NIP-11 information let request_nip11_future = reqwest::Client::builder() .timeout(std::time::Duration::new(30, 0)) .redirect(reqwest::redirect::Policy::none()) @@ -109,9 +115,8 @@ impl Minion { .header("Host", host) .header("Accept", "application/nostr+json") .send(); - - // Read NIP-11 information - match request_nip11_future.await?.text().await { + let response = request_nip11_future.await?; + match Self::text_with_charset(response, "utf-8").await { Ok(text) => match serde_json::from_str::(&text) { Ok(nip11) => { tracing::info!("{}: {}", &self.url, nip11); @@ -239,6 +244,8 @@ impl Minion { } }?; + GLOBALS.bytes_read.fetch_add(ws_message.len(), Ordering::Relaxed); + tracing::trace!("{}: Handling message", &self.url); match ws_message { WsMessage::Text(t) => { @@ -918,4 +925,33 @@ impl Minion { } Ok(()) } + + // This replictes reqwest Response text_with_charset to handle decoding + // whatever charset they used into UTF-8, as well as counting the bytes. + async fn text_with_charset( + response: Response, + default_encoding: &str, + ) -> Result { + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + let encoding_name = content_type + .as_ref() + .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str())) + .unwrap_or(default_encoding); + let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8); + let full = response.bytes().await?; + GLOBALS.bytes_read.fetch_add(full.len(), Ordering::Relaxed); + let (text, _, _) = encoding.decode(&full); + if let Cow::Owned(s) = text { + return Ok(s); + } + unsafe { + // decoding returned Cow::Borrowed, meaning these bytes + // are already valid utf8 + Ok(String::from_utf8_unchecked(full.to_vec())) + } + } } diff --git a/src/ui/help/stats.rs b/src/ui/help/stats.rs index 4aebab89..0752aad6 100644 --- a/src/ui/help/stats.rs +++ b/src/ui/help/stats.rs @@ -1,7 +1,15 @@ use super::GossipUi; +use crate::globals::GLOBALS; use eframe::egui; use egui::{Context, Ui}; +use humansize::{format_size, DECIMAL}; +use std::sync::atomic::Ordering; pub(super) fn update(_app: &mut GossipUi, _ctx: &Context, _frame: &mut eframe::Frame, ui: &mut Ui) { ui.label("STATS PAGE - Coming Soon".to_string()); + + ui.label(format!( + "BYTES READ: {}", + format_size(GLOBALS.bytes_read.load(Ordering::Relaxed), DECIMAL) + )); }