From 1b84e22ea9b1bad6e1d975272fa723e8352548a7 Mon Sep 17 00:00:00 2001 From: kieran Date: Wed, 11 Dec 2024 09:46:00 +0000 Subject: [PATCH] refactor: yaml configs --- Cargo.lock | 106 +----------------------------------- Cargo.toml | 4 +- README.md | 4 +- config.prod.toml | 17 ------ config.prod.yaml | 17 ++++++ config.toml | 32 ----------- config.yaml | 31 +++++++++++ docker-compose.prod.yml | 2 +- src/analytics/plausible.rs | 6 +- src/bin/main.rs | 2 +- src/bin/void_cat_migrate.rs | 2 +- src/filesystem.rs | 9 +-- src/processing/labeling.rs | 1 + src/settings.rs | 11 ++-- src/webhook.rs | 5 +- 15 files changed, 77 insertions(+), 172 deletions(-) delete mode 100644 config.prod.toml create mode 100644 config.prod.yaml delete mode 100644 config.toml create mode 100644 config.yaml diff --git a/Cargo.lock b/Cargo.lock index 3a44f57..6f6945a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,17 +27,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.11" @@ -289,7 +278,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex", "syn", ] @@ -1699,7 +1688,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -2110,11 +2098,10 @@ dependencies = [ [[package]] name = "nostr" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ad56c1d9a59f4edc46b17bc64a217b38b99baefddc0080f85ad98a0855336d" +checksum = "8aad4b767bbed24ac5eb4465bfb83bc1210522eb99d67cf4e547ec2ec7e47786" dependencies = [ - "aes", "async-trait", "base64 0.22.1", "bech32", @@ -2125,19 +2112,14 @@ dependencies = [ "chacha20poly1305", "getrandom", "instant", - "js-sys", "negentropy 0.3.1", "negentropy 0.4.3", "once_cell", - "reqwest", "scrypt", "serde", "serde_json", "unicode-normalization", "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", ] [[package]] @@ -2643,54 +2625,6 @@ dependencies = [ "reborrow", ] -[[package]] -name = "quinn" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" -dependencies = [ - "bytes", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash 2.0.0", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "quinn-proto" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" -dependencies = [ - "bytes", - "rand", - "ring", - "rustc-hash 2.0.0", - "rustls", - "slab", - "thiserror", - "tinyvec", - "tracing", -] - -[[package]] -name = "quinn-udp" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" -dependencies = [ - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "quote" version = "1.0.37" @@ -2884,10 +2818,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", "rustls-pemfile", - "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2895,14 +2826,11 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", - "tokio-socks", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", "windows-registry", ] @@ -3088,12 +3016,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc-hash" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" - [[package]] name = "rustix" version = "0.38.35" @@ -3114,7 +3036,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3954,18 +3875,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-socks" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" -dependencies = [ - "either", - "futures-util", - "thiserror", - "tokio", -] - [[package]] name = "tokio-stream" version = "0.1.15" @@ -4377,15 +4286,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "whoami" version = "1.5.1" diff --git a/Cargo.toml b/Cargo.toml index 8f3868b..1a6def7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ void-cat-redirects = ["dep:sqlx-postgres"] [dependencies] log = "0.4.21" -nostr = "0.36.0" +nostr = "0.37.0" pretty_env_logger = "0.5.0" rocket = { version = "0.5.0", features = ["json"] } tokio = { version = "1.37.0", features = ["rt", "rt-multi-thread", "macros"] } @@ -38,7 +38,7 @@ uuid = { version = "1.8.0", features = ["v4"] } anyhow = "^1.0.82" sha2 = "0.10.8" sqlx = { version = "0.8.1", features = ["mysql", "runtime-tokio", "chrono", "uuid"] } -config = { version = "0.14.0", features = ["toml"] } +config = { version = "0.14.0", features = ["yaml"] } chrono = { version = "0.4.38", features = ["serde"] } url = "2.5.0" serde_with = { version = "3.8.1", features = ["hex"] } diff --git a/README.md b/README.md index ddec98a..91b1fa4 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ docker compose -f docker-compose.prod.yml up ### Manual -Assuming you already created your `config.toml` and configured the `database` run: +Assuming you already created your `config.yaml` and configured the `database` run: ```bash docker run --rm -it \ -p 8000:8000 \ - -v ./config.toml:/app/config.toml \ + -v ./config.yaml:/app/config.yaml \ -e "RUST_LOG=info" \ voidic/route96 ``` diff --git a/config.prod.toml b/config.prod.toml deleted file mode 100644 index b6edc70..0000000 --- a/config.prod.toml +++ /dev/null @@ -1,17 +0,0 @@ -# Listen address for webserver -listen = "0.0.0.0:8000" - -# Database connection string (MYSQL) -database = "mysql://root:root@db:3306/route96" - -# Directory to store uploads -storage_dir = "/app/data" - -# Maximum support filesize for uploading -max_upload_bytes = 104857600 - -# Public facing url -public_url = "http://localhost:8000" - -# Whitelisted pubkeys, leave out to disable -# whitelist = ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"] \ No newline at end of file diff --git a/config.prod.yaml b/config.prod.yaml new file mode 100644 index 0000000..c490aba --- /dev/null +++ b/config.prod.yaml @@ -0,0 +1,17 @@ +# Listen address for webserver +listen: "0.0.0.0:8000" + +# Database connection string (MYSQL) +database: "mysql://root:root@db:3306/route96" + +# Directory to store uploads +storage_dir: "/app/data" + +# Maximum support filesize for uploading +max_upload_bytes: 104857600 + +# Public facing url +public_url: "http://localhost:8000" + +# Whitelisted pubkeys, leave out to disable +# whitelist: ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"] \ No newline at end of file diff --git a/config.toml b/config.toml deleted file mode 100644 index 84187fc..0000000 --- a/config.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Listen address for webserver -listen = "127.0.0.1:8000" - -# Database connection string (MYSQL) -database = "mysql://root:root@localhost:3366/route96" - -# Directory to store uploads -storage_dir = "./data" - -# Maximum support filesize for uploading -max_upload_bytes = 5e+9 - -# Public facing url -public_url = "http://localhost:8000" - -# Whitelisted pubkeys, leave out to disable -# whitelist = ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"] - -# Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224) -#vit_model_path = "falcon_nsfw.safetensors" - -# Path for ViT(224) image model config.json -#vit_model_config_path = "falcon_nsfw.json" - -# Webhook api endpoint -# webhook_url = "https://api.snort.social/api/v1/media/webhook" - -# Analytics support -# plausible_url = "https://plausible.com/" - -# Support legacy void -# void_cat_database = "postgres://postgres:postgres@localhost:41911/void" \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..d1e7a1a --- /dev/null +++ b/config.yaml @@ -0,0 +1,31 @@ +# Listen address for webserver +listen: "127.0.0.1:8000" + +# Database connection string (MYSQL) +database: "mysql://root:root@localhost:3366/route96" + +# Directory to store uploads +storage_dir: "./data" + +# Maximum support filesize for uploading +max_upload_bytes: 5e+9 + +# Public facing url +public_url: "http://localhost:8000" + +# Whitelisted pubkeys, leave out to disable +# whitelist: ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"] + +# Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224) +vit_model: + model: "/home/kieran/Downloads/falcon_nsfw.safetensors" + config: "/home/kieran/Downloads/falcon_nsfw.json" + +# Webhook api endpoint +# webhook_url: "https://api.snort.social/api/v1/media/webhook" + +# Analytics support +# plausible_url: "https://plausible.com/" + +# Support legacy void +# void_cat_database: "postgres://postgres:postgres@localhost:41911/void" \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1c58c6d..0be37e0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,4 +18,4 @@ services: - "8000:8000" volumes: - "files:/app/data" - - "./config.prod.toml:/app/config.toml" + - "./config.prod.yaml:/app/config.yaml" diff --git a/src/analytics/plausible.rs b/src/analytics/plausible.rs index e6270ee..512e4df 100644 --- a/src/analytics/plausible.rs +++ b/src/analytics/plausible.rs @@ -2,6 +2,7 @@ use crate::analytics::Analytics; use crate::settings::Settings; use anyhow::Error; use log::{info, warn}; +use nostr::serde_json; use reqwest::ClientBuilder; use rocket::Request; use serde::{Deserialize, Serialize}; @@ -36,6 +37,8 @@ impl PlausibleAnalytics { tokio::spawn(async move { while let Some(mut msg) = rx.recv().await { msg.url = format!("{}{}", pub_url, msg.url); + + let body = serde_json::to_string(&msg).unwrap(); match c .post(format!("{}/api/event", url)) .header( @@ -52,7 +55,8 @@ impl PlausibleAnalytics { None => "", }, ) - .json(&msg) + .header("content-type", "application/json") + .body(body) .timeout(Duration::from_secs(30)) .send() .await diff --git a/src/bin/main.rs b/src/bin/main.rs index ed5d943..ea3aaba 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -39,7 +39,7 @@ async fn main() -> Result<(), Error> { .add_source(config::File::with_name(if let Some(ref c) = args.config { c.as_str() } else { - "config.toml" + "config.yaml" })) .add_source(config::Environment::with_prefix("APP")) .build()?; diff --git a/src/bin/void_cat_migrate.rs b/src/bin/void_cat_migrate.rs index 3668f87..2c24edc 100644 --- a/src/bin/void_cat_migrate.rs +++ b/src/bin/void_cat_migrate.rs @@ -36,7 +36,7 @@ async fn main() -> Result<(), Error> { pretty_env_logger::init(); let builder = Config::builder() - .add_source(config::File::with_name("config.toml")) + .add_source(config::File::with_name("config.yaml")) .add_source(config::Environment::with_prefix("APP")) .build()?; diff --git a/src/filesystem.rs b/src/filesystem.rs index b9cca12..937e666 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -108,13 +108,8 @@ impl FileStore { let start = SystemTime::now(); #[cfg(feature = "labels")] - let labels = if let Some(mp) = &self.settings.vit_model_path { - let config_path = if let Some(c) = &self.settings.vit_model_config_path { - c - } else { - bail!("Missing vit_model_config_path"); - }; - label_frame(&new_temp.result, mp.clone(), config_path.clone())? + let labels = if let Some(mp) = &self.settings.vit_model { + label_frame(&new_temp.result, mp.model.clone(), mp.config.clone())? .iter() .map(|l| FileLabel::new(l.0.clone(), "vit224".to_string())) .collect() diff --git a/src/processing/labeling.rs b/src/processing/labeling.rs index 6d17504..bdefb80 100644 --- a/src/processing/labeling.rs +++ b/src/processing/labeling.rs @@ -55,6 +55,7 @@ unsafe fn load_image(path_buf: &Path, width: usize, height: usize) -> Result>, /// Path for ViT image model - pub vit_model_path: Option, - - /// Path for ViT image model config.json - pub vit_model_config_path: Option, + pub vit_model: Option, /// Webhook api endpoint pub webhook_url: Option, @@ -36,3 +33,9 @@ pub struct Settings { #[cfg(feature = "void-cat-redirects")] pub void_cat_database: Option, } + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VitModelConfig { + pub model: PathBuf, + pub config: PathBuf, +} diff --git a/src/webhook.rs b/src/webhook.rs index 0a96e2c..4053896 100644 --- a/src/webhook.rs +++ b/src/webhook.rs @@ -1,4 +1,5 @@ use anyhow::Error; +use nostr::serde_json; use reqwest::{Client, ClientBuilder}; use serde::{Deserialize, Serialize}; @@ -31,11 +32,13 @@ impl Webhook { subject: Some(hex::encode(pubkey)), payload: fs, }; + let body = serde_json::to_string(&body)?; let req = self .client .post(&self.url) .header("accept", "application/json") - .json(&body) + .header("content-type", "application/json") + .body(body) .send() .await?;