Kieran/route96
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
kieran 2025-02-04 13:06:26 +00:00
parent 5530f39779
commit f06ca98460
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
17 changed files with 643 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

656
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

32
Cargo.toml
View File

@ -3,14 +3,6 @@ name = "route96"
version = "0.4.0"
edition = "2021"
[[bin]]
name = "void_cat_migrate"
required-features = ["bin-void-cat-migrate"]
[[bin]]
name = "void_cat_forced_migrate"
required-features = ["bin-void-cat-force-migrate"]
[[bin]]
name = "route96"
path = "src/bin/main.rs"
@ -19,22 +11,18 @@ path = "src/bin/main.rs"
name = "route96"
[features]
default = ["nip96", "blossom", "analytics", "ranges", "react-ui"]
default = ["nip96", "blossom", "analytics", "react-ui", "payments"]
media-compression = ["dep:ffmpeg-rs-raw", "dep:libc"]
labels = ["nip96", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers"]
labels = ["media-compression", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers"]
nip96 = ["media-compression"]
blossom = []
bin-void-cat-migrate = ["dep:sqlx-postgres"]
bin-void-cat-force-migrate = ["dep:regex", "dep:nostr-cursor"]
torrent-v2 = []
analytics = []
void-cat-redirects = ["dep:sqlx-postgres"]
ranges = ["dep:http-range-header"]
react-ui = []
payments = ["dep:fedimint-tonic-lnd"]
[dependencies]
log = "0.4.21"
nostr = "0.37.0"
nostr = "0.39.0"
pretty_env_logger = "0.5.0"
rocket = { version = "0.5.1", features = ["json"] }
tokio = { version = "1.37.0", features = ["rt", "rt-multi-thread", "macros"] }
@ -45,21 +33,19 @@ uuid = { version = "1.8.0", features = ["v4", "serde"] }
anyhow = "^1.0.82"
sha2 = "0.10.8"
sqlx = { version = "0.8.1", features = ["mysql", "runtime-tokio", "chrono", "uuid"] }
config = { version = "0.14.0", features = ["yaml"] }
config = { version = "0.15.7", features = ["yaml"] }
chrono = { version = "0.4.38", features = ["serde"] }
serde_with = { version = "3.8.1", features = ["hex"] }
reqwest = { version = "0.12.8", features = ["stream"] }
clap = { version = "4.5.18", features = ["derive"] }
mime2ext = "0.1.53"
infer = "0.16.0"
infer = "0.19.0"
tokio-util = { version = "0.7.13", features = ["io", "io-util"] }
http-range-header = { version = "0.4.2" }
base58 = "0.2.0"
libc = { version = "0.2.153", optional = true }
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "a63b88ef3c8f58c7c0ac57d361d06ff0bb3ed385", optional = true }
candle-core = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-nn = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-transformers = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
sqlx-postgres = { version = "0.8.2", optional = true, features = ["chrono", "uuid"] }
http-range-header = { version = "0.4.2", optional = true }
nostr-cursor = { git = "https://git.v0l.io/Kieran/nostr_backup_proc.git", branch = "main", optional = true }
regex = { version = "1.11.1", optional = true }
fedimint-tonic-lnd = { version = "0.2.0", optional = true, features = ["invoicesrpc"] }

7
README.md
View File

@ -12,6 +12,7 @@ Image hosting service
- [BUD-05](https://github.com/hzrd149/blossom/blob/master/buds/05.md)
- [BUD-06](https://github.com/hzrd149/blossom/blob/master/buds/06.md)
- [BUD-08](https://github.com/hzrd149/blossom/blob/master/buds/08.md)
- [BUD-10](https://github.com/hzrd149/blossom/pull/57)
- Image compression to WebP
- Blurhash calculation
- AI image labeling ([ViT224](https://huggingface.co/google/vit-base-patch16-224))
@ -46,3 +47,9 @@ docker run --rm -it \
### Manual
See [install.md](docs/debian.md)
## Blossom only mode (No FFMPEG)
If you don't want to support NIP-96 or any media compression you can build like so:
```bash
cargo build --release --no-default-features --features blossom
```

12
config.prod.yaml
View File

@ -1,17 +1,5 @@
# Listen address for webserver
listen: "0.0.0.0:8000"
# Database connection string (MYSQL)
database: "mysql://root:root@db:3306/route96"
# Directory to store uploads
storage_dir: "/app/data"
# Maximum support filesize for uploading
max_upload_bytes: 104857600
# Public facing url
public_url: "http://localhost:8000"
# Whitelisted pubkeys, leave out to disable
# whitelist: ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"]

38
config.yaml
View File

@ -13,19 +13,37 @@ max_upload_bytes: 5e+9
# Public facing url
public_url: "http://localhost:8000"
# Whitelisted pubkeys, leave out to disable
# (Optional) Whitelisted pubkeys, leave out to disable
# whitelist: ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"]
# Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224)
vit_model:
model: "/home/kieran/Downloads/falcon_nsfw.safetensors"
config: "/home/kieran/Downloads/falcon_nsfw.json"
# (Optional) Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224)
# vit_model:
# model: "falcon_nsfw.safetensors"
# config: "falcon_nsfw.json"
# Analytics support
# (Optional) Analytics support
# plausible_url: "https://plausible.com/"
# Support legacy void
# void_cat_database: "postgres://postgres:postgres@localhost:41911/void"
# Legacy file path for void.cat uploads
# (Optional) Legacy file path for void.cat uploads
# void_cat_files: "/my/void.cat/data"
# (Optional) Payment system config
payments:
# (Optional) Fiat currency used to track exchange rate along with invoices
# If [cost] is using a fiat currency, exchange rates will always be stored
# in that currency, so this config is not needed
fiat: "USD"
# LND node config
lnd:
endpoint: "https://mylnd:10002"
tls: "tls.crt"
macaroon: "admin.macaroon"
# Cost per unit (BTC/USD/EUR/AUD/CAD/JPY/GBP)
cost:
currency: "BTC"
amount: 0.00000100
# Unit metric used to calculate quote (GBSpace, GBEgress)
unit: "GBSpace"
# Billing interval (day / month / year)
interval:
month: 1

22
migrations/20250202135844_payments.sql Normal file
View File

@ -0,0 +1,22 @@
-- Add migration script here
alter table users
add column paid_until timestamp,
add column paid_space integer unsigned not null;
create table payments
(
payment_hash binary(32) not null primary key,
user_id integer unsigned not null,
created timestamp default current_timestamp,
amount integer unsigned not null,
is_paid bit(1) not null,
days_value integer unsigned not null,
size_value integer unsigned not null,
index integer unsigned not null,
rate double not null,
constraint fk_payments_user
foreign key (user_id) references users (id)
on delete cascade
on update restrict
);

12
src/bin/main.rs
View File

@ -73,7 +73,13 @@ async fn main() -> Result<(), Error> {
.attach(Shield::new()) // disable
.mount(
"/",
routes![root, get_blob, head_blob, routes::void_cat_redirect, routes::void_cat_redirect_head],
routes![
root,
get_blob,
head_blob,
routes::void_cat_redirect,
routes::void_cat_redirect_head
],
)
.mount("/admin", routes::admin_routes());
@ -95,6 +101,10 @@ async fn main() -> Result<(), Error> {
{
rocket = rocket.mount("/", routes![routes::get_blob_thumb]);
}
#[cfg(feature = "payments")]
{
rocket = rocket.mount("/", routes::payment::routes());
}
let jh = start_background_tasks(db, fs);

70
src/bin/void_cat_forced_migrate.rs
View File

@ -1,70 +0,0 @@
use clap::Parser;
use log::{info, warn};
use nostr::serde_json;
use nostr_cursor::cursor::NostrCursor;
use regex::Regex;
use rocket::futures::StreamExt;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use uuid::Uuid;
#[derive(Parser, Debug)]
#[command(version, about)]
struct ProgramArgs {
/// Directory pointing to archives to scan
#[arg(short, long)]
pub archive: PathBuf,
/// Output path .csv
#[arg(short, long)]
pub output: PathBuf,
}
#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
pretty_env_logger::init();
let args: ProgramArgs = ProgramArgs::parse();
let mut report: HashMap<String, HashSet<String>> = HashMap::new();
let mut binding = NostrCursor::new(args.archive);
let mut cursor = Box::pin(binding.walk());
let matcher = Regex::new(r"void\.cat/d/(\w+)")?;
while let Some(Ok(e)) = cursor.next().await {
if e.content.contains("void.cat") {
let links = matcher.captures_iter(&e.content).collect::<Vec<_>>();
for link in links {
let g = link.get(1).unwrap().as_str();
let base58 = if let Ok(b) = nostr::bitcoin::base58::decode(g) {
b
} else {
warn!("Invalid base58 id {}", g);
continue;
};
let _uuid = if let Ok(u) = Uuid::from_slice_le(base58.as_slice()) {
u
} else {
warn!("Invalid uuid {}", g);
continue;
};
info!("Got link: {} => {}", g, e.pubkey);
if let Some(ur) = report.get_mut(&e.pubkey) {
ur.insert(g.to_string());
} else {
report.insert(e.pubkey.clone(), HashSet::from([g.to_string()]));
}
}
}
}
let json = serde_json::to_string(&report)?;
File::create(args.output)
.await?
.write_all(json.as_bytes())
.await?;
Ok(())
}

147
src/bin/void_cat_migrate.rs
View File

@ -1,147 +0,0 @@
use anyhow::Error;
use clap::Parser;
use config::Config;
use log::{info, warn};
use nostr::bitcoin::base58;
use route96::db::{Database, FileUpload};
use route96::filesystem::FileStore;
use route96::settings::Settings;
use route96::void_db::VoidCatDb;
use route96::void_file::VoidFile;
use std::path::PathBuf;
use tokio::io::{AsyncWriteExt, BufWriter};
#[derive(Debug, Clone, clap::ValueEnum)]
enum ArgOperation {
Migrate,
ExportNginxRedirects,
}
#[derive(Parser, Debug)]
#[command(version, about)]
struct Args {
/// Database connection string for void.cat DB
#[arg(long)]
pub database: String,
/// Path to filestore on void.cat
#[arg(long)]
pub data_path: String,
#[arg(long)]
pub operation: ArgOperation,
}
#[tokio::main]
async fn main() -> Result<(), Error> {
pretty_env_logger::init();
let builder = Config::builder()
.add_source(config::File::with_name("config.yaml"))
.add_source(config::Environment::with_prefix("APP"))
.build()?;
let settings: Settings = builder.try_deserialize()?;
let db = Database::new(&settings.database).await?;
let fs = FileStore::new(settings.clone());
let args: Args = Args::parse();
let db_void = VoidCatDb::connect(&args.database).await?;
match args.operation {
ArgOperation::Migrate => {
let mut page = 0;
loop {
let files = db_void.list_files(page).await?;
if files.is_empty() {
break;
}
for f in files {
if let Err(e) = migrate_file(&f, &db, &fs, &args).await {
warn!("Failed to migrate file: {}, {}", &f.id, e);
}
}
page += 1;
}
}
ArgOperation::ExportNginxRedirects => {
let path: PathBuf = args.data_path.parse()?;
let conf_path = &path.join("nginx.conf");
info!("Writing redirects to {}", conf_path.to_str().unwrap());
let mut fout = BufWriter::new(tokio::fs::File::create(conf_path).await?);
let mut page = 0;
loop {
let files = db_void.list_files(page).await?;
if files.is_empty() {
break;
}
for f in files {
let legacy_id = base58::encode(f.id.to_bytes_le().as_slice());
let redirect = format!("location ^\\/d\\/{}(?:\\.\\w+)?$ {{\n\treturn 301 https://nostr.download/{};\n}}\n", &legacy_id, &f.digest);
fout.write_all(redirect.as_bytes()).await?;
}
page += 1;
}
}
}
Ok(())
}
async fn migrate_file(
f: &VoidFile,
db: &Database,
fs: &FileStore,
args: &Args,
) -> Result<(), Error> {
let pubkey_vec = hex::decode(&f.email)?;
let id_vec = hex::decode(&f.digest)?;
// copy file
let src_path = PathBuf::new()
.join(&args.data_path)
.join(VoidFile::map_to_path(&f.id));
let dst_path = fs.map_path(&id_vec);
if src_path.exists() && !dst_path.exists() {
info!(
"Copying file: {} from {} => {}",
&f.id,
src_path.to_str().unwrap(),
dst_path.to_str().unwrap()
);
tokio::fs::create_dir_all(dst_path.parent().unwrap()).await?;
tokio::fs::copy(src_path, dst_path).await?;
} else if dst_path.exists() {
info!("File already exists {}, continuing...", &f.id);
} else {
anyhow::bail!("Source file not found {}", src_path.to_str().unwrap());
}
let uid = db.upsert_user(&pubkey_vec).await?;
info!("Mapped user {} => {}", &f.email, uid);
let md: Option<Vec<&str>> = f.media_dimensions.as_ref().map(|s| s.split("x").collect());
let fu = FileUpload {
id: id_vec,
name: f.name.clone(),
size: f.size as u64,
mime_type: f.mime_type.clone(),
created: f.uploaded,
width: match &md {
Some(s) => Some(s[0].parse::<u32>()?),
None => None,
},
height: match &md {
Some(s) => Some(s[1].parse::<u32>()?),
None => None,
},
blur_hash: None,
alt: f.description.clone(),
duration: None,
bitrate: None,
};
db.add_file(&fu, uid).await?;
Ok(())
}

4
src/db.rs
View File

@ -61,6 +61,10 @@ pub struct User {
pub pubkey: Vec<u8>,
pub created: DateTime<Utc>,
pub is_admin: bool,
#[cfg(feature = "payments")]
pub paid_until: Option<DateTime<Utc>>,
#[cfg(feature = "payments")]
pub paid_space: u64,
}
#[cfg(feature = "labels")]

3
src/filesystem.rs
View File

@ -1,6 +1,5 @@
#[cfg(feature = "labels")]
use crate::db::FileLabel;
use crate::processing::can_compress;
#[cfg(feature = "labels")]
use crate::processing::labeling::label_frame;
#[cfg(feature = "media-compression")]
@ -18,6 +17,7 @@ use std::path::PathBuf;
use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncReadExt};
use uuid::Uuid;
use crate::can_compress;
#[derive(Clone)]
pub enum FileSystemResult {
@ -180,6 +180,7 @@ impl FileStore {
}
}
#[cfg(feature = "media-compression")]
async fn compress_file(&self, input: &PathBuf, mime_type: &str) -> Result<NewFileResult> {
let compressed_result = compress_file(input, mime_type, &self.temp_dir())?;
#[cfg(feature = "labels")]

7
src/lib.rs
View File

@ -9,6 +9,9 @@ pub mod filesystem;
pub mod processing;
pub mod routes;
pub mod settings;
#[cfg(any(feature = "void-cat-redirects", feature = "bin-void-cat-migrate"))]
pub mod void_db;
pub mod void_file;
pub fn can_compress(mime_type: &str) -> bool {
mime_type.starts_with("image/")
}

5
src/processing/mod.rs
View File

@ -5,6 +5,7 @@ use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo, Encoder, Scaler, StreamType,
use std::path::{Path, PathBuf};
use std::ptr;
use uuid::Uuid;
use crate::can_compress;
#[cfg(feature = "labels")]
pub mod labeling;
@ -136,10 +137,6 @@ pub struct NewFileProcessorResult {
pub bitrate: u32,
}
pub fn can_compress(mime_type: &str) -> bool {
mime_type.starts_with("image/")
}
pub fn compress_file(
path: &Path,
mime_type: &str,

10
src/routes/admin.rs
View File

@ -53,6 +53,10 @@ pub struct SelfUser {
pub is_admin: bool,
pub file_count: u64,
pub total_size: u64,
#[cfg(feature = "payments")]
pub paid_until: u64,
#[cfg(feature = "payments")]
pub quota: u64,
}
#[derive(Serialize)]
@ -77,6 +81,12 @@ async fn admin_get_self(auth: Nip98Auth, db: &State<Database>) -> AdminResponse<
is_admin: user.is_admin,
file_count: s.file_count,
total_size: s.total_size,
paid_until: if let Some(u) = &user.paid_until {
u.timestamp() as u64
} else {
0
},
quota: user.paid_space,
})
}
Err(_) => AdminResponse::error("User not found"),

28
src/routes/mod.rs
View File

@ -10,6 +10,7 @@ pub use crate::routes::nip96::nip96_routes;
use crate::settings::Settings;
use crate::void_file::VoidFile;
use anyhow::Error;
use base58::FromBase58;
use http_range_header::{parse_range_header, EndPosition, StartPosition};
use log::{debug, warn};
use nostr::Event;
@ -33,6 +34,8 @@ mod blossom;
mod nip96;
mod admin;
#[cfg(feature = "payments")]
pub mod payment;
pub struct FilePayload {
pub file: File,
@ -183,8 +186,6 @@ impl<'r> Responder<'r, 'static> for FilePayload {
response.set_header(Header::new("cache-control", "max-age=31536000, immutable"));
// handle ranges
#[cfg(feature = "ranges")]
{
const MAX_UNBOUNDED_RANGE: u64 = 1024 * 1024;
// only use range response for files > 1MiB
if self.info.size < MAX_UNBOUNDED_RANGE {
@ -194,9 +195,7 @@ impl<'r> Responder<'r, 'static> for FilePayload {
if let Some(r) = request.headers().get("range").next() {
if let Ok(ranges) = parse_range_header(r) {
if ranges.ranges.len() > 1 {
warn!(
"Multipart ranges are not supported, fallback to non-range request"
);
warn!("Multipart ranges are not supported, fallback to non-range request");
response.set_streamed_body(self.file);
} else {
let single_range = ranges.ranges.first().unwrap();
@ -217,12 +216,7 @@ impl<'r> Responder<'r, 'static> for FilePayload {
response.set_header(Header::new("content-length", r_len.to_string()));
response.set_header(Header::new(
"content-range",
format!(
"bytes {}-{}/{}",
range_start,
range_end - 1,
self.info.size
),
format!("bytes {}-{}/{}", range_start, range_end - 1, self.info.size),
));
response.set_streamed_body(Box::pin(r_body));
}
@ -231,11 +225,6 @@ impl<'r> Responder<'r, 'static> for FilePayload {
response.set_sized_body(None, self.file);
}
}
}
#[cfg(not(feature = "ranges"))]
{
response.set_sized_body(None, self.file);
}
if let Ok(ct) = ContentType::from_str(&self.info.mime_type) {
response.set_header(ct);
@ -437,9 +426,7 @@ pub async fn void_cat_redirect(id: &str, settings: &State<Settings>) -> Option<N
id
};
if let Some(base) = &settings.void_cat_files {
let uuid =
uuid::Uuid::from_slice_le(nostr::bitcoin::base58::decode(id).unwrap().as_slice())
.unwrap();
let uuid = uuid::Uuid::from_slice_le(id.from_base58().unwrap().as_slice()).unwrap();
let f = base.join(VoidFile::map_to_path(&uuid));
debug!("Legacy file map: {} => {}", id, f.display());
if let Ok(f) = NamedFile::open(f).await {
@ -459,8 +446,7 @@ pub async fn void_cat_redirect_head(id: &str) -> VoidCatFile {
} else {
id
};
let uuid =
uuid::Uuid::from_slice_le(nostr::bitcoin::base58::decode(id).unwrap().as_slice()).unwrap();
let uuid = uuid::Uuid::from_slice_le(id.from_base58().unwrap().as_slice()).unwrap();
VoidCatFile {
status: Status::Ok,
uuid: Header::new("X-UUID", uuid.to_string()),

55
src/routes/payment.rs Normal file
View File

@ -0,0 +1,55 @@
use crate::auth::nip98::Nip98Auth;
use crate::db::Database;
use crate::settings::{PaymentAmount, PaymentInterval, PaymentUnit, Settings};
use rocket::serde::json::Json;
use rocket::{routes, Route, State};
use serde::{Deserialize, Serialize};
pub fn routes() -> Vec<Route> {
routes![get_payment]
}
#[derive(Deserialize, Serialize)]
struct PaymentInfo {
/// Billing quota metric
pub unit: PaymentUnit,
/// Amount of time to bill units (GB/mo, Gb Egress/day etc.)
pub interval: PaymentInterval,
/// Value amount of payment
pub cost: PaymentAmount,
}
#[derive(Deserialize, Serialize)]
struct PaymentRequest {
/// Number of units requested to make payment
pub units: f32,
/// Quantity of orders to make
pub quantity: u16,
}
#[derive(Deserialize, Serialize)]
struct PaymentResponse {}
#[rocket::get("/payment")]
async fn get_payment(settings: &State<Settings>) -> Option<Json<PaymentInfo>> {
settings.payments.as_ref().map(|p| {
Json::from(PaymentInfo {
unit: p.unit.clone(),
interval: p.interval.clone(),
cost: p.cost.clone(),
})
})
}
#[rocket::post("/payment", data = "<req>", format = "json")]
async fn req_payment(
auth: Nip98Auth,
db: &State<Database>,
settings: &State<Settings>,
req: Json<PaymentRequest>,
) -> Json<PaymentResponse> {
Json::from(PaymentResponse {})
}

69
src/settings.rs
View File

@ -30,11 +30,12 @@ pub struct Settings {
/// Analytics tracking
pub plausible_url: Option<String>,
#[cfg(feature = "void-cat-redirects")]
pub void_cat_database: Option<String>,
/// Path to void.cat uploads (files-v2)
pub void_cat_files: Option<PathBuf>,
#[cfg(feature = "payments")]
/// Payment options for paid storage
pub payments: Option<PaymentConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -42,3 +43,65 @@ pub struct VitModelConfig {
pub model: PathBuf,
pub config: PathBuf,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaymentConfig {
/// LND connection details
pub lnd: LndConfig,
/// Pricing per unit
pub cost: PaymentAmount,
/// What metric to bill payments on
pub unit: PaymentUnit,
/// Billing interval time per unit
pub interval: PaymentInterval,
/// Fiat base currency to store exchange rates along with invoice
pub fiat: Option<Currency>,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaymentAmount {
pub currency: Currency,
pub amount: f32,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Currency {
BTC,
USD,
EUR,
GBP,
JPY,
CAD,
AUD,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PaymentUnit {
GBSpace,
GBEgress,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PaymentInterval {
Day(u16),
Month(u16),
Year(u16),
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LndConfig {
pub endpoint: String,
pub tls: PathBuf,
pub macaroon: PathBuf,
}