Compare commits

..

1 Commits

Author SHA1 Message Date
2238a157e4 wip 2025-01-20 11:16:48 +00:00
35 changed files with 706 additions and 177197 deletions

View File

@ -5,6 +5,11 @@ metadata:
namespace: git
concurrency:
limit: 1
trigger:
branch:
- main
event:
- push
steps:
- name: build
image: docker
@ -16,8 +21,9 @@ steps:
from_secret: docker_hub
commands:
- dockerd &
- docker login -u kieran -p $TOKEN git.v0l.io
- docker login -u voidic -p $TOKEN_DOCKER
- docker buildx build --push -t voidic/route96:latest .
- docker buildx build --push -t git.v0l.io/kieran/route96:latest -t voidic/route96:latest .
- kill $(cat /var/run/docker.pid)
---
kind: pipeline
@ -41,6 +47,7 @@ steps:
from_secret: docker_hub
commands:
- dockerd &
- docker login -u kieran -p $TOKEN git.v0l.io
- docker login -u voidic -p $TOKEN_DOCKER
- docker buildx build --push voidic/route96:$DRONE_TAG .
- docker buildx build --push -t git.v0l.io/kieran/route96:$DRONE_TAG -t voidic/route96:$DRONE_TAG .
- kill $(cat /var/run/docker.pid)

1090
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -3,6 +3,14 @@ name = "route96"
version = "0.4.0"
edition = "2021"
[[bin]]
name = "void_cat_migrate"
required-features = ["bin-void-cat-migrate"]
[[bin]]
name = "void_cat_forced_migrate"
required-features = ["bin-void-cat-force-migrate"]
[[bin]]
name = "route96"
path = "src/bin/main.rs"
@ -11,18 +19,22 @@ path = "src/bin/main.rs"
name = "route96"
[features]
default = ["nip96", "blossom", "analytics", "react-ui", "payments"]
default = ["nip96", "blossom", "analytics", "ranges", "react-ui"]
media-compression = ["dep:ffmpeg-rs-raw", "dep:libc"]
labels = ["media-compression", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers"]
labels = ["nip96", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers"]
nip96 = ["media-compression"]
blossom = []
bin-void-cat-migrate = ["dep:sqlx-postgres"]
bin-void-cat-force-migrate = ["dep:regex", "dep:nostr-cursor"]
torrent-v2 = []
analytics = []
void-cat-redirects = ["dep:sqlx-postgres"]
ranges = ["dep:http-range-header"]
react-ui = []
payments = ["dep:fedimint-tonic-lnd"]
[dependencies]
log = "0.4.21"
nostr = "0.39.0"
nostr = "0.37.0"
pretty_env_logger = "0.5.0"
rocket = { version = "0.5.1", features = ["json"] }
tokio = { version = "1.37.0", features = ["rt", "rt-multi-thread", "macros"] }
@ -33,19 +45,21 @@ uuid = { version = "1.8.0", features = ["v4", "serde"] }
anyhow = "^1.0.82"
sha2 = "0.10.8"
sqlx = { version = "0.8.1", features = ["mysql", "runtime-tokio", "chrono", "uuid"] }
config = { version = "0.15.7", features = ["yaml"] }
config = { version = "0.14.0", features = ["yaml"] }
chrono = { version = "0.4.38", features = ["serde"] }
reqwest = { version = "0.12.8", features = ["stream", "http2"] }
serde_with = { version = "3.8.1", features = ["hex"] }
reqwest = { version = "0.12.8", features = ["stream"] }
clap = { version = "4.5.18", features = ["derive"] }
mime2ext = "0.1.53"
infer = "0.19.0"
infer = "0.16.0"
tokio-util = { version = "0.7.13", features = ["io", "io-util"] }
http-range-header = { version = "0.4.2" }
base58 = "0.2.0"
libc = { version = "0.2.153", optional = true }
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "a63b88ef3c8f58c7c0ac57d361d06ff0bb3ed385", optional = true }
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "de2050cec07a095bace38d3ccf9c4c4f9b03b217", optional = true }
candle-core = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-nn = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-transformers = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
fedimint-tonic-lnd = { version = "0.2.0", optional = true, default-features = false, features = ["invoicesrpc", "lightningrpc"] }
sqlx-postgres = { version = "0.8.2", optional = true, features = ["chrono", "uuid"] }
http-range-header = { version = "0.4.2", optional = true }
nostr-cursor = { git = "https://git.v0l.io/Kieran/nostr_backup_proc.git", branch = "main", optional = true }
regex = { version = "1.11.1", optional = true }

View File

@ -13,39 +13,19 @@ max_upload_bytes: 5e+9
# Public facing url
public_url: "http://localhost:8000"
# (Optional) Whitelisted pubkeys, leave out to disable
# Whitelisted pubkeys, leave out to disable
# whitelist: ["63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed"]
# (Optional) Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224)
# vit_model:
# model: "falcon_nsfw.safetensors"
# config: "falcon_nsfw.json"
# Path for ViT(224) image model (https://huggingface.co/google/vit-base-patch16-224)
vit_model:
model: "/home/kieran/Downloads/falcon_nsfw.safetensors"
config: "/home/kieran/Downloads/falcon_nsfw.json"
# (Optional) Analytics support
# Analytics support
# plausible_url: "https://plausible.com/"
# (Optional) Legacy file path for void.cat uploads
# void_cat_files: "/my/void.cat/data"
# Support legacy void
# void_cat_database: "postgres://postgres:postgres@localhost:41911/void"
# (Optional) Payment system config
payments:
# (Optional) Free quota in bytes for users without payments (default: 100MB)
free_quota_bytes: 104857600
# (Optional) Fiat currency used to track exchange rate along with invoices
# If [cost] is using a fiat currency, exchange rates will always be stored
# in that currency, so this config is not needed
fiat: "USD"
# LND node config
lnd:
endpoint: "https://127.0.0.1:10001"
tls: "/home/kieran/.polar/networks/3/volumes/lnd/alice/tls.cert"
macaroon: "/home/kieran/.polar/networks/3/volumes/lnd/alice/data/chain/bitcoin/regtest/admin.macaroon"
# Cost per unit (BTC/USD/EUR/AUD/CAD/JPY/GBP)
cost:
currency: "BTC"
amount: 0.00000100
# Unit metric used to calculate quote (GBSpace, GBEgress)
unit: "GBSpace"
# Billing interval (day / month / year)
interval:
month: 1
# Legacy file path for void.cat uploads
# void_cat_files: "/my/void.cat/data"

View File

@ -18,13 +18,13 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"id": 15,
"links": [],
"panels": [
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -93,18 +93,18 @@
}
]
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "void_cat",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n hex(users.pubkey) as pubkey, \n count(user_uploads.file) as uploads, \n sum(uploads.size) as size\nFROM users, user_uploads, uploads\nwhere users.id = user_uploads.user_id\nand user_uploads.file = uploads.id\ngroup by users.pubkey",
"rawSql": "SELECT \n hex(void_cat.users.pubkey) as pubkey, \n count(void_cat.user_uploads.file) as uploads, \n sum(void_cat.uploads.size) as size\nFROM void_cat.users, void_cat.user_uploads, void_cat.uploads\nwhere void_cat.users.id = void_cat.user_uploads.user_id\nand void_cat.user_uploads.file = void_cat.uploads.id\ngroup by void_cat.users.pubkey",
"refId": "A",
"sql": {
"columns": [
@ -137,7 +137,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -174,7 +174,6 @@
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -186,18 +185,18 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select \n sum(uploads.size) as size\nfrom uploads",
"rawSql": "select \n sum(uploads.size) as size\nfrom void_cat.uploads",
"refId": "A",
"sql": {
"columns": [
@ -224,7 +223,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -261,7 +260,6 @@
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -273,18 +271,18 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select \n count(users.pubkey) as users\nfrom users",
"rawSql": "select \n count(users.pubkey) as users\nfrom void_cat.users",
"refId": "A",
"sql": {
"columns": [
@ -311,7 +309,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -348,7 +346,6 @@
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -360,18 +357,18 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select \n count(uploads.id) as files\nfrom uploads",
"rawSql": "select \n count(uploads.id) as files\nfrom void_cat.uploads",
"refId": "A",
"sql": {
"columns": [
@ -398,7 +395,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -448,18 +445,18 @@
},
"showHeader": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select \n hex(uploads.id) as sha256,\n hex(users.pubkey) as uploader,\n uploads.name,\n sys.format_bytes(uploads.size) as size,\n uploads.mime_type,\n uploads.created,\n uploads.width,\n uploads.height\nfrom uploads, user_uploads, users\nwhere uploads.id = user_uploads.file\nand users.id = user_uploads.user_id\norder by uploads.created desc\nlimit 50",
"rawSql": "select \n hex(uploads.id) as sha256,\n hex(users.pubkey) as uploader,\n uploads.name,\n sys.format_bytes(uploads.size) as size,\n uploads.mime_type,\n uploads.created,\n uploads.width,\n uploads.height\nfrom void_cat.uploads, void_cat.user_uploads, void_cat.users\nwhere uploads.id = user_uploads.file\nand users.id = user_uploads.user_id\norder by uploads.created desc\nlimit 50",
"refId": "A",
"sql": {
"columns": [
@ -486,7 +483,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -536,18 +533,18 @@
},
"showHeader": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select \n hex(uploads.id) as sha256,\n hex(users.pubkey) as uploader,\n uploads.name,\n sys.format_bytes(uploads.size) as size,\n uploads.mime_type,\n uploads.created\nfrom uploads, user_uploads, users\nwhere uploads.id = user_uploads.file\nand users.id = user_uploads.user_id\norder by uploads.size desc\nlimit 50",
"rawSql": "select \n hex(uploads.id) as sha256,\n hex(users.pubkey) as uploader,\n uploads.name,\n sys.format_bytes(uploads.size) as size,\n uploads.mime_type,\n uploads.created\nfrom void_cat.uploads, void_cat.user_uploads, void_cat.users\nwhere uploads.id = user_uploads.file\nand users.id = user_uploads.user_id\norder by uploads.size desc\nlimit 50",
"refId": "A",
"sql": {
"columns": [
@ -574,7 +571,7 @@
{
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"fieldConfig": {
"defaults": {
@ -624,13 +621,13 @@
},
"showHeader": true
},
"pluginVersion": "11.5.2",
"pluginVersion": "10.4.2",
"targets": [
{
"dataset": "mysql",
"datasource": {
"type": "mysql",
"uid": "behhij20nn4zka"
"uid": "cdnhzi5uxm8zkb"
},
"editorMode": "code",
"format": "table",
@ -660,9 +657,7 @@
"type": "table"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 40,
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
@ -675,6 +670,6 @@
"timezone": "browser",
"title": "route96",
"uid": "ddni0rqotyltse",
"version": 3,
"version": 12,
"weekStart": ""
}

View File

@ -0,0 +1,15 @@
-- Add migration script here
create table reports
(
file_id binary(32) not null,
report_id binary(32) not null,
reporter_pubkey binary(32) not null,
event varchar(8192) not null,
received timestamp default current_timestamp,
constraint fk_reports_file_id
foreign key (file_id) references uploads (id)
on delete cascade
on update restrict
);
create unique index ix_reports_file_report_id on reports (file_id, report_id);

View File

@ -1,4 +0,0 @@
-- Add migration script here
alter table uploads
add column duration float,
add column bitrate integer unsigned;

View File

@ -1,22 +0,0 @@
-- Add migration script here
alter table users
add column paid_until timestamp,
add column paid_size integer unsigned not null;
create table payments
(
payment_hash binary(32) not null primary key,
user_id integer unsigned not null,
created timestamp default current_timestamp,
amount integer unsigned not null,
is_paid bit(1) not null default 0,
days_value integer unsigned not null,
size_value integer unsigned not null,
settle_index integer unsigned,
rate float,
constraint fk_payments_user_id
foreign key (user_id) references users (id)
on delete cascade
on update restrict
);

View File

@ -1,7 +1,7 @@
use crate::analytics::Analytics;
use crate::settings::Settings;
use anyhow::Error;
use log::{debug, warn};
use log::{info, warn};
use nostr::serde_json;
use reqwest::ClientBuilder;
use rocket::Request;
@ -61,7 +61,7 @@ impl PlausibleAnalytics {
.send()
.await
{
Ok(_v) => debug!("Sent {:?}", msg),
Ok(_v) => info!("Sent {:?}", msg),
Err(e) => warn!("Failed to track: {}", e),
}
}

View File

@ -33,12 +33,10 @@ impl<'r> FromRequest<'r> for Nip98Auth {
if event.kind != Kind::HttpAuth {
return Outcome::Error((Status::new(401), "Wrong event kind"));
}
if (event.created_at.as_u64() as i64 -
Timestamp::now().as_u64() as i64).abs() >= 60
{
if event.created_at > Timestamp::now() {
return Outcome::Error((
Status::new(401),
"Created timestamp is out of range",
"Created timestamp is in the future",
));
}

View File

@ -1,98 +0,0 @@
use crate::db::{Database, FileUpload};
use crate::filesystem::FileStore;
use crate::processing::probe_file;
use anyhow::Result;
use log::{error, info, warn};
pub struct MediaMetadata {
db: Database,
fs: FileStore,
}
impl MediaMetadata {
pub fn new(db: Database, fs: FileStore) -> Self {
Self { db, fs }
}
pub async fn process(&mut self) -> Result<()> {
let to_migrate = self.db.get_missing_media_metadata().await?;
info!("{} files are missing metadata", to_migrate.len());
for file in to_migrate {
// probe file and update metadata
let path = self.fs.get(&file.id);
match probe_file(&path) {
Ok(data) => {
let bv = data.best_video();
let duration = if data.duration < 0.0 {
None
} else {
Some(data.duration)
};
let bitrate = if data.bitrate == 0 {
None
} else {
Some(data.bitrate as u32)
};
info!(
"Updating metadata: id={}, dim={}x{}, dur={}, br={}",
hex::encode(&file.id),
bv.map(|v| v.width).unwrap_or(0),
bv.map(|v| v.height).unwrap_or(0),
duration.unwrap_or(0.0),
bitrate.unwrap_or(0)
);
if let Err(e) = self
.db
.update_metadata(
&file.id,
bv.map(|v| v.width as u32),
bv.map(|v| v.height as u32),
duration,
bitrate,
)
.await
{
error!("Failed to update metadata: {}", e);
}
}
Err(e) => {
warn!("Skipping missing file: {}, {}", hex::encode(&file.id), e);
}
}
}
Ok(())
}
}
impl Database {
pub async fn get_missing_media_metadata(&mut self) -> Result<Vec<FileUpload>> {
let results: Vec<FileUpload> = sqlx::query_as("select * from uploads where \
(mime_type like 'image/%' and (width is null or height is null)) or \
(mime_type like 'video/%' and (width is null or height is null or bitrate is null or duration is null))")
.fetch_all(&self.pool)
.await?;
Ok(results)
}
pub async fn update_metadata(
&mut self,
id: &Vec<u8>,
width: Option<u32>,
height: Option<u32>,
duration: Option<f32>,
bitrate: Option<u32>,
) -> Result<()> {
sqlx::query("update uploads set width=?, height=?, duration=?, bitrate=? where id=?")
.bind(width)
.bind(height)
.bind(duration)
.bind(bitrate)
.bind(id)
.execute(&self.pool)
.await?;
Ok(())
}
}

View File

@ -1,54 +0,0 @@
use crate::db::Database;
use crate::filesystem::FileStore;
use log::{error, info, warn};
use tokio::sync::broadcast;
use tokio::task::JoinHandle;
#[cfg(feature = "media-compression")]
mod media_metadata;
#[cfg(feature = "payments")]
mod payments;
pub fn start_background_tasks(
db: Database,
file_store: FileStore,
shutdown_rx: broadcast::Receiver<()>,
#[cfg(feature = "payments")] client: Option<fedimint_tonic_lnd::Client>,
) -> Vec<JoinHandle<()>> {
let mut ret = vec![];
#[cfg(feature = "media-compression")]
{
let db = db.clone();
let rx = shutdown_rx.resubscribe();
ret.push(tokio::spawn(async move {
info!("Starting MediaMetadata background task");
let mut m = media_metadata::MediaMetadata::new(db, file_store.clone());
if let Err(e) = m.process(rx).await {
error!("MediaMetadata failed: {}", e);
} else {
info!("MediaMetadata background task completed");
}
}));
}
#[cfg(feature = "payments")]
{
if let Some(client) = client {
let db = db.clone();
let rx = shutdown_rx.resubscribe();
ret.push(tokio::spawn(async move {
info!("Starting PaymentsHandler background task");
let mut m = payments::PaymentsHandler::new(client, db);
if let Err(e) = m.process(rx).await {
error!("PaymentsHandler failed: {}", e);
} else {
info!("PaymentsHandler background task completed");
}
}));
} else {
warn!("Not starting PaymentsHandler, configuration missing")
}
}
ret
}

View File

@ -1,71 +0,0 @@
use crate::db::Database;
use anyhow::Result;
use fedimint_tonic_lnd::lnrpc::invoice::InvoiceState;
use fedimint_tonic_lnd::lnrpc::InvoiceSubscription;
use fedimint_tonic_lnd::Client;
use log::{error, info};
use rocket::futures::StreamExt;
use sqlx::Row;
use tokio::sync::broadcast;
pub struct PaymentsHandler {
client: Client,
database: Database,
}
impl PaymentsHandler {
pub fn new(client: Client, database: Database) -> Self {
PaymentsHandler { client, database }
}
pub async fn process(&mut self, mut rx: broadcast::Receiver<()>) -> Result<()> {
let start_idx = self.database.get_last_settle_index().await?;
let mut invoices = self
.client
.lightning()
.subscribe_invoices(InvoiceSubscription {
add_index: 0,
settle_index: start_idx,
})
.await?;
info!("Starting invoice subscription from {}", start_idx);
let invoices = invoices.get_mut();
loop {
tokio::select! {
Ok(_) = rx.recv() => {
break;
}
Some(Ok(msg)) = invoices.next() => {
if msg.state == InvoiceState::Settled as i32 {
if let Ok(Some(mut p)) = self.database.get_payment(&msg.r_hash).await {
p.settle_index = Some(msg.settle_index);
p.is_paid = true;
match self.database.complete_payment(&p).await {
Ok(()) => info!(
"Successfully completed payment: {}",
hex::encode(&msg.r_hash)
),
Err(e) => error!("Failed to complete payment: {}", e),
}
}
}
}
}
}
Ok(())
}
}
impl Database {
async fn get_last_settle_index(&self) -> Result<u64> {
Ok(
sqlx::query("select max(settle_index) from payments where is_paid = true")
.fetch_one(&self.pool)
.await?
.try_get(0)
.unwrap_or(0),
)
}
}

View File

@ -3,8 +3,6 @@ use std::net::{IpAddr, SocketAddr};
use anyhow::Error;
use clap::Parser;
use config::Config;
#[cfg(feature = "payments")]
use fedimint_tonic_lnd::lnrpc::GetInfoRequest;
use log::{error, info};
use rocket::config::Ident;
use rocket::data::{ByteUnit, Limits};
@ -14,14 +12,12 @@ use rocket::shield::Shield;
use route96::analytics::plausible::PlausibleAnalytics;
#[cfg(feature = "analytics")]
use route96::analytics::AnalyticsFairing;
use route96::background::start_background_tasks;
use route96::cors::CORS;
use route96::db::Database;
use route96::filesystem::FileStore;
use route96::routes;
use route96::routes::{get_blob, head_blob, root};
use route96::settings::Settings;
use tokio::sync::broadcast;
#[derive(Parser, Debug)]
#[command(version, about)]
@ -67,22 +63,15 @@ async fn main() -> Result<(), Error> {
.limit("form", upload_limit);
config.ident = Ident::try_new("route96").unwrap();
let fs = FileStore::new(settings.clone());
let mut rocket = rocket::Rocket::custom(config)
.manage(fs.clone())
.manage(FileStore::new(settings.clone()))
.manage(settings.clone())
.manage(db.clone())
.attach(CORS)
.attach(Shield::new()) // disable
.mount(
"/",
routes![
root,
get_blob,
head_blob,
routes::void_cat_redirect,
routes::void_cat_redirect_head
],
routes![root, get_blob, head_blob, routes::void_cat_redirect],
)
.mount("/admin", routes::admin_routes());
@ -100,51 +89,10 @@ async fn main() -> Result<(), Error> {
{
rocket = rocket.mount("/", routes::nip96_routes());
}
#[cfg(feature = "media-compression")]
{
rocket = rocket.mount("/", routes![routes::get_blob_thumb]);
}
#[cfg(feature = "payments")]
let lnd = {
if let Some(lnd) = settings.payments.as_ref().map(|p| &p.lnd) {
let lnd = fedimint_tonic_lnd::connect(
lnd.endpoint.clone(),
lnd.tls.clone(),
lnd.macaroon.clone(),
)
.await?;
let info = {
let mut lnd = lnd.clone();
lnd.lightning().get_info(GetInfoRequest::default()).await?
};
info!(
"LND connected: {} v{}",
info.get_ref().alias,
info.get_ref().version
);
rocket = rocket
.manage(lnd.clone())
.mount("/", routes::payment::routes());
Some(lnd)
} else {
None
}
};
let (shutdown_tx, shutdown_rx) = broadcast::channel(1);
let jh = start_background_tasks(db, fs, shutdown_rx, lnd);
if let Err(e) = rocket.launch().await {
error!("Rocker error {}", e);
Err(Error::from(e))
} else {
Ok(())
}
shutdown_tx
.send(())
.expect("Failed to send shutdown signal");
for j in jh {
j.await?;
}
Ok(())
}

View File

@ -1,228 +0,0 @@
use anyhow::{Context, Error, Result};
use clap::{Parser, Subcommand};
use config::Config;
use indicatif::{ProgressBar, ProgressStyle};
use log::{error, info};
use route96::db::{Database, FileUpload};
use route96::filesystem::{FileStore, FileSystemResult};
use route96::processing::probe_file;
use route96::settings::Settings;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::time::SystemTime;
use tokio::sync::Semaphore;
#[derive(Parser, Debug)]
#[command(version, about)]
struct Args {
#[arg(long)]
pub config: Option<String>,
#[clap(subcommand)]
pub command: Commands,
}
#[derive(Debug, Subcommand)]
enum Commands {
/// Check file hash matches filename / path
Check {
#[arg(long)]
delete: Option<bool>,
},
/// Import a directory into the filesystem
/// (does NOT import files into the database, use database-import command for that)
Import {
#[arg(long)]
from: PathBuf,
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
probe_media: Option<bool>,
},
/// Import files from filesystem into database
DatabaseImport {
/// Don't actually import data and just print which files WOULD be imported
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
dry_run: Option<bool>,
},
}
#[tokio::main]
async fn main() -> Result<(), Error> {
if std::env::var("RUST_LOG").is_err() {
std::env::set_var("RUST_LOG", "info");
}
env_logger::init();
let args: Args = Args::parse();
let builder = Config::builder()
.add_source(config::File::with_name(if let Some(ref c) = args.config {
c.as_str()
} else {
"config.yaml"
}))
.add_source(config::Environment::with_prefix("APP"))
.build()?;
let settings: Settings = builder.try_deserialize()?;
match args.command {
Commands::Check { delete } => {
info!("Checking files in: {}", settings.storage_dir);
let fs = FileStore::new(settings.clone());
iter_files(&fs.storage_dir(), 4, |entry, p| {
let p = p.clone();
Box::pin(async move {
let id = if let Some(i) = id_from_path(&entry) {
i
} else {
p.set_message(format!("Skipping invalid file: {}", &entry.display()));
return Ok(());
};
let hash = FileStore::hash_file(&entry).await?;
if hash != id {
if delete.unwrap_or(false) {
p.set_message(format!("Deleting corrupt file: {}", &entry.display()));
tokio::fs::remove_file(&entry).await?;
} else {
p.set_message(format!("File is corrupted: {}", &entry.display()));
}
}
Ok(())
})
})
.await?;
}
Commands::Import { from, probe_media } => {
let fs = FileStore::new(settings.clone());
let db = Database::new(&settings.database).await?;
db.migrate().await?;
info!("Importing from: {}", fs.storage_dir().display());
iter_files(&from, 4, |entry, p| {
let fs = fs.clone();
let p = p.clone();
Box::pin(async move {
let mime = infer::get_from_path(&entry)?
.map(|m| m.mime_type())
.unwrap_or("application/octet-stream");
// test media is not corrupt
if probe_media.unwrap_or(true)
&& (mime.starts_with("image/") || mime.starts_with("video/"))
&& probe_file(&entry).is_err()
{
p.set_message(format!("Skipping media invalid file: {}", &entry.display()));
return Ok(());
}
let file = tokio::fs::File::open(&entry).await?;
let dst = fs.put(file, mime, false).await?;
match dst {
FileSystemResult::AlreadyExists(_) => {
p.set_message(format!("Duplicate file: {}", &entry.display()));
}
FileSystemResult::NewFile(_) => {
p.set_message(format!("Imported: {}", &entry.display()));
}
}
Ok(())
})
})
.await?;
}
Commands::DatabaseImport { dry_run } => {
let fs = FileStore::new(settings.clone());
let db = Database::new(&settings.database).await?;
db.migrate().await?;
info!("Importing to DB from: {}", fs.storage_dir().display());
iter_files(&fs.storage_dir(), 4, |entry, p| {
let db = db.clone();
let p = p.clone();
Box::pin(async move {
let id = if let Some(i) = id_from_path(&entry) {
i
} else {
p.set_message(format!("Skipping invalid file: {}", &entry.display()));
return Ok(());
};
let u = db.get_file(&id).await.context("db get_file")?;
if u.is_none() {
if !dry_run.unwrap_or(false) {
p.set_message(format!("Importing file: {}", &entry.display()));
let mime = infer::get_from_path(&entry)
.context("infer")?
.map(|m| m.mime_type())
.unwrap_or("application/octet-stream")
.to_string();
let meta = entry.metadata().context("file metadata")?;
let entry = FileUpload {
id,
name: None,
size: meta.len(),
mime_type: mime,
created: meta.created().unwrap_or(SystemTime::now()).into(),
width: None,
height: None,
blur_hash: None,
alt: None,
duration: None,
bitrate: None,
};
db.add_file(&entry, None).await.context("db add_file")?;
} else {
p.set_message(format!(
"[DRY-RUN] Importing file: {}",
&entry.display()
));
}
}
Ok(())
})
})
.await?;
}
}
Ok(())
}
fn id_from_path(path: &Path) -> Option<Vec<u8>> {
hex::decode(path.file_name()?.to_str()?).ok()
}
async fn iter_files<F>(p: &Path, threads: usize, mut op: F) -> Result<()>
where
F: FnMut(PathBuf, ProgressBar) -> Pin<Box<dyn Future<Output = Result<()>> + Send>>,
{
let semaphore = Arc::new(Semaphore::new(threads));
info!("Scanning files: {}", p.display());
let entries = walkdir::WalkDir::new(p);
let dir = entries
.into_iter()
.filter_map(Result::ok)
.filter(|e| e.file_type().is_file())
.collect::<Vec<_>>();
let p = ProgressBar::new(dir.len() as u64).with_style(ProgressStyle::with_template(
"{spinner} [{pos}/{len}] {msg}",
)?);
let mut all_tasks = vec![];
for entry in dir {
let _lock = semaphore.clone().acquire_owned().await?;
p.inc(1);
let fut = op(entry.path().to_path_buf(), p.clone());
all_tasks.push(tokio::spawn(async move {
if let Err(e) = fut.await {
error!("Error processing file: {} {}", entry.path().display(), e);
}
drop(_lock);
}));
}
for task in all_tasks {
task.await?;
}
p.finish_with_message("Done!");
Ok(())
}

View File

@ -24,7 +24,7 @@ struct ProgramArgs {
#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
env_logger::init();
pretty_env_logger::init();
let args: ProgramArgs = ProgramArgs::parse();

View File

@ -34,7 +34,7 @@ struct Args {
#[tokio::main]
async fn main() -> Result<(), Error> {
env_logger::init();
pretty_env_logger::init();
let builder = Config::builder()
.add_source(config::File::with_name("config.yaml"))
@ -103,7 +103,7 @@ async fn migrate_file(
let src_path = PathBuf::new()
.join(&args.data_path)
.join(VoidFile::map_to_path(&f.id));
let dst_path = fs.get(&id_vec);
let dst_path = fs.map_path(&id_vec);
if src_path.exists() && !dst_path.exists() {
info!(
"Copying file: {} from {} => {}",
@ -139,9 +139,7 @@ async fn migrate_file(
},
blur_hash: None,
alt: f.description.clone(),
duration: None,
bitrate: None,
};
db.add_file(&fu, Some(uid)).await?;
db.add_file(&fu, uid).await?;
Ok(())
}

112
src/db.rs
View File

@ -25,10 +25,6 @@ pub struct FileUpload {
pub blur_hash: Option<String>,
/// Alt text of the media
pub alt: Option<String>,
/// Duration of media in seconds
pub duration: Option<f32>,
/// Average bitrate in bits/s
pub bitrate: Option<u32>,
#[sqlx(skip)]
#[cfg(feature = "labels")]
@ -47,8 +43,6 @@ impl From<&NewFileResult> for FileUpload {
height: value.height,
blur_hash: value.blur_hash.clone(),
alt: None,
duration: value.duration,
bitrate: value.bitrate,
#[cfg(feature = "labels")]
labels: value.labels.clone(),
}
@ -61,10 +55,6 @@ pub struct User {
pub pubkey: Vec<u8>,
pub created: DateTime<Utc>,
pub is_admin: bool,
#[cfg(feature = "payments")]
pub paid_until: Option<DateTime<Utc>>,
#[cfg(feature = "payments")]
pub paid_size: u64,
}
#[cfg(feature = "labels")]
@ -94,20 +84,6 @@ pub struct UserStats {
pub total_size: u64,
}
#[cfg(feature = "payments")]
#[derive(Clone, FromRow, Serialize)]
pub struct Payment {
pub payment_hash: Vec<u8>,
pub user_id: u64,
pub created: DateTime<Utc>,
pub amount: u64,
pub is_paid: bool,
pub days_value: u64,
pub size_value: u64,
pub settle_index: Option<u64>,
pub rate: Option<f32>,
}
#[derive(Clone)]
pub struct Database {
pub(crate) pool: sqlx::pool::Pool<sqlx::mysql::MySql>,
@ -169,7 +145,7 @@ impl Database {
pub async fn add_file(&self, file: &FileUpload, user_id: u64) -> Result<(), Error> {
let mut tx = self.pool.begin().await?;
let q = sqlx::query("insert ignore into \
uploads(id,name,size,mime_type,blur_hash,width,height,alt,created,duration,bitrate) values(?,?,?,?,?,?,?,?,?,?,?)")
uploads(id,name,size,mime_type,blur_hash,width,height,alt,created) values(?,?,?,?,?,?,?,?,?)")
.bind(&file.id)
.bind(&file.name)
.bind(file.size)
@ -178,9 +154,7 @@ impl Database {
.bind(file.width)
.bind(file.height)
.bind(&file.alt)
.bind(file.created)
.bind(file.duration)
.bind(file.bitrate);
.bind(file.created);
tx.execute(q).await?;
let q2 = sqlx::query("insert ignore into user_uploads(file,user_id) values(?,?)")
@ -288,85 +262,3 @@ impl Database {
Ok((results, count))
}
}
#[cfg(feature = "payments")]
impl Database {
pub async fn insert_payment(&self, payment: &Payment) -> Result<(), Error> {
sqlx::query("insert into payments(payment_hash,user_id,amount,days_value,size_value,rate) values(?,?,?,?,?,?)")
.bind(&payment.payment_hash)
.bind(payment.user_id)
.bind(payment.amount)
.bind(payment.days_value)
.bind(payment.size_value)
.bind(payment.rate)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn get_payment(&self, payment_hash: &Vec<u8>) -> Result<Option<Payment>, Error> {
sqlx::query_as("select * from payments where payment_hash = ?")
.bind(payment_hash)
.fetch_optional(&self.pool)
.await
}
pub async fn get_user_payments(&self, uid: u64) -> Result<Vec<Payment>, Error> {
sqlx::query_as("select * from payments where user_id = ?")
.bind(uid)
.fetch_all(&self.pool)
.await
}
pub async fn complete_payment(&self, payment: &Payment) -> Result<(), Error> {
let mut tx = self.pool.begin().await?;
sqlx::query("update payments set is_paid = true, settle_index = ? where payment_hash = ?")
.bind(payment.settle_index)
.bind(&payment.payment_hash)
.execute(&mut *tx)
.await?;
// TODO: check space is not downgraded
sqlx::query("update users set paid_until = TIMESTAMPADD(DAY, ?, IFNULL(paid_until, current_timestamp)), paid_size = ? where id = ?")
.bind(payment.days_value)
.bind(payment.size_value)
.bind(payment.user_id)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
/// Check if user has sufficient quota for an upload
pub async fn check_user_quota(&self, pubkey: &Vec<u8>, upload_size: u64, free_quota_bytes: u64) -> Result<bool, Error> {
// Get or create user
let user_id = self.upsert_user(pubkey).await?;
// Get user's current storage usage
let user_stats = self.get_user_stats(user_id).await.unwrap_or(UserStats {
file_count: 0,
total_size: 0
});
// Get user's paid quota
let user = self.get_user(pubkey).await?;
let (paid_size, paid_until) = (user.paid_size, user.paid_until);
// Calculate total available quota
let mut available_quota = free_quota_bytes;
// Add paid quota if still valid
if let Some(paid_until) = paid_until {
if paid_until > chrono::Utc::now() {
available_quota += paid_size;
}
}
// Check if upload would exceed quota
Ok(user_stats.total_size + upload_size <= available_quota)
}
}

View File

@ -1,6 +1,5 @@
#[cfg(feature = "labels")]
use crate::db::FileLabel;
#[cfg(feature = "labels")]
use crate::processing::labeling::label_frame;
#[cfg(feature = "media-compression")]
@ -10,12 +9,11 @@ use anyhow::Error;
use anyhow::Result;
#[cfg(feature = "media-compression")]
use ffmpeg_rs_raw::DemuxerInfo;
use ffmpeg_rs_raw::StreamInfo;
#[cfg(feature = "media-compression")]
use rocket::form::validate::Contains;
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncReadExt};
use uuid::Uuid;
@ -38,13 +36,10 @@ pub struct NewFileResult {
pub width: Option<u32>,
pub height: Option<u32>,
pub blur_hash: Option<String>,
pub duration: Option<f32>,
pub bitrate: Option<u32>,
#[cfg(feature = "labels")]
pub labels: Vec<FileLabel>,
}
#[derive(Clone)]
pub struct FileStore {
settings: Settings,
}
@ -62,7 +57,7 @@ impl FileStore {
/// Store a new file
pub async fn put<'r, S>(
&self,
path: S,
stream: S,
mime_type: &str,
compress: bool,
) -> Result<FileSystemResult>
@ -70,7 +65,7 @@ impl FileStore {
S: AsyncRead + Unpin + 'r,
{
// store file in temp path and hash the file
let (temp_file, size, hash) = self.store_hash_temp_file(path).await?;
let (temp_file, size, hash) = self.store_hash_temp_file(stream).await?;
let dst_path = self.map_path(&hash);
// check if file hash already exists
@ -79,7 +74,7 @@ impl FileStore {
return Ok(FileSystemResult::AlreadyExists(hash));
}
let mut res = if compress && crate::can_compress(mime_type) {
let mut res = if compress {
#[cfg(feature = "media-compression")]
{
let res = match self.compress_file(&temp_file, mime_type).await {
@ -97,30 +92,20 @@ impl FileStore {
anyhow::bail!("Compression not supported!");
}
} else {
let (width, height, mime_type, duration, bitrate) = {
let (width, height, mime_type) = {
#[cfg(feature = "media-compression")]
{
let probe = probe_file(&temp_file).ok();
let v_stream = probe.as_ref().and_then(|p| p.best_video());
let mime = Self::hack_mime_type(mime_type, &probe, &v_stream, &temp_file);
let mime = Self::hack_mime_type(mime_type, &probe, &temp_file);
(
v_stream.map(|v| v.width as u32),
v_stream.map(|v| v.height as u32),
mime,
probe
.as_ref()
.map(|p| if p.duration < 0. { 0.0 } else { p.duration }),
probe.as_ref().map(|p| p.bitrate as u32),
)
}
#[cfg(not(feature = "media-compression"))]
(
None,
None,
Self::infer_mime_type(mime_type, &temp_file),
None,
None,
)
(None, None, Self::infer_mime_type(mime_type, &temp_file))
};
NewFileResult {
path: temp_file,
@ -130,8 +115,6 @@ impl FileStore {
width,
height,
blur_hash: None,
duration,
bitrate,
}
};
@ -153,50 +136,18 @@ impl FileStore {
#[cfg(feature = "media-compression")]
/// Try to replace the mime-type when unknown using ffmpeg probe result
fn hack_mime_type(
mime_type: &str,
p: &Option<DemuxerInfo>,
stream: &Option<&StreamInfo>,
out_path: &PathBuf,
) -> String {
fn hack_mime_type(mime_type: &str, p: &Option<DemuxerInfo>, out_path: &PathBuf) -> String {
if let Some(p) = p {
let mime = if p.format.contains("mp4") {
Some("video/mp4")
if p.format.contains("mp4") {
return "video/mp4".to_string();
} else if p.format.contains("webp") {
Some("image/webp")
return "image/webp".to_string();
} else if p.format.contains("jpeg") {
Some("image/jpeg")
return "image/jpeg".to_string();
} else if p.format.contains("png") {
Some("image/png")
return "image/png".to_string();
} else if p.format.contains("gif") {
Some("image/gif")
} else {
None
};
let codec = if let Some(s) = stream {
match s.codec {
27 => Some("avc1".to_owned()), //AV_CODEC_ID_H264
173 => Some("hvc1".to_owned()), //AV_CODEC_ID_HEVC
86016 => Some("mp4a.40.33".to_string()), //AV_CODEC_ID_MP2
86017 => Some("mp4a.40.34".to_string()), //AV_CODEC_ID_MP3
86018 => Some("mp4a.40.2".to_string()), //AV_CODEC_ID_AAC
86019 => Some("ac-3".to_string()), //AV_CODEC_ID_AC3
86056 => Some("ec-3".to_string()), //AV_CODEC_ID_EAC3
_ => None,
}
} else {
None
};
if let Some(m) = mime {
return format!(
"{}{}",
m,
if let Some(c) = codec {
format!("; codecs=\"{}\"", c)
} else {
"".to_owned()
}
);
return "image/gif".to_string();
}
}
@ -213,8 +164,7 @@ impl FileStore {
}
}
#[cfg(feature = "media-compression")]
async fn compress_file(&self, input: &Path, mime_type: &str) -> Result<NewFileResult> {
async fn compress_file(&self, input: &PathBuf, mime_type: &str) -> Result<NewFileResult> {
let compressed_result = compress_file(input, mime_type, &self.temp_dir())?;
#[cfg(feature = "labels")]
let labels = if let Some(mp) = &self.settings.vit_model {
@ -244,8 +194,6 @@ impl FileStore {
height: Some(compressed_result.height as u32),
blur_hash: None,
mime_type: compressed_result.mime_type,
duration: Some(compressed_result.duration),
bitrate: Some(compressed_result.bitrate),
#[cfg(feature = "labels")]
labels,
})
@ -266,7 +214,7 @@ impl FileStore {
Ok((out_path, n, hash))
}
pub async fn hash_file(p: &Path) -> Result<Vec<u8>, Error> {
async fn hash_file(p: &PathBuf) -> Result<Vec<u8>, Error> {
let mut file = File::open(p).await?;
let mut hasher = Sha256::new();
let mut buf = [0; 4096];
@ -281,7 +229,7 @@ impl FileStore {
Ok(res.to_vec())
}
fn map_path(&self, id: &Vec<u8>) -> PathBuf {
pub fn map_path(&self, id: &Vec<u8>) -> PathBuf {
let id = hex::encode(id);
self.storage_dir().join(&id[0..2]).join(&id[2..4]).join(id)
}

View File

@ -1,18 +1,13 @@
#[cfg(feature = "analytics")]
pub mod analytics;
pub mod auth;
pub mod background;
pub mod cors;
pub mod db;
pub mod filesystem;
#[cfg(feature = "payments")]
pub mod payments;
#[cfg(feature = "media-compression")]
pub mod processing;
pub mod routes;
pub mod settings;
#[cfg(any(feature = "void-cat-redirects", feature = "bin-void-cat-migrate"))]
pub mod void_db;
pub mod void_file;
pub fn can_compress(mime_type: &str) -> bool {
mime_type.starts_with("image/")
}

View File

@ -1,53 +0,0 @@
use serde::{Deserialize, Serialize};
use std::fmt::{Display, Formatter};
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaymentAmount {
pub currency: Currency,
pub amount: f32,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Currency {
BTC,
USD,
EUR,
GBP,
JPY,
CAD,
AUD,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PaymentUnit {
GBSpace,
GBEgress,
}
impl PaymentUnit {
/// Get the total size from a number of units
pub fn to_size(&self, units: f32) -> u64 {
(1000f32 * 1000f32 * 1000f32 * units) as u64
}
}
impl Display for PaymentUnit {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
PaymentUnit::GBSpace => write!(f, "GB Space"),
PaymentUnit::GBEgress => write!(f, "GB Egress"),
}
}
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PaymentInterval {
Day(u16),
Month(u16),
Year(u16),
}

View File

@ -1,9 +1,8 @@
use std::path::{Path, PathBuf};
use anyhow::{bail, Error, Result};
use ffmpeg_rs_raw::ffmpeg_sys_the_third::AVPixelFormat::AV_PIX_FMT_YUV420P;
use ffmpeg_rs_raw::ffmpeg_sys_the_third::{av_frame_free, av_packet_free, AVFrame};
use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo, Encoder, Scaler, StreamType, Transcoder};
use std::path::{Path, PathBuf};
use std::ptr;
use ffmpeg_rs_raw::{Demuxer, DemuxerInfo, Encoder, StreamType, Transcoder};
use uuid::Uuid;
#[cfg(feature = "labels")]
@ -22,7 +21,7 @@ impl WebpProcessor {
Self
}
pub fn compress(
pub fn process_file(
&mut self,
input: &Path,
mime_type: &str,
@ -66,65 +65,9 @@ impl WebpProcessor {
mime_type: "image/webp".to_string(),
width: image_stream.width,
height: image_stream.height,
duration: if probe.duration < 0. {
0.
} else {
probe.duration
},
bitrate: probe.bitrate as u32,
})
}
}
pub fn thumbnail(&mut self, input: &Path, out_path: &Path) -> Result<()> {
use ffmpeg_rs_raw::ffmpeg_sys_the_third::AVCodecID::AV_CODEC_ID_WEBP;
unsafe {
let mut input = Demuxer::new(input.to_str().unwrap())?;
let probe = input.probe_input()?;
let image_stream = probe
.streams
.iter()
.find(|c| c.stream_type == StreamType::Video)
.ok_or(Error::msg("No image found, cant compress"))?;
let w = 512u16;
let scale = w as f32 / image_stream.width as f32;
let h = (image_stream.height as f32 * scale) as u16;
let enc = Encoder::new(AV_CODEC_ID_WEBP)?
.with_height(h as i32)
.with_width(w as i32)
.with_pix_fmt(AV_PIX_FMT_YUV420P)
.with_framerate(1.0)?
.open(None)?;
let mut sws = Scaler::new();
let mut decoder = Decoder::new();
decoder.setup_decoder(image_stream, None)?;
while let Ok((mut pkt, _stream)) = input.get_packet() {
let mut frame_save: *mut AVFrame = ptr::null_mut();
for (mut frame, _stream) in decoder.decode_pkt(pkt)? {
if frame_save.is_null() {
frame_save = sws.process_frame(frame, w, h, AV_PIX_FMT_YUV420P)?;
}
av_frame_free(&mut frame);
}
av_packet_free(&mut pkt);
if !frame_save.is_null() {
enc.save_picture(frame_save, out_path.to_str().unwrap())?;
av_frame_free(&mut frame_save);
return Ok(());
}
}
Ok(())
}
}
}
pub struct NewFileProcessorResult {
@ -132,27 +75,29 @@ pub struct NewFileProcessorResult {
pub mime_type: String,
pub width: usize,
pub height: usize,
pub duration: f32,
pub bitrate: u32,
}
pub fn can_compress(mime_type: &str) -> bool {
mime_type.starts_with("image/")
}
pub fn compress_file(
path: &Path,
stream: &Path,
mime_type: &str,
out_dir: &Path,
) -> Result<NewFileProcessorResult, Error> {
if !crate::can_compress(mime_type) {
if !can_compress(mime_type) {
bail!("MIME type not supported");
}
if mime_type.starts_with("image/") {
let mut proc = WebpProcessor::new();
return proc.compress(path, mime_type, out_dir);
return proc.process_file(stream, mime_type, out_dir);
}
bail!("No media processor")
}
pub fn probe_file(path: &Path) -> Result<DemuxerInfo> {
let mut demuxer = Demuxer::new(path.to_str().unwrap())?;
pub fn probe_file(stream: &Path) -> Result<DemuxerInfo> {
let mut demuxer = Demuxer::new(stream.to_str().unwrap())?;
unsafe { demuxer.probe_input() }
}

View File

@ -1,11 +1,11 @@
use crate::auth::nip98::Nip98Auth;
use crate::db::{Database, FileUpload, User};
use crate::db::{Database, FileUpload};
use crate::routes::{Nip94Event, PagedResult};
use crate::settings::Settings;
use rocket::serde::json::Json;
use rocket::serde::Serialize;
use rocket::{routes, Responder, Route, State};
use sqlx::{Error, QueryBuilder, Row};
use sqlx::{Error, Row};
pub fn admin_routes() -> Vec<Route> {
routes![admin_list_files, admin_get_self]
@ -53,25 +53,10 @@ pub struct SelfUser {
pub is_admin: bool,
pub file_count: u64,
pub total_size: u64,
#[cfg(feature = "payments")]
pub paid_until: u64,
#[cfg(feature = "payments")]
pub quota: u64,
#[cfg(feature = "payments")]
pub free_quota: u64,
#[cfg(feature = "payments")]
pub total_available_quota: u64,
}
#[derive(Serialize)]
pub struct AdminNip94File {
#[serde(flatten)]
pub inner: Nip94Event,
pub uploader: Vec<String>,
}
#[rocket::get("/self")]
async fn admin_get_self(auth: Nip98Auth, db: &State<Database>, settings: &State<Settings>) -> AdminResponse<SelfUser> {
async fn admin_get_self(auth: Nip98Auth, db: &State<Database>) -> AdminResponse<SelfUser> {
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
match db.get_user(&pubkey_vec).await {
Ok(user) => {
@ -81,55 +66,24 @@ async fn admin_get_self(auth: Nip98Auth, db: &State<Database>, settings: &State<
return AdminResponse::error(&format!("Failed to load user stats: {}", e))
}
};
#[cfg(feature = "payments")]
let (free_quota, total_available_quota) = {
let free_quota = settings.payments.as_ref()
.and_then(|p| p.free_quota_bytes)
.unwrap_or(104857600);
let mut total_available = free_quota;
// Add paid quota if still valid
if let Some(paid_until) = &user.paid_until {
if *paid_until > chrono::Utc::now() {
total_available += user.paid_size;
}
}
(free_quota, total_available)
};
AdminResponse::success(SelfUser {
is_admin: user.is_admin,
file_count: s.file_count,
total_size: s.total_size,
#[cfg(feature = "payments")]
paid_until: if let Some(u) = &user.paid_until {
u.timestamp() as u64
} else {
0
},
#[cfg(feature = "payments")]
quota: user.paid_size,
#[cfg(feature = "payments")]
free_quota,
#[cfg(feature = "payments")]
total_available_quota,
})
}
Err(_) => AdminResponse::error("User not found"),
}
}
#[rocket::get("/files?<page>&<count>&<mime_type>")]
#[rocket::get("/files?<page>&<count>")]
async fn admin_list_files(
auth: Nip98Auth,
page: u32,
count: u32,
mime_type: Option<String>,
db: &State<Database>,
settings: &State<Settings>,
) -> AdminResponse<PagedResult<AdminNip94File>> {
) -> AdminResponse<PagedResult<Nip94Event>> {
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
let server_count = count.clamp(1, 5_000);
@ -141,20 +95,14 @@ async fn admin_list_files(
if !user.is_admin {
return AdminResponse::error("User is not an admin");
}
match db
.list_all_files(page * server_count, server_count, mime_type)
.await
{
match db.list_all_files(page * server_count, server_count).await {
Ok((files, count)) => AdminResponse::success(PagedResult {
count: files.len() as u32,
page,
total: count as u32,
files: files
.into_iter()
.map(|f| AdminNip94File {
inner: Nip94Event::from_upload(settings, &f.0),
uploader: f.1.into_iter().map(|u| hex::encode(&u.pubkey)).collect(),
})
.iter()
.map(|f| Nip94Event::from_upload(settings, f))
.collect(),
}),
Err(e) => AdminResponse::error(&format!("Could not list files: {}", e)),
@ -166,29 +114,21 @@ impl Database {
&self,
offset: u32,
limit: u32,
mime_type: Option<String>,
) -> Result<(Vec<(FileUpload, Vec<User>)>, i64), Error> {
let mut q = QueryBuilder::new("select u.* from uploads u ");
if let Some(m) = mime_type {
q.push("where u.mime_type = ");
q.push_bind(m);
}
q.push(" order by u.created desc limit ");
q.push_bind(limit);
q.push(" offset ");
q.push_bind(offset);
let results: Vec<FileUpload> = q.build_query_as().fetch_all(&self.pool).await?;
) -> Result<(Vec<FileUpload>, i64), Error> {
let results: Vec<FileUpload> = sqlx::query_as(
"select u.* \
from uploads u \
order by u.created desc \
limit ? offset ?",
)
.bind(limit)
.bind(offset)
.fetch_all(&self.pool)
.await?;
let count: i64 = sqlx::query("select count(u.id) from uploads u")
.fetch_one(&self.pool)
.await?
.try_get(0)?;
let mut res = Vec::with_capacity(results.len());
for upload in results.into_iter() {
let upd = self.get_file_owners(&upload.id).await?;
res.push((upload, upd));
}
Ok((res, count))
Ok((results, count))
}
}

View File

@ -5,7 +5,7 @@ use crate::routes::{delete_file, Nip94Event};
use crate::settings::Settings;
use log::error;
use nostr::prelude::hex;
use nostr::{Alphabet, SingleLetterTag, TagKind};
use nostr::{Alphabet, Event, SingleLetterTag, TagKind};
use rocket::data::ByteUnit;
use rocket::futures::StreamExt;
use rocket::http::{Header, Status};
@ -13,6 +13,7 @@ use rocket::response::Responder;
use rocket::serde::json::Json;
use rocket::{routes, Data, Request, Response, Route, State};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use tokio::io::AsyncRead;
use tokio_util::io::StreamReader;
@ -26,7 +27,7 @@ pub struct BlobDescriptor {
pub mime_type: Option<String>,
pub created: u64,
#[serde(rename = "nip94", skip_serializing_if = "Option::is_none")]
pub nip94: Option<Vec<Vec<String>>>,
pub nip94: Option<HashMap<String, String>>,
}
impl BlobDescriptor {
@ -45,7 +46,14 @@ impl BlobDescriptor {
size: value.size,
mime_type: Some(value.mime_type.clone()),
created: value.created.timestamp() as u64,
nip94: Some(Nip94Event::from_upload(settings, value).tags),
nip94: Some(
Nip94Event::from_upload(settings, value)
.tags
.iter()
.map(|r| (r[0].clone(), r[1].clone()))
.collect(),
),
}
}
}
@ -273,6 +281,15 @@ async fn upload_media(
process_upload("media", true, auth, fs, db, settings, data).await
}
#[rocket::put("/report", data = "<data>", format = "json")]
async fn report(
auth: BlossomAuth,
db: &State<Database>,
data: Json<Event>,
) -> Status {
}
fn check_head(auth: BlossomAuth, settings: &State<Settings>) -> BlossomHead {
if !check_method(&auth.event, "upload") {
return BlossomHead {
@ -354,21 +371,6 @@ async fn process_upload(
return e;
}
// check quota
#[cfg(feature = "payments")]
if let Some(upload_size) = size {
let free_quota = settings.payments.as_ref()
.and_then(|p| p.free_quota_bytes)
.unwrap_or(104857600); // Default to 100MB
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
match db.check_user_quota(&pubkey_vec, upload_size, free_quota).await {
Ok(false) => return BlossomResponse::error("Upload would exceed quota"),
Err(_) => return BlossomResponse::error("Failed to check quota"),
Ok(true) => {} // Quota check passed
}
}
process_stream(
data.open(ByteUnit::Byte(settings.max_upload_bytes)),
&auth

View File

@ -1,7 +1,5 @@
use crate::db::{Database, FileUpload};
use crate::filesystem::FileStore;
#[cfg(feature = "media-compression")]
use crate::processing::WebpProcessor;
pub use crate::routes::admin::admin_routes;
#[cfg(feature = "blossom")]
pub use crate::routes::blossom::blossom_routes;
@ -9,10 +7,8 @@ pub use crate::routes::blossom::blossom_routes;
pub use crate::routes::nip96::nip96_routes;
use crate::settings::Settings;
use crate::void_file::VoidFile;
use anyhow::{Error, Result};
use http_range_header::{
parse_range_header, EndPosition, StartPosition, SyntacticallyCorrectRange,
};
use anyhow::Error;
use http_range_header::{parse_range_header, EndPosition, StartPosition};
use log::{debug, warn};
use nostr::Event;
use rocket::fs::NamedFile;
@ -20,7 +16,6 @@ use rocket::http::{ContentType, Header, Status};
use rocket::response::Responder;
use rocket::serde::Serialize;
use rocket::{Request, Response, State};
use std::env::temp_dir;
use std::io::SeekFrom;
use std::ops::Range;
use std::pin::{pin, Pin};
@ -61,40 +56,28 @@ struct PagedResult<T> {
impl Nip94Event {
pub fn from_upload(settings: &Settings, upload: &FileUpload) -> Self {
let hex_id = hex::encode(&upload.id);
let ext = if upload.mime_type != "application/octet-stream" {
mime2ext::mime2ext(&upload.mime_type)
} else {
None
};
let mut tags = vec![
vec![
"url".to_string(),
format!("{}/{}.{}", &settings.public_url, &hex_id, ext.unwrap_or("")),
format!(
"{}/{}{}",
&settings.public_url,
&hex_id,
mime2ext::mime2ext(&upload.mime_type)
.map(|m| format!(".{m}"))
.unwrap_or("".to_string())
),
],
vec!["x".to_string(), hex_id.clone()],
vec!["x".to_string(), hex_id],
vec!["m".to_string(), upload.mime_type.clone()],
vec!["size".to_string(), upload.size.to_string()],
];
if upload.mime_type.starts_with("image/") || upload.mime_type.starts_with("video/") {
tags.push(vec![
"thumb".to_string(),
format!("{}/thumb/{}.webp", &settings.public_url, &hex_id),
]);
}
if let Some(bh) = &upload.blur_hash {
tags.push(vec!["blurhash".to_string(), bh.clone()]);
}
if let (Some(w), Some(h)) = (upload.width, upload.height) {
tags.push(vec!["dim".to_string(), format!("{}x{}", w, h)])
}
if let Some(d) = &upload.duration {
tags.push(vec!["duration".to_string(), d.to_string()]);
}
if let Some(b) = &upload.bitrate {
tags.push(vec!["bitrate".to_string(), b.to_string()]);
}
#[cfg(feature = "labels")]
for l in &upload.labels {
let val = if l.label.contains(',') {
@ -121,47 +104,18 @@ struct RangeBody {
range_end: u64,
current_offset: u64,
poll_complete: bool,
file_size: u64,
}
const MAX_UNBOUNDED_RANGE: u64 = 1024 * 1024;
impl RangeBody {
pub fn new(file: File, file_size: u64, range: Range<u64>) -> Self {
pub fn new(file: File, range: Range<u64>) -> Self {
Self {
file,
file_size,
range_start: range.start,
range_end: range.end,
current_offset: 0,
poll_complete: false,
}
}
pub fn get_range(file_size: u64, header: &SyntacticallyCorrectRange) -> Range<u64> {
let range_start = match header.start {
StartPosition::Index(i) => i,
StartPosition::FromLast(i) => file_size.saturating_sub(i),
};
let range_end = match header.end {
EndPosition::Index(i) => i,
EndPosition::LastByte => (file_size - 1).min(range_start + MAX_UNBOUNDED_RANGE),
};
range_start..range_end
}
pub fn get_headers(&self) -> Vec<Header<'static>> {
let r_len = (self.range_end - self.range_start) + 1;
vec![
Header::new("content-length", r_len.to_string()),
Header::new(
"content-range",
format!(
"bytes {}-{}/{}",
self.range_start, self.range_end, self.file_size
),
),
]
}
}
impl AsyncRead for RangeBody {
@ -171,7 +125,7 @@ impl AsyncRead for RangeBody {
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let range_start = self.range_start + self.current_offset;
let range_len = self.range_end.saturating_sub(range_start) + 1;
let range_len = self.range_end - range_start;
let bytes_to_read = buf.remaining().min(range_len as usize) as u64;
if bytes_to_read == 0 {
@ -216,13 +170,10 @@ impl AsyncRead for RangeBody {
impl<'r> Responder<'r, 'static> for FilePayload {
fn respond_to(self, request: &'r Request<'_>) -> rocket::response::Result<'static> {
let mut response = Response::new();
response.set_header(Header::new("cache-control", "max-age=31536000, immutable"));
// handle ranges
// only use range response for files > 1MiB
if self.info.size < MAX_UNBOUNDED_RANGE {
response.set_sized_body(None, self.file);
} else {
#[cfg(feature = "ranges")]
{
response.set_header(Header::new("accept-ranges", "bytes"));
if let Some(r) = request.headers().get("range").next() {
if let Ok(ranges) = parse_range_header(r) {
@ -230,22 +181,39 @@ impl<'r> Responder<'r, 'static> for FilePayload {
warn!("Multipart ranges are not supported, fallback to non-range request");
response.set_streamed_body(self.file);
} else {
const MAX_UNBOUNDED_RANGE: u64 = 1024 * 1024;
let single_range = ranges.ranges.first().unwrap();
let range = RangeBody::get_range(self.info.size, single_range);
let r_body = RangeBody::new(self.file, self.info.size, range.clone());
let range_start = match single_range.start {
StartPosition::Index(i) => i,
StartPosition::FromLast(i) => self.info.size - i,
};
let range_end = match single_range.end {
EndPosition::Index(i) => i,
EndPosition::LastByte => {
(range_start + MAX_UNBOUNDED_RANGE).min(self.info.size)
}
};
let r_len = range_end - range_start;
let r_body = RangeBody::new(self.file, range_start..range_end);
response.set_status(Status::PartialContent);
let headers = r_body.get_headers();
for h in headers {
response.set_header(h);
}
response.set_header(Header::new("content-length", r_len.to_string()));
response.set_header(Header::new(
"content-range",
format!("bytes {}-{}/{}", range_start, range_end - 1, self.info.size),
));
response.set_streamed_body(Box::pin(r_body));
}
}
} else {
response.set_sized_body(None, self.file);
response.set_streamed_body(self.file);
}
}
#[cfg(not(feature = "ranges"))]
{
response.set_streamed_body(self.file);
response.set_header(Header::new("content-length", self.info.size.to_string()));
}
if let Ok(ct) = ContentType::from_str(&self.info.mime_type) {
response.set_header(ct);
@ -384,64 +352,6 @@ pub async fn head_blob(sha256: &str, fs: &State<FileStore>) -> Status {
}
}
/// Generate thumbnail for image / video
#[cfg(feature = "media-compression")]
#[rocket::get("/thumb/<sha256>")]
pub async fn get_blob_thumb(
sha256: &str,
fs: &State<FileStore>,
db: &State<Database>,
) -> Result<FilePayload, Status> {
let sha256 = if sha256.contains(".") {
sha256.split('.').next().unwrap()
} else {
sha256
};
let id = if let Ok(i) = hex::decode(sha256) {
i
} else {
return Err(Status::NotFound);
};
if id.len() != 32 {
return Err(Status::NotFound);
}
let info = if let Ok(Some(info)) = db.get_file(&id).await {
info
} else {
return Err(Status::NotFound);
};
if !(info.mime_type.starts_with("image/") || info.mime_type.starts_with("video/")) {
return Err(Status::NotFound);
}
let file_path = fs.get(&id);
let mut thumb_file = temp_dir().join(format!("thumb_{}", sha256));
thumb_file.set_extension("webp");
if !thumb_file.exists() {
let mut p = WebpProcessor::new();
if p.thumbnail(&file_path, &thumb_file).is_err() {
return Err(Status::InternalServerError);
}
};
if let Ok(f) = File::open(&thumb_file).await {
Ok(FilePayload {
file: f,
info: FileUpload {
size: thumb_file.metadata().unwrap().len(),
mime_type: "image/webp".to_string(),
..info
},
})
} else {
Err(Status::NotFound)
}
}
/// Legacy URL redirect for void.cat uploads
#[rocket::get("/d/<id>")]
pub async fn void_cat_redirect(id: &str, settings: &State<Settings>) -> Option<NamedFile> {
@ -451,15 +361,10 @@ pub async fn void_cat_redirect(id: &str, settings: &State<Settings>) -> Option<N
id
};
if let Some(base) = &settings.void_cat_files {
let uuid = if let Ok(b58) = nostr::bitcoin::base58::decode(id) {
uuid::Uuid::from_slice_le(b58.as_slice())
} else {
uuid::Uuid::parse_str(id)
};
if uuid.is_err() {
return None;
}
let f = base.join(VoidFile::map_to_path(&uuid.unwrap()));
let uuid =
uuid::Uuid::from_slice_le(nostr::bitcoin::base58::decode(id).unwrap().as_slice())
.unwrap();
let f = base.join(VoidFile::map_to_path(&uuid));
debug!("Legacy file map: {} => {}", id, f.display());
if let Ok(f) = NamedFile::open(f).await {
Some(f)
@ -470,55 +375,3 @@ pub async fn void_cat_redirect(id: &str, settings: &State<Settings>) -> Option<N
None
}
}
#[rocket::head("/d/<id>")]
pub async fn void_cat_redirect_head(id: &str) -> VoidCatFile {
let id = if id.contains(".") {
id.split('.').next().unwrap()
} else {
id
};
let uuid =
uuid::Uuid::from_slice_le(nostr::bitcoin::base58::decode(id).unwrap().as_slice()).unwrap();
VoidCatFile {
status: Status::Ok,
uuid: Header::new("X-UUID", uuid.to_string()),
}
}
#[derive(Responder)]
pub struct VoidCatFile {
pub status: Status,
pub uuid: Header<'static>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_ranges() -> Result<()> {
let size = 16482469;
let req = parse_range_header("bytes=0-1023")?;
let r = RangeBody::get_range(size, req.ranges.first().unwrap());
assert_eq!(r.start, 0);
assert_eq!(r.end, 1023);
let req = parse_range_header("bytes=16482467-")?;
let r = RangeBody::get_range(size, req.ranges.first().unwrap());
assert_eq!(r.start, 16482467);
assert_eq!(r.end, 16482468);
let req = parse_range_header("bytes=-10")?;
let r = RangeBody::get_range(size, req.ranges.first().unwrap());
assert_eq!(r.start, 16482459);
assert_eq!(r.end, 16482468);
let req = parse_range_header("bytes=-16482470")?;
let r = RangeBody::get_range(size, req.ranges.first().unwrap());
assert_eq!(r.start, 0);
assert_eq!(r.end, MAX_UNBOUNDED_RANGE);
Ok(())
}
}

View File

@ -207,20 +207,6 @@ async fn upload(
}
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
// check quota
#[cfg(feature = "payments")]
{
let free_quota = settings.payments.as_ref()
.and_then(|p| p.free_quota_bytes)
.unwrap_or(104857600); // Default to 100MB
match db.check_user_quota(&pubkey_vec, form.size, free_quota).await {
Ok(false) => return Nip96Response::error("Upload would exceed quota"),
Err(_) => return Nip96Response::error("Failed to check quota"),
Ok(true) => {} // Quota check passed
}
}
let upload = match fs
.put(file, content_type, !form.no_transform.unwrap_or(false))
.await

View File

@ -1,131 +0,0 @@
use crate::auth::nip98::Nip98Auth;
use crate::db::{Database, Payment};
use crate::payments::{Currency, PaymentAmount, PaymentInterval, PaymentUnit};
use crate::settings::Settings;
use chrono::{Months, Utc};
use fedimint_tonic_lnd::lnrpc::Invoice;
use fedimint_tonic_lnd::Client;
use log::{error, info};
use rocket::serde::json::Json;
use rocket::{routes, Route, State};
use serde::{Deserialize, Serialize};
use std::ops::{Add, Deref};
pub fn routes() -> Vec<Route> {
routes![get_payment, req_payment]
}
#[derive(Deserialize, Serialize)]
struct PaymentInfo {
/// Billing quota metric
pub unit: PaymentUnit,
/// Amount of time to bill units (GB/mo, Gb Egress/day etc.)
pub interval: PaymentInterval,
/// Value amount of payment
pub cost: PaymentAmount,
}
#[derive(Deserialize, Serialize)]
struct PaymentRequest {
/// Number of units requested to make payment
pub units: f32,
/// Quantity of orders to make
pub quantity: u16,
}
#[derive(Deserialize, Serialize)]
struct PaymentResponse {
pub pr: String,
}
#[rocket::get("/payment")]
async fn get_payment(settings: &State<Settings>) -> Option<Json<PaymentInfo>> {
settings.payments.as_ref().map(|p| {
Json::from(PaymentInfo {
unit: p.unit.clone(),
interval: p.interval.clone(),
cost: p.cost.clone(),
})
})
}
#[rocket::post("/payment", data = "<req>", format = "json")]
async fn req_payment(
auth: Nip98Auth,
db: &State<Database>,
settings: &State<Settings>,
lnd: &State<Client>,
req: Json<PaymentRequest>,
) -> Result<Json<PaymentResponse>, String> {
let cfg = if let Some(p) = &settings.payments {
p
} else {
return Err("Payment not enabled, missing configuration option(s)".to_string());
};
let btc_amount = match cfg.cost.currency {
Currency::BTC => cfg.cost.amount,
_ => return Err("Currency not supported".to_string()),
};
let amount = btc_amount * req.units * req.quantity as f32;
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
let uid = db
.upsert_user(&pubkey_vec)
.await
.map_err(|_| "Failed to get user account".to_string())?;
let mut lnd = lnd.deref().clone();
let c = lnd.lightning();
let msat = (amount * 1e11f32) as u64;
let memo = format!(
"{}x {} {} for {}",
req.quantity, req.units, cfg.unit, auth.event.pubkey
);
info!("Requesting {} msats: {}", msat, memo);
let invoice = c
.add_invoice(Invoice {
value_msat: msat as i64,
memo,
..Default::default()
})
.await
.map_err(|e| e.message().to_string())?;
let days_value = match cfg.interval {
PaymentInterval::Day(d) => d as u64,
PaymentInterval::Month(m) => {
let now = Utc::now();
(now.add(Months::new(m as u32)) - now).num_days() as u64
}
PaymentInterval::Year(y) => {
let now = Utc::now();
(now.add(Months::new(12 * y as u32)) - now).num_days() as u64
}
};
let record = Payment {
payment_hash: invoice.get_ref().r_hash.clone(),
user_id: uid,
created: Default::default(),
amount: msat,
is_paid: false,
days_value,
size_value: cfg.unit.to_size(req.units),
settle_index: None,
rate: None,
};
if let Err(e) = db.insert_payment(&record).await {
error!("Failed to insert payment: {}", e);
return Err("Failed to insert payment".to_string());
}
Ok(Json(PaymentResponse {
pr: invoice.get_ref().payment_request.clone(),
}))
}

View File

@ -1,5 +1,3 @@
#[cfg(feature = "payments")]
use crate::payments::{Currency, PaymentAmount, PaymentInterval, PaymentUnit};
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
@ -32,12 +30,11 @@ pub struct Settings {
/// Analytics tracking
pub plausible_url: Option<String>,
#[cfg(feature = "void-cat-redirects")]
pub void_cat_database: Option<String>,
/// Path to void.cat uploads (files-v2)
pub void_cat_files: Option<PathBuf>,
#[cfg(feature = "payments")]
/// Payment options for paid storage
pub payments: Option<PaymentConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -45,33 +42,3 @@ pub struct VitModelConfig {
pub model: PathBuf,
pub config: PathBuf,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaymentConfig {
/// LND connection details
pub lnd: LndConfig,
/// Pricing per unit
pub cost: PaymentAmount,
/// What metric to bill payments on
pub unit: PaymentUnit,
/// Billing interval time per unit
pub interval: PaymentInterval,
/// Fiat base currency to store exchange rates along with invoice
pub fiat: Option<Currency>,
/// Free quota in bytes for users without payments (default: 100MB)
pub free_quota_bytes: Option<u64>,
}
#[cfg(feature = "payments")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LndConfig {
pub endpoint: String,
pub tls: PathBuf,
pub macaroon: PathBuf,
}

View File

@ -2,32 +2,19 @@ import { hexToBech32 } from "@snort/shared";
import { NostrLink } from "@snort/system";
import { useUserProfile } from "@snort/system-react";
export default function Profile({
link,
size,
showName,
}: {
link: NostrLink;
size?: number;
showName?: boolean;
}) {
export default function Profile({ link }: { link: NostrLink }) {
const profile = useUserProfile(link.id);
const s = size ?? 40;
return (
<a className="flex gap-2 items-center" href={`https://snort.social/${link.encode()}`} target="_blank">
<div className="flex gap-2 items-center">
<img
src={profile?.picture}
width={s}
height={s}
className="rounded-full object-fit object-center"
className="rounded-full w-12 h-12 object-fit object-center"
/>
{(showName ?? true) && (
<div>
{profile?.display_name ??
profile?.name ??
hexToBech32("npub", link.id).slice(0, 12)}
</div>
)}
</a>
<div>
{profile?.display_name ??
profile?.name ??
hexToBech32("npub", link.id).slice(0, 12)}
</div>
</div>
);
}

View File

@ -8,5 +8,5 @@ body {
}
hr {
@apply border-neutral-500;
}
@apply border-neutral-500
}

File diff suppressed because one or more lines are too long

View File

@ -2,15 +2,7 @@ import { base64 } from "@scure/base";
import { throwIfOffline } from "@snort/shared";
import { EventKind, EventPublisher, NostrEvent } from "@snort/system";
export interface AdminSelf {
is_admin: boolean;
file_count: number;
total_size: number;
paid_until?: number;
quota?: number;
free_quota?: number;
total_available_quota?: number;
}
export interface AdminSelf { is_admin: boolean, file_count: number, total_size: number }
export class Route96 {
constructor(
@ -22,13 +14,14 @@ export class Route96 {
async getSelf() {
const rsp = await this.#req("admin/self", "GET");
const data = await this.#handleResponse<AdminResponse<AdminSelf>>(rsp);
const data =
await this.#handleResponse<AdminResponse<AdminSelf>>(rsp);
return data;
}
async listFiles(page = 0, count = 10, mime: string | undefined) {
async listFiles(page = 0, count = 10) {
const rsp = await this.#req(
`admin/files?page=${page}&count=${count}${mime ? `&mime_type=${mime}` : ""}`,
`admin/files?page=${page}&count=${count}`,
"GET",
);
const data = await this.#handleResponse<AdminResponseFileList>(rsp);

View File

@ -41,7 +41,7 @@ export class Blossom {
);
const tags = [["x", bytesToString("hex", new Uint8Array(hash))]];
const rsp = await this.#req("media", "PUT", "media", file, tags);
const rsp = await this.#req("media", "PUT", "upload", file, tags);
if (rsp.ok) {
return (await rsp.json()) as BlobDescriptor;
} else {
@ -51,16 +51,9 @@ export class Blossom {
}
async mirror(url: string) {
const rsp = await this.#req(
"mirror",
"PUT",
"mirror",
JSON.stringify({ url }),
undefined,
{
"content-type": "application/json",
},
);
const rsp = await this.#req("mirror", "PUT", "mirror", JSON.stringify({ url }), undefined, {
"content-type": "application/json"
});
if (rsp.ok) {
return (await rsp.json()) as BlobDescriptor;
} else {

View File

@ -1,8 +1,7 @@
import { NostrEvent, NostrLink } from "@snort/system";
import { NostrEvent } from "@snort/system";
import { useState } from "react";
import { FormatBytes } from "../const";
import classNames from "classnames";
import Profile from "../components/profile";
interface FileInfo {
id: string;
@ -10,7 +9,6 @@ interface FileInfo {
name?: string;
type?: string;
size?: number;
uploader?: Array<string>;
}
export default function FileList({
@ -32,17 +30,15 @@ export default function FileList({
}
function renderInner(f: FileInfo) {
if (
f.type?.startsWith("image/") ||
f.type?.startsWith("video/") ||
!f.type
) {
if (f.type?.startsWith("image/") || !f.type) {
return (
<img
src={f.url.replace(`/${f.id}`, `/thumb/${f.id}`)}
className="w-full h-full object-contain object-center"
loading="lazy"
/>
<img src={f.url} className="w-full h-full object-contain object-center" loading="lazy" />
);
} else if (f.type?.startsWith("video/")) {
return (
<div className="w-full h-full flex items-center justify-center">
Video
</div>
);
}
}
@ -58,7 +54,6 @@ export default function FileList({
name: f.content,
type: f.tags.find((a) => a[0] === "m")?.at(1),
size: Number(f.tags.find((a) => a[0] === "size")?.at(1)),
uploader: "uploader" in f ? (f.uploader as Array<string>) : undefined,
};
} else {
return {
@ -79,14 +74,12 @@ export default function FileList({
ret.push(
<div
onClick={() => onPage?.(x)}
className={classNames(
"bg-neutral-700 hover:bg-neutral-600 min-w-8 text-center cursor-pointer font-bold",
className={classNames("bg-neutral-700 hover:bg-neutral-600 min-w-8 text-center cursor-pointer font-bold",
{
"rounded-l-md": x === start,
"rounded-r-md": x + 1 === n,
"rounded-r-md": (x + 1) === n,
"bg-neutral-400": page === x,
},
)}
})}
>
{x + 1}
</div>,
@ -109,39 +102,24 @@ export default function FileList({
>
<div className="absolute flex flex-col items-center justify-center w-full h-full text-wrap text-sm break-all text-center opacity-0 hover:opacity-100 hover:bg-black/80">
<div>
{(info.name?.length ?? 0) === 0
? "Untitled"
: info.name!.length > 20
? `${info.name?.substring(0, 10)}...${info.name?.substring(info.name.length - 10)}`
: info.name}
{(info.name?.length ?? 0) === 0 ? "Untitled" : info.name}
</div>
<div>
{info.size && !isNaN(info.size)
? FormatBytes(info.size, 2)
: ""}
</div>
<div>{info.type}</div>
<div className="flex gap-2">
<a href={info.url} className="underline" target="_blank">
Link
</a>
{onDelete && (
<a
href="#"
onClick={(e) => {
e.preventDefault();
onDelete?.(info.id);
}}
className="underline"
>
Delete
</a>
)}
{onDelete && <a href="#" onClick={e => {
e.preventDefault();
onDelete?.(info.id)
}} className="underline">
Delete
</a>}
</div>
{info.uploader &&
info.uploader.map((a) => (
<Profile link={NostrLink.publicKey(a)} size={20} />
))}
</div>
{renderInner(info)}
</div>
@ -156,9 +134,6 @@ export default function FileList({
<table className="table-auto text-sm">
<thead>
<tr>
<th className="border border-neutral-400 bg-neutral-500 py-1 px-2">
Preview
</th>
<th className="border border-neutral-400 bg-neutral-500 py-1 px-2">
Name
</th>
@ -168,11 +143,6 @@ export default function FileList({
<th className="border border-neutral-400 bg-neutral-500 py-1 px-2">
Size
</th>
{files.some((i) => "uploader" in i) && (
<th className="border border-neutral-400 bg-neutral-500 py-1 px-2">
Uploader
</th>
)}
<th className="border border-neutral-400 bg-neutral-500 py-1 px-2">
Actions
</th>
@ -183,9 +153,6 @@ export default function FileList({
const info = getInfo(a);
return (
<tr key={info.id}>
<td className="border border-neutral-500 py-1 px-2 w-8 h-8">
{renderInner(info)}
</td>
<td className="border border-neutral-500 py-1 px-2 break-all">
{(info.name?.length ?? 0) === 0 ? "<Untitled>" : info.name}
</td>
@ -197,30 +164,17 @@ export default function FileList({
? FormatBytes(info.size, 2)
: ""}
</td>
{info.uploader && (
<td className="border border-neutral-500 py-1 px-2">
{info.uploader.map((a) => (
<Profile link={NostrLink.publicKey(a)} size={20} />
))}
</td>
)}
<td className="border border-neutral-500 py-1 px-2">
<div className="flex gap-2">
<a href={info.url} className="underline" target="_blank">
Link
</a>
{onDelete && (
<a
href="#"
onClick={(e) => {
e.preventDefault();
onDelete?.(info.id);
}}
className="underline"
>
Delete
</a>
)}
{onDelete && <a href="#" onClick={e => {
e.preventDefault();
onDelete?.(info.id)
}} className="underline">
Delete
</a>}
</div>
</td>
</tr>

View File

@ -23,7 +23,6 @@ export default function Upload() {
const [adminListedFiles, setAdminListedFiles] = useState<Nip96FileList>();
const [listedPage, setListedPage] = useState(0);
const [adminListedPage, setAdminListedPage] = useState(0);
const [mimeFilter, setMimeFilter] = useState<string>();
const login = useLogin();
const pub = usePublisher();
@ -31,8 +30,7 @@ export default function Upload() {
const legacyFiles = Report as Record<string, Array<string>>;
const myLegacyFiles = login ? (legacyFiles[login.pubkey] ?? []) : [];
const url =
import.meta.env.VITE_API_URL || `${location.protocol}//${location.host}`;
const url = import.meta.env.VITE_API_URL || `${location.protocol}//${location.host}`;
async function doUpload() {
if (!pub) return;
if (!toUpload) return;
@ -40,9 +38,7 @@ export default function Upload() {
setError(undefined);
if (type === "blossom") {
const uploader = new Blossom(url, pub);
const result = noCompress
? await uploader.upload(toUpload)
: await uploader.media(toUpload);
const result = noCompress ? await uploader.upload(toUpload) : await uploader.media(toUpload);
setResults((s) => [...s, result]);
}
if (type === "nip96") {
@ -86,7 +82,7 @@ export default function Upload() {
try {
setError(undefined);
const uploader = new Route96(url, pub);
const result = await uploader.listFiles(n, 50, mimeFilter);
const result = await uploader.listFiles(n, 50);
setAdminListedFiles(result);
} catch (e) {
if (e instanceof Error) {
@ -136,7 +132,7 @@ export default function Upload() {
useEffect(() => {
listAllUploads(adminListedPage);
}, [adminListedPage, mimeFilter]);
}, [adminListedPage]);
useEffect(() => {
if (pub && !self) {
@ -191,33 +187,29 @@ export default function Upload() {
</Button>
<Button
className="flex-1"
onClick={doUpload}
disabled={login === undefined}
>
onClick={doUpload} disabled={login === undefined}>
Upload
</Button>
</div>
<hr />
{!listedFiles && (
<Button disabled={login === undefined} onClick={() => listUploads(0)}>
List Uploads
</Button>
)}
{!listedFiles && <Button disabled={login === undefined} onClick={() => listUploads(0)}>
List Uploads
</Button>}
{self && (
<div className="flex justify-between font-medium">
<div>Uploads: {self.file_count.toLocaleString()}</div>
<div>Total Size: {FormatBytes(self.total_size)}</div>
</div>
)}
{self && <div className="flex justify-between font-medium">
<div>Uploads: {self.file_count.toLocaleString()}</div>
<div>Total Size: {FormatBytes(self.total_size)}</div>
</div>}
{login && myLegacyFiles.length > 0 && (
<div className="flex flex-col gap-4 font-bold">
You have {myLegacyFiles.length.toLocaleString()} files which can be
migrated from void.cat
You have {myLegacyFiles.length.toLocaleString()} files which can be migrated from void.cat
<div className="flex gap-2">
<Button onClick={() => migrateLegacy()}>Migrate Files</Button>
<Button onClick={() => setShowLegacy((s) => !s)}>
<Button onClick={() => migrateLegacy()}>
Migrate Files
</Button>
<Button onClick={() => setShowLegacy(s => !s)}>
{!showLegacy ? "Show Files" : "Hide Files"}
</Button>
</div>
@ -226,10 +218,7 @@ export default function Upload() {
)}
{showLegacy && (
<FileList
files={myLegacyFiles.map((f) => ({
id: f,
url: `https://void.cat/d/${f}`,
}))}
files={myLegacyFiles.map(f => ({ id: f, url: `https://void.cat/d/${f}` }))}
/>
)}
{listedFiles && (
@ -250,18 +239,6 @@ export default function Upload() {
<hr />
<h3>Admin File List:</h3>
<Button onClick={() => listAllUploads(0)}>List All Uploads</Button>
<div>
<select value={mimeFilter} onChange={e => setMimeFilter(e.target.value)}>
<option value={""}>All</option>
<option>image/webp</option>
<option>image/jpeg</option>
<option>image/jpg</option>
<option>image/png</option>
<option>image/gif</option>
<option>video/mp4</option>
<option>video/mov</option>
</select>
</div>
{adminListedFiles && (
<FileList
files={adminListedFiles.files}
@ -271,7 +248,8 @@ export default function Upload() {
onDelete={async (x) => {
await deleteFile(x);
await listAllUploads(adminListedPage);
}}
}
}
/>
)}
</>