Kieran/route96
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: bud-04 (mirror)

Browse Source 
feat: migrate void.cat
This commit is contained in:
kieran 2024-12-16 17:20:26 +00:00
parent 5e92134b5c
commit c024cd4a4e
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
10 changed files with 228 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Cargo.lock generated
View File

@ -1121,7 +1121,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "ffmpeg-rs-raw"
version = "0.1.0"
source = "git+https://git.v0l.io/Kieran/ffmpeg-rs-raw.git?rev=b358b3e4209da827e021d979c7d35876594d0285#b358b3e4209da827e021d979c7d35876594d0285"
source = "git+https://git.v0l.io/Kieran/ffmpeg-rs-raw.git?rev=76333375d8c7c825cd9e45c041866f2c655c7bbd#76333375d8c7c825cd9e45c041866f2c655c7bbd"
dependencies = [
"anyhow",
"ffmpeg-sys-the-third",
@ -1266,6 +1266,17 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
@ -1287,6 +1298,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
@ -3029,10 +3041,12 @@ dependencies = [
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"windows-registry",
]
@ -3176,6 +3190,7 @@ dependencies = [
"sqlx",
"sqlx-postgres",
"tokio",
"tokio-util",
"url",
"uuid",
]
@ -4502,6 +4517,19 @@ version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.76"

6
Cargo.toml
View File

@ -49,12 +49,13 @@ config = { version = "0.14.0", features = ["yaml"] }
chrono = { version = "0.4.38", features = ["serde"] }
url = "2.5.0"
serde_with = { version = "3.8.1", features = ["hex"] }
reqwest = "0.12.8"
reqwest = { version = "0.12.8", features = ["stream"] }
clap = { version = "4.5.18", features = ["derive"] }
mime2ext = "0.1.53"
tokio-util = { version = "0.7.13", features = ["io"] }
libc = { version = "0.2.153", optional = true }
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "b358b3e4209da827e021d979c7d35876594d0285", optional = true }
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "76333375d8c7c825cd9e45c041866f2c655c7bbd", optional = true }
candle-core = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-nn = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
candle-transformers = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
@ -63,3 +64,4 @@ http-range-header = { version = "0.4.2", optional = true }
nostr-cursor = { git = "https://git.v0l.io/Kieran/nostr_backup_proc.git", branch = "main", optional = true }
regex = { version = "1.11.1", optional = true }

1
README.md
View File

@ -8,6 +8,7 @@ Image hosting service
- [Blossom Support](https://github.com/hzrd149/blossom/blob/master/buds/01.md)
- [BUD-01](https://github.com/hzrd149/blossom/blob/master/buds/01.md)
- [BUD-02](https://github.com/hzrd149/blossom/blob/master/buds/02.md)
- [BUD-04](https://github.com/hzrd149/blossom/blob/master/buds/04.md)
- [BUD-05](https://github.com/hzrd149/blossom/blob/master/buds/05.md)
- [BUD-06](https://github.com/hzrd149/blossom/blob/master/buds/06.md)
- [BUD-08](https://github.com/hzrd149/blossom/blob/master/buds/08.md)

42
src/filesystem.rs
View File

@ -6,7 +6,9 @@ use std::time::SystemTime;
use anyhow::Error;
use chrono::Utc;
use ffmpeg_rs_raw::DemuxerInfo;
use log::info;
use rocket::form::validate::Contains;
use serde::Serialize;
use sha2::{Digest, Sha256};
use tokio::fs::File;
@ -42,14 +44,14 @@ impl FileStore {
}
/// Store a new file
pub async fn put<TStream>(
pub async fn put<S>(
&self,
stream: TStream,
stream: S,
mime_type: &str,
compress: bool,
) -> Result<FileSystemResult, Error>
where
TStream: AsyncRead + Unpin,
S: AsyncRead + Unpin,
{
let result = self
.store_compress_file(stream, mime_type, compress)
@ -75,14 +77,35 @@ impl FileStore {
}
}
async fn store_compress_file<TStream>(
/// Try to replace the mime-type when unknown using ffmpeg probe result
fn hack_mime_type(mime_type: &str, p: &DemuxerInfo) -> String {
if mime_type == "application/octet-stream" {
if p.format.contains("mp4") {
"video/mp4".to_string()
} else if p.format.contains("webp") {
"image/webp".to_string()
} else if p.format.contains("jpeg") {
"image/jpeg".to_string()
} else if p.format.contains("png") {
"image/png".to_string()
} else if p.format.contains("gif") {
"image/gif".to_string()
} else {
mime_type.to_string()
}
} else {
mime_type.to_string()
}
}
async fn store_compress_file<S>(
&self,
mut stream: TStream,
mut stream: S,
mime_type: &str,
compress: bool,
) -> Result<FileSystemResult, Error>
where
TStream: AsyncRead + Unpin,
S: AsyncRead + Unpin,
{
let random_id = uuid::Uuid::new_v4();
let tmp_path = FileStore::map_temp(random_id);
@ -159,6 +182,7 @@ impl FileStore {
} else if let Ok(p) = probe_file(tmp_path.clone()) {
let n = file.metadata().await?.len();
let hash = FileStore::hash_file(&mut file).await?;
let v_stream = p.best_video();
return Ok(FileSystemResult {
path: tmp_path,
upload: FileUpload {
@ -166,9 +190,9 @@ impl FileStore {
name: "".to_string(),
size: n,
created: Utc::now(),
mime_type: mime_type.to_string(),
width: p.map(|v| v.0 as u32),
height: p.map(|v| v.1 as u32),
mime_type: Self::hack_mime_type(mime_type, &p),
width: v_stream.map(|v| v.width as u32),
height: v_stream.map(|v| v.height as u32),
..Default::default()
},
});

22
src/processing/mod.rs
View File

@ -3,7 +3,7 @@ use std::path::PathBuf;
use crate::processing::probe::FFProbe;
use anyhow::{bail, Error, Result};
use ffmpeg_rs_raw::ffmpeg_sys_the_third::AVPixelFormat::AV_PIX_FMT_YUV420P;
use ffmpeg_rs_raw::{Encoder, StreamType, Transcoder};
use ffmpeg_rs_raw::{DemuxerInfo, Encoder, StreamType, Transcoder};
#[cfg(feature = "labels")]
pub mod labeling;
@ -64,22 +64,6 @@ impl WebpProcessor {
}
}
pub struct ProbeResult {
pub streams: Vec<ProbeStream>,
}
pub enum ProbeStream {
Video {
width: u32,
height: u32,
codec: String,
},
Audio {
sample_rate: u32,
codec: String,
},
}
pub enum FileProcessorResult {
NewFile(NewFileProcessorResult),
Skip,
@ -105,8 +89,8 @@ pub fn compress_file(in_file: PathBuf, mime_type: &str) -> Result<FileProcessorR
}
}
pub fn probe_file(in_file: PathBuf) -> Result<Option<(usize, usize)>> {
pub fn probe_file(in_file: PathBuf) -> Result<DemuxerInfo> {
let proc = FFProbe::new();
let info = proc.process_file(in_file)?;
Ok(info.best_video().map(|v| (v.width, v.height)))
Ok(info)
}

150
src/routes/blossom.rs
View File

@ -1,21 +1,23 @@
use log::error;
use nostr::prelude::hex;
use nostr::{Alphabet, SingleLetterTag, TagKind};
use rocket::data::ByteUnit;
use rocket::http::{Header, Status};
use rocket::response::Responder;
use rocket::serde::json::Json;
use rocket::{routes, Data, Request, Response, Route, State};
use serde::Serialize;
use std::collections::HashMap;
use std::fs;
use crate::auth::blossom::BlossomAuth;
use crate::db::{Database, FileUpload};
use crate::filesystem::FileStore;
use crate::routes::{delete_file, Nip94Event};
use crate::settings::Settings;
use crate::webhook::Webhook;
use log::error;
use nostr::prelude::hex;
use nostr::{Alphabet, SingleLetterTag, TagKind};
use rocket::data::ByteUnit;
use rocket::futures::StreamExt;
use rocket::http::{Header, Status};
use rocket::response::Responder;
use rocket::serde::json::Json;
use rocket::{routes, Data, Request, Response, Route, State};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use tokio::io::AsyncRead;
use tokio_util::io::StreamReader;
#[derive(Debug, Clone, Serialize)]
#[serde(crate = "rocket::serde")]
@ -57,6 +59,11 @@ impl BlobDescriptor {
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MirrorRequest {
pub url: String,
}
#[cfg(feature = "media-compression")]
pub fn blossom_routes() -> Vec<Route> {
routes![
@ -65,13 +72,14 @@ pub fn blossom_routes() -> Vec<Route> {
list_files,
upload_head,
upload_media,
head_media
head_media,
mirror
]
}
#[cfg(not(feature = "media-compression"))]
pub fn blossom_routes() -> Vec<Route> {
routes![delete_blob, upload, list_files, upload_head]
routes![delete_blob, upload, list_files, upload_head, mirror]
}
/// Generic holder response, mostly for errors
@ -143,6 +151,19 @@ fn check_method(event: &nostr::Event, method: &str) -> bool {
false
}
fn check_whitelist(auth: &BlossomAuth, settings: &Settings) -> Option<BlossomResponse> {
// check whitelist
if let Some(wl) = &settings.whitelist {
if !wl.contains(&auth.event.pubkey.to_hex()) {
return Some(BlossomResponse::Generic(BlossomGenericResponse {
status: Status::Forbidden,
message: Some("Not on whitelist".to_string()),
}));
}
}
None
}
#[rocket::delete("/<sha256>")]
async fn delete_blob(
sha256: &str,
@ -198,6 +219,55 @@ async fn upload(
process_upload("upload", false, auth, fs, db, settings, webhook, data).await
}
#[rocket::put("/mirror", data = "<req>", format = "json")]
async fn mirror(
auth: BlossomAuth,
fs: &State<FileStore>,
db: &State<Database>,
settings: &State<Settings>,
webhook: &State<Option<Webhook>>,
req: Json<MirrorRequest>,
) -> BlossomResponse {
if !check_method(&auth.event, "mirror") {
return BlossomResponse::error("Invalid request method tag");
}
if let Some(e) = check_whitelist(&auth, settings) {
return e;
}
// download file
let rsp = match reqwest::get(&req.url).await {
Err(e) => {
error!("Error downloading file: {}", e);
return BlossomResponse::error("Failed to mirror file");
}
Ok(rsp) => rsp,
};
let mime_type = rsp
.headers()
.get("content-type")
.map(|h| h.to_str().unwrap())
.unwrap_or("application/octet-stream")
.to_string();
let pubkey = auth.event.pubkey.to_bytes().to_vec();
process_stream(
StreamReader::new(rsp.bytes_stream().map(|result| {
result.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
})),
&mime_type,
&None,
&pubkey,
false,
fs,
db,
settings,
webhook,
)
.await
}
#[cfg(feature = "media-compression")]
#[rocket::head("/media")]
fn head_media(auth: BlossomAuth, settings: &State<Settings>) -> BlossomHead {
@ -293,33 +363,47 @@ async fn process_upload(
return BlossomResponse::error("File too large");
}
}
let mime_type = auth
.content_type
.unwrap_or("application/octet-stream".to_string());
// check whitelist
if let Some(wl) = &settings.whitelist {
if !wl.contains(&auth.event.pubkey.to_hex()) {
return BlossomResponse::Generic(BlossomGenericResponse {
status: Status::Forbidden,
message: Some("Not on whitelist".to_string()),
});
if let Some(e) = check_whitelist(&auth, settings) {
return e;
}
}
match fs
.put(
data.open(ByteUnit::from(settings.max_upload_bytes)),
&mime_type,
process_stream(
data.open(ByteUnit::Byte(settings.max_upload_bytes)),
&auth
.content_type
.unwrap_or("application/octet-stream".to_string()),
&name,
&auth.event.pubkey.to_bytes().to_vec(),
compress,
fs,
db,
settings,
webhook,
)
.await
{
}
async fn process_stream<S>(
stream: S,
mime_type: &str,
name: &Option<&str>,
pubkey: &Vec<u8>,
compress: bool,
fs: &State<FileStore>,
db: &State<Database>,
settings: &State<Settings>,
webhook: &State<Option<Webhook>>,
) -> BlossomResponse
where
S: AsyncRead + Unpin,
{
match fs.put(stream, mime_type, compress).await {
Ok(mut blob) => {
blob.upload.name = name.unwrap_or("").to_owned();
let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
if let Some(wh) = webhook.as_ref() {
match wh.store_file(&pubkey_vec, blob.clone()).await {
match wh.store_file(pubkey, blob.clone()).await {
Ok(store) => {
if !store {
let _ = fs::remove_file(blob.path);
@ -335,7 +419,7 @@ async fn process_upload(
}
}
}
let user_id = match db.upsert_user(&pubkey_vec).await {
let user_id = match db.upsert_user(pubkey).await {
Ok(u) => u,
Err(e) => {
return BlossomResponse::error(format!("Failed to save file (db): {}", e));

4
ui_src/src/components/button.tsx
View File

@ -15,14 +15,14 @@ export default function Button({
if (!onClick) return;
try {
setLoading(true);
onClick(e);
await onClick(e);
} finally {
setLoading(false);
}
}
return (
<button
className={`py-2 px-4 rounded-md border-0 text-sm font-semibold bg-neutral-700 hover:bg-neutral-600 ${className} ${props.disabled ? "opacity-50" : ""}`}
className={`py-2 px-4 rounded-md border-0 text-sm font-semibold bg-neutral-700 hover:bg-neutral-600 ${className} ${props.disabled || loading ? "opacity-50" : ""}`}
onClick={doClick}
{...props}
disabled={loading || (props.disabled ?? false)}

2
ui_src/src/report.json
View File

File diff suppressed because one or more lines are too long

14
ui_src/src/upload/blossom.ts
View File

@ -50,6 +50,18 @@ export class Blossom {
}
}
async mirror(url: string) {
const rsp = await this.#req("mirror", "PUT", "mirror", JSON.stringify({ url }), undefined, {
"content-type": "application/json"
});
if (rsp.ok) {
return (await rsp.json()) as BlobDescriptor;
} else {
const text = await rsp.text();
throw new Error(text);
}
}
async list(pk: string) {
const rsp = await this.#req(`list/${pk}`, "GET", "list");
if (rsp.ok) {
@ -76,6 +88,7 @@ export class Blossom {
term: string,
body?: BodyInit,
tags?: Array<Array<string>>,
headers?: Record<string, string>,
) {
throwIfOffline();
@ -100,6 +113,7 @@ export class Blossom {
method,
body,
headers: {
...headers,
accept: "application/json",
authorization: await auth(url, method),
},

25
ui_src/src/views/upload.tsx
View File

@ -17,6 +17,7 @@ export default function Upload() {
const [toUpload, setToUpload] = useState<File>();
const [self, setSelf] = useState<AdminSelf>();
const [error, setError] = useState<string>();
const [bulkPrgress, setBulkProgress] = useState<number>();
const [results, setResults] = useState<Array<object>>([]);
const [listedFiles, setListedFiles] = useState<Nip96FileList>();
const [adminListedFiles, setAdminListedFiles] = useState<Nip96FileList>();
@ -26,7 +27,8 @@ export default function Upload() {
const login = useLogin();
const pub = usePublisher();
const myLegacyFiles = login ? (Report as Record<string, Array<string>>)[login.pubkey] : [];
const legacyFiles = Report as Record<string, Array<string>>;
const myLegacyFiles = login ? legacyFiles[login.pubkey] : [];
const url = import.meta.env.VITE_API_URL || `${location.protocol}//${location.host}`;
async function doUpload() {
@ -110,6 +112,20 @@ export default function Upload() {
}
}
async function migrateLegacy() {
if (!pub) return;
const uploader = new Blossom(url, pub);
let ctr = 0;
for (const f of myLegacyFiles) {
try {
await uploader.mirror(`https://void.cat/d/${f}`);
} catch (e) {
console.error(e);
}
setBulkProgress(ctr++ / myLegacyFiles.length);
}
}
useEffect(() => {
listUploads(listedPage);
}, [listedPage]);
@ -190,13 +206,14 @@ export default function Upload() {
<div className="flex flex-col gap-4 font-bold">
You have {myLegacyFiles.length.toLocaleString()} files which can be migrated from void.cat
<div className="flex gap-2">
<Button>
<Button onClick={() => migrateLegacy()}>
Migrate Files
</Button>
<Button onClick={() => setShowLegacy(true)}>
Show Files
<Button onClick={() => setShowLegacy(s => !s)}>
{!showLegacy ? "Show Files" : "Hide Files"}
</Button>
</div>
{bulkPrgress !== undefined && <progress value={bulkPrgress} />}
</div>
)}
{showLegacy && (