feat: bud-04 (mirror)

feat: migrate void.cat

Cargo.lock

@@ -1121,7 +1121,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 [[package]]
 name = "ffmpeg-rs-raw"
 version = "0.1.0"
-source = "git+https://git.v0l.io/Kieran/ffmpeg-rs-raw.git?rev=b358b3e4209da827e021d979c7d35876594d0285#b358b3e4209da827e021d979c7d35876594d0285"
+source = "git+https://git.v0l.io/Kieran/ffmpeg-rs-raw.git?rev=76333375d8c7c825cd9e45c041866f2c655c7bbd#76333375d8c7c825cd9e45c041866f2c655c7bbd"
 dependencies = [
  "anyhow",
  "ffmpeg-sys-the-third",
@@ -1266,6 +1266,17 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.31"
@@ -1287,6 +1298,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-io",
+ "futures-macro",
  "futures-sink",
  "futures-task",
  "memchr",
@@ -3029,10 +3041,12 @@ dependencies = [
  "system-configuration",
  "tokio",
  "tokio-native-tls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "windows-registry",
 ]
@@ -3176,6 +3190,7 @@ dependencies = [
  "sqlx",
  "sqlx-postgres",
  "tokio",
+ "tokio-util",
  "url",
  "uuid",
 ]
@@ -4502,6 +4517,19 @@ version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
 
+[[package]]
+name = "wasm-streams"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.76"

Cargo.toml

@@ -49,12 +49,13 @@ config = { version = "0.14.0", features = ["yaml"] }
 chrono = { version = "0.4.38", features = ["serde"] }
 url = "2.5.0"
 serde_with = { version = "3.8.1", features = ["hex"] }
-reqwest = "0.12.8"
+reqwest = { version = "0.12.8", features = ["stream"] }
 clap = { version = "4.5.18", features = ["derive"] }
 mime2ext = "0.1.53"
+tokio-util = { version = "0.7.13", features = ["io"] }
 
 libc = { version = "0.2.153", optional = true }
-ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "b358b3e4209da827e021d979c7d35876594d0285", optional = true }
+ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "76333375d8c7c825cd9e45c041866f2c655c7bbd", optional = true }
 candle-core = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
 candle-nn = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
 candle-transformers = { git = "https://git.v0l.io/huggingface/candle.git", tag = "0.8.1", optional = true }
@@ -63,3 +64,4 @@ http-range-header = { version = "0.4.2", optional = true }
 nostr-cursor = { git = "https://git.v0l.io/Kieran/nostr_backup_proc.git", branch = "main", optional = true }
 regex = { version = "1.11.1", optional = true }
 
+

README.md

@@ -8,6 +8,7 @@ Image hosting service
 - [Blossom Support](https://github.com/hzrd149/blossom/blob/master/buds/01.md)
   - [BUD-01](https://github.com/hzrd149/blossom/blob/master/buds/01.md)
   - [BUD-02](https://github.com/hzrd149/blossom/blob/master/buds/02.md)
+  - [BUD-04](https://github.com/hzrd149/blossom/blob/master/buds/04.md)
   - [BUD-05](https://github.com/hzrd149/blossom/blob/master/buds/05.md)
   - [BUD-06](https://github.com/hzrd149/blossom/blob/master/buds/06.md)
   - [BUD-08](https://github.com/hzrd149/blossom/blob/master/buds/08.md)

src/filesystem.rs

@@ -6,7 +6,9 @@ use std::time::SystemTime;
 
 use anyhow::Error;
 use chrono::Utc;
+use ffmpeg_rs_raw::DemuxerInfo;
 use log::info;
+use rocket::form::validate::Contains;
 use serde::Serialize;
 use sha2::{Digest, Sha256};
 use tokio::fs::File;
@@ -42,14 +44,14 @@ impl FileStore {
     }
 
     /// Store a new file
-    pub async fn put<TStream>(
+    pub async fn put<S>(
         &self,
-        stream: TStream,
+        stream: S,
         mime_type: &str,
         compress: bool,
     ) -> Result<FileSystemResult, Error>
     where
-        TStream: AsyncRead + Unpin,
+        S: AsyncRead + Unpin,
     {
         let result = self
             .store_compress_file(stream, mime_type, compress)
@@ -75,14 +77,35 @@ impl FileStore {
         }
     }
 
-    async fn store_compress_file<TStream>(
+    /// Try to replace the mime-type when unknown using ffmpeg probe result
+    fn hack_mime_type(mime_type: &str, p: &DemuxerInfo) -> String {
+        if mime_type == "application/octet-stream" {
+            if p.format.contains("mp4") {
+                "video/mp4".to_string()
+            } else if p.format.contains("webp") {
+                "image/webp".to_string()
+            } else if p.format.contains("jpeg") {
+                "image/jpeg".to_string()
+            } else if p.format.contains("png") {
+                "image/png".to_string()
+            } else if p.format.contains("gif") {
+                "image/gif".to_string()
+            } else {
+                mime_type.to_string()
+            }
+        } else {
+            mime_type.to_string()
+        }
+    }
+
+    async fn store_compress_file<S>(
         &self,
-        mut stream: TStream,
+        mut stream: S,
         mime_type: &str,
         compress: bool,
     ) -> Result<FileSystemResult, Error>
     where
-        TStream: AsyncRead + Unpin,
+        S: AsyncRead + Unpin,
     {
         let random_id = uuid::Uuid::new_v4();
         let tmp_path = FileStore::map_temp(random_id);
@@ -159,6 +182,7 @@ impl FileStore {
         } else if let Ok(p) = probe_file(tmp_path.clone()) {
             let n = file.metadata().await?.len();
             let hash = FileStore::hash_file(&mut file).await?;
+            let v_stream = p.best_video();
             return Ok(FileSystemResult {
                 path: tmp_path,
                 upload: FileUpload {
@@ -166,9 +190,9 @@ impl FileStore {
                     name: "".to_string(),
                     size: n,
                     created: Utc::now(),
-                    mime_type: mime_type.to_string(),
+                    mime_type: Self::hack_mime_type(mime_type, &p),
-                    width: p.map(|v| v.0 as u32),
+                    width: v_stream.map(|v| v.width as u32),
-                    height: p.map(|v| v.1 as u32),
+                    height: v_stream.map(|v| v.height as u32),
                     ..Default::default()
                 },
             });

src/processing/mod.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 use crate::processing::probe::FFProbe;
 use anyhow::{bail, Error, Result};
 use ffmpeg_rs_raw::ffmpeg_sys_the_third::AVPixelFormat::AV_PIX_FMT_YUV420P;
-use ffmpeg_rs_raw::{Encoder, StreamType, Transcoder};
+use ffmpeg_rs_raw::{DemuxerInfo, Encoder, StreamType, Transcoder};
 
 #[cfg(feature = "labels")]
 pub mod labeling;
@@ -64,22 +64,6 @@ impl WebpProcessor {
     }
 }
 
-pub struct ProbeResult {
-    pub streams: Vec<ProbeStream>,
-}
-
-pub enum ProbeStream {
-    Video {
-        width: u32,
-        height: u32,
-        codec: String,
-    },
-    Audio {
-        sample_rate: u32,
-        codec: String,
-    },
-}
-
 pub enum FileProcessorResult {
     NewFile(NewFileProcessorResult),
     Skip,
@@ -105,8 +89,8 @@ pub fn compress_file(in_file: PathBuf, mime_type: &str) -> Result<FileProcessorR
     }
 }
 
-pub fn probe_file(in_file: PathBuf) -> Result<Option<(usize, usize)>> {
+pub fn probe_file(in_file: PathBuf) -> Result<DemuxerInfo> {
     let proc = FFProbe::new();
     let info = proc.process_file(in_file)?;
-    Ok(info.best_video().map(|v| (v.width, v.height)))
+    Ok(info)
 }

src/routes/blossom.rs

@@ -1,21 +1,23 @@
-use log::error;
-use nostr::prelude::hex;
-use nostr::{Alphabet, SingleLetterTag, TagKind};
-use rocket::data::ByteUnit;
-use rocket::http::{Header, Status};
-use rocket::response::Responder;
-use rocket::serde::json::Json;
-use rocket::{routes, Data, Request, Response, Route, State};
-use serde::Serialize;
-use std::collections::HashMap;
-use std::fs;
-
 use crate::auth::blossom::BlossomAuth;
 use crate::db::{Database, FileUpload};
 use crate::filesystem::FileStore;
 use crate::routes::{delete_file, Nip94Event};
 use crate::settings::Settings;
 use crate::webhook::Webhook;
+use log::error;
+use nostr::prelude::hex;
+use nostr::{Alphabet, SingleLetterTag, TagKind};
+use rocket::data::ByteUnit;
+use rocket::futures::StreamExt;
+use rocket::http::{Header, Status};
+use rocket::response::Responder;
+use rocket::serde::json::Json;
+use rocket::{routes, Data, Request, Response, Route, State};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fs;
+use tokio::io::AsyncRead;
+use tokio_util::io::StreamReader;
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(crate = "rocket::serde")]
@@ -57,6 +59,11 @@ impl BlobDescriptor {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct MirrorRequest {
+    pub url: String,
+}
+
 #[cfg(feature = "media-compression")]
 pub fn blossom_routes() -> Vec<Route> {
     routes![
@@ -65,13 +72,14 @@ pub fn blossom_routes() -> Vec<Route> {
         list_files,
         upload_head,
         upload_media,
-        head_media
+        head_media,
+        mirror
     ]
 }
 
 #[cfg(not(feature = "media-compression"))]
 pub fn blossom_routes() -> Vec<Route> {
-    routes![delete_blob, upload, list_files, upload_head]
+    routes![delete_blob, upload, list_files, upload_head, mirror]
 }
 
 /// Generic holder response, mostly for errors
@@ -143,6 +151,19 @@ fn check_method(event: &nostr::Event, method: &str) -> bool {
     false
 }
 
+fn check_whitelist(auth: &BlossomAuth, settings: &Settings) -> Option<BlossomResponse> {
+    // check whitelist
+    if let Some(wl) = &settings.whitelist {
+        if !wl.contains(&auth.event.pubkey.to_hex()) {
+            return Some(BlossomResponse::Generic(BlossomGenericResponse {
+                status: Status::Forbidden,
+                message: Some("Not on whitelist".to_string()),
+            }));
+        }
+    }
+    None
+}
+
 #[rocket::delete("/<sha256>")]
 async fn delete_blob(
     sha256: &str,
@@ -198,6 +219,55 @@ async fn upload(
     process_upload("upload", false, auth, fs, db, settings, webhook, data).await
 }
 
+#[rocket::put("/mirror", data = "<req>", format = "json")]
+async fn mirror(
+    auth: BlossomAuth,
+    fs: &State<FileStore>,
+    db: &State<Database>,
+    settings: &State<Settings>,
+    webhook: &State<Option<Webhook>>,
+    req: Json<MirrorRequest>,
+) -> BlossomResponse {
+    if !check_method(&auth.event, "mirror") {
+        return BlossomResponse::error("Invalid request method tag");
+    }
+    if let Some(e) = check_whitelist(&auth, settings) {
+        return e;
+    }
+
+    // download file
+    let rsp = match reqwest::get(&req.url).await {
+        Err(e) => {
+            error!("Error downloading file: {}", e);
+            return BlossomResponse::error("Failed to mirror file");
+        }
+        Ok(rsp) => rsp,
+    };
+
+    let mime_type = rsp
+        .headers()
+        .get("content-type")
+        .map(|h| h.to_str().unwrap())
+        .unwrap_or("application/octet-stream")
+        .to_string();
+    let pubkey = auth.event.pubkey.to_bytes().to_vec();
+
+    process_stream(
+        StreamReader::new(rsp.bytes_stream().map(|result| {
+            result.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
+        })),
+        &mime_type,
+        &None,
+        &pubkey,
+        false,
+        fs,
+        db,
+        settings,
+        webhook,
+    )
+    .await
+}
+
 #[cfg(feature = "media-compression")]
 #[rocket::head("/media")]
 fn head_media(auth: BlossomAuth, settings: &State<Settings>) -> BlossomHead {
@@ -293,33 +363,47 @@ async fn process_upload(
             return BlossomResponse::error("File too large");
         }
     }
-    let mime_type = auth
-        .content_type
-        .unwrap_or("application/octet-stream".to_string());
 
     // check whitelist
-    if let Some(wl) = &settings.whitelist {
+    if let Some(e) = check_whitelist(&auth, settings) {
-        if !wl.contains(&auth.event.pubkey.to_hex()) {
+        return e;
-            return BlossomResponse::Generic(BlossomGenericResponse {
-                status: Status::Forbidden,
-                message: Some("Not on whitelist".to_string()),
-            });
     }
-    }
+
-    match fs
+    process_stream(
-        .put(
+        data.open(ByteUnit::Byte(settings.max_upload_bytes)),
-            data.open(ByteUnit::from(settings.max_upload_bytes)),
+        &auth
-            &mime_type,
+            .content_type
+            .unwrap_or("application/octet-stream".to_string()),
+        &name,
+        &auth.event.pubkey.to_bytes().to_vec(),
         compress,
+        fs,
+        db,
+        settings,
+        webhook,
     )
     .await
-    {
+}
+
+async fn process_stream<S>(
+    stream: S,
+    mime_type: &str,
+    name: &Option<&str>,
+    pubkey: &Vec<u8>,
+    compress: bool,
+    fs: &State<FileStore>,
+    db: &State<Database>,
+    settings: &State<Settings>,
+    webhook: &State<Option<Webhook>>,
+) -> BlossomResponse
+where
+    S: AsyncRead + Unpin,
+{
+    match fs.put(stream, mime_type, compress).await {
         Ok(mut blob) => {
             blob.upload.name = name.unwrap_or("").to_owned();
-
-            let pubkey_vec = auth.event.pubkey.to_bytes().to_vec();
             if let Some(wh) = webhook.as_ref() {
-                match wh.store_file(&pubkey_vec, blob.clone()).await {
+                match wh.store_file(pubkey, blob.clone()).await {
                     Ok(store) => {
                         if !store {
                             let _ = fs::remove_file(blob.path);
@@ -335,7 +419,7 @@ async fn process_upload(
                     }
                 }
             }
-            let user_id = match db.upsert_user(&pubkey_vec).await {
+            let user_id = match db.upsert_user(pubkey).await {
                 Ok(u) => u,
                 Err(e) => {
                     return BlossomResponse::error(format!("Failed to save file (db): {}", e));

ui_src/src/components/button.tsx

@@ -15,14 +15,14 @@ export default function Button({
     if (!onClick) return;
     try {
       setLoading(true);
-      onClick(e);
+      await onClick(e);
     } finally {
       setLoading(false);
     }
   }
   return (
     <button
-      className={`py-2 px-4 rounded-md border-0 text-sm font-semibold bg-neutral-700 hover:bg-neutral-600 ${className} ${props.disabled ? "opacity-50" : ""}`}
+      className={`py-2 px-4 rounded-md border-0 text-sm font-semibold bg-neutral-700 hover:bg-neutral-600 ${className} ${props.disabled || loading ? "opacity-50" : ""}`}
       onClick={doClick}
       {...props}
       disabled={loading || (props.disabled ?? false)}

ui_src/src/upload/blossom.ts

@@ -50,6 +50,18 @@ export class Blossom {
     }
   }
 
+  async mirror(url: string) {
+    const rsp = await this.#req("mirror", "PUT", "mirror", JSON.stringify({ url }), undefined, {
+      "content-type": "application/json"
+    });
+    if (rsp.ok) {
+      return (await rsp.json()) as BlobDescriptor;
+    } else {
+      const text = await rsp.text();
+      throw new Error(text);
+    }
+  }
+
   async list(pk: string) {
     const rsp = await this.#req(`list/${pk}`, "GET", "list");
     if (rsp.ok) {
@@ -76,6 +88,7 @@ export class Blossom {
     term: string,
     body?: BodyInit,
     tags?: Array<Array<string>>,
+    headers?: Record<string, string>,
   ) {
     throwIfOffline();
 
@@ -100,6 +113,7 @@ export class Blossom {
       method,
       body,
       headers: {
+        ...headers,
         accept: "application/json",
         authorization: await auth(url, method),
       },

ui_src/src/views/upload.tsx

@@ -17,6 +17,7 @@ export default function Upload() {
   const [toUpload, setToUpload] = useState<File>();
   const [self, setSelf] = useState<AdminSelf>();
   const [error, setError] = useState<string>();
+  const [bulkPrgress, setBulkProgress] = useState<number>();
   const [results, setResults] = useState<Array<object>>([]);
   const [listedFiles, setListedFiles] = useState<Nip96FileList>();
   const [adminListedFiles, setAdminListedFiles] = useState<Nip96FileList>();
@@ -26,7 +27,8 @@ export default function Upload() {
   const login = useLogin();
   const pub = usePublisher();
 
-  const myLegacyFiles = login ? (Report as Record<string, Array<string>>)[login.pubkey] : [];
+  const legacyFiles = Report as Record<string, Array<string>>;
+  const myLegacyFiles = login ? legacyFiles[login.pubkey] : [];
 
   const url = import.meta.env.VITE_API_URL || `${location.protocol}//${location.host}`;
   async function doUpload() {
@@ -110,6 +112,20 @@ export default function Upload() {
     }
   }
 
+  async function migrateLegacy() {
+    if (!pub) return;
+    const uploader = new Blossom(url, pub);
+    let ctr = 0;
+    for (const f of myLegacyFiles) {
+      try {
+        await uploader.mirror(`https://void.cat/d/${f}`);
+      } catch (e) {
+        console.error(e);
+      }
+      setBulkProgress(ctr++ / myLegacyFiles.length);
+    }
+  }
+
   useEffect(() => {
     listUploads(listedPage);
   }, [listedPage]);
@@ -190,13 +206,14 @@ export default function Upload() {
         <div className="flex flex-col gap-4 font-bold">
           You have {myLegacyFiles.length.toLocaleString()} files which can be migrated from void.cat
           <div className="flex gap-2">
-            <Button>
+            <Button onClick={() => migrateLegacy()}>
               Migrate Files
             </Button>
-            <Button onClick={() => setShowLegacy(true)}>
+            <Button onClick={() => setShowLegacy(s => !s)}>
-              Show Files
+              {!showLegacy ? "Show Files" : "Hide Files"}
             </Button>
           </div>
+          {bulkPrgress !== undefined && <progress value={bulkPrgress} />}
         </div>
       )}
       {showLegacy && (