use anyhow::Error; use chrono::{DateTime, Utc}; use clap::Parser; use config::Config; use log::{info, warn}; use route96::db::{Database, FileUpload}; use route96::filesystem::FileStore; use route96::settings::Settings; use sqlx::FromRow; use sqlx_postgres::{PgPool, Postgres}; use std::path::PathBuf; use tokio::fs::File; use uuid::Uuid; #[derive(Parser, Debug)] #[command(version, about)] struct Args { /// Database connection string for void.cat DB #[arg(long)] pub database: String, /// Path to filestore on void.cat #[arg(long)] pub data_path: String, } #[tokio::main] async fn main() -> Result<(), Error> { pretty_env_logger::init(); let builder = Config::builder() .add_source(config::File::with_name("config.toml")) .add_source(config::Environment::with_prefix("APP")) .build()?; let settings: Settings = builder.try_deserialize()?; let db = Database::new(&settings.database).await?; let fs = FileStore::new(settings.clone()); let args: Args = Args::parse(); let db_void = VoidCatDb::connect(&args.database).await?; let mut page = 0; loop { let files = db_void.list_files(page).await?; if files.len() == 0 { break; } for f in files { if let Err(e) = migrate_file(&f, &db, &fs, &args).await { warn!("Failed to migrate file: {}, {}", &f.id, e); } } page += 1; } Ok(()) } async fn migrate_file( f: &VoidFile, db: &Database, fs: &FileStore, args: &Args, ) -> Result<(), Error> { let pubkey_vec = hex::decode(&f.email)?; let id_vec = hex::decode(&f.digest)?; // copy file let src_path = PathBuf::new().join(&args.data_path).join(f.map_to_path()); let dst_path = fs.map_path(&id_vec); if src_path.exists() && !dst_path.exists() { info!( "Copying file: {} from {} => {}", &f.id, src_path.to_str().unwrap(), dst_path.to_str().unwrap() ); tokio::fs::copy(src_path, dst_path).await?; } else if dst_path.exists() { info!("File already exists {}, continuing...", &f.id); } else { anyhow::bail!("Source file not found {}", src_path.to_str().unwrap()); } let uid = db.upsert_user(&pubkey_vec).await?; info!("Mapped user {} => {}", &f.email, uid); let md: Option> = match &f.media_dimensions { Some(s) => Some(s.split("x").collect()), _ => None, }; let fu = FileUpload { id: id_vec, name: match &f.name { Some(n) => n.to_string(), None => "".to_string(), }, size: f.size as u64, mime_type: f.mime_type.clone(), created: f.uploaded, width: match &md { Some(s) => Some(s[0].parse::()?), None => None, }, height: match &md { Some(s) => Some(s[1].parse::()?), None => None, }, blur_hash: None, alt: f.description.clone(), }; db.add_file(&fu, uid).await?; Ok(()) } #[derive(FromRow)] struct VoidFile { #[sqlx(rename = "Id")] pub id: Uuid, #[sqlx(rename = "Name")] pub name: Option, #[sqlx(rename = "Size")] pub size: i64, #[sqlx(rename = "Uploaded")] pub uploaded: DateTime, #[sqlx(rename = "Description")] pub description: Option, #[sqlx(rename = "MimeType")] pub mime_type: String, #[sqlx(rename = "Digest")] pub digest: String, #[sqlx(rename = "MediaDimensions")] pub media_dimensions: Option, #[sqlx(rename = "Email")] pub email: String, } impl VoidFile { fn map_to_path(&self) -> PathBuf { let id_str = self.id.as_hyphenated().to_string(); PathBuf::new() .join("files-v2/") .join(&id_str[..2]) .join(&id_str[2..4]) .join(&id_str) } } struct VoidCatDb { pub pool: PgPool, } impl VoidCatDb { async fn connect(conn: &str) -> Result { let pool = PgPool::connect(conn).await?; Ok(Self { pool }) } async fn list_files(&self, page: usize) -> Result, sqlx::Error> { let page_size = 100; sqlx::query_as(format!("select f.\"Id\", f.\"Name\", CAST(f.\"Size\" as BIGINT) \"Size\", f.\"Uploaded\", f.\"Description\", f.\"MimeType\", f.\"Digest\", f.\"MediaDimensions\", u.\"Email\" from \"Files\" f, \"UserFiles\" uf, \"Users\" u where f.\"Id\" = uf.\"FileId\" and uf.\"UserId\" = u.\"Id\" and u.\"AuthType\" = 4\ offset {} limit {}", page * page_size, page_size).as_str()) .fetch_all(&self.pool) .await } }