feat: r96util progress
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
parent
4dad339c09
commit
069aa30d52
61
Cargo.lock
generated
61
Cargo.lock
generated
@ -737,6 +737,19 @@ dependencies = [
|
||||
"yaml-rust2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"unicode-width",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.9.6"
|
||||
@ -1040,6 +1053,12 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
@ -2017,6 +2036,19 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
|
||||
dependencies = [
|
||||
"console",
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"unicode-width",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "infer"
|
||||
version = "0.16.0"
|
||||
@ -2525,6 +2557,12 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.5"
|
||||
@ -2801,6 +2839,12 @@ dependencies = [
|
||||
"universal-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
@ -3195,6 +3239,7 @@ dependencies = [
|
||||
"ffmpeg-rs-raw",
|
||||
"hex",
|
||||
"http-range-header",
|
||||
"indicatif",
|
||||
"infer",
|
||||
"libc",
|
||||
"log",
|
||||
@ -4354,6 +4399,12 @@ version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
@ -4561,6 +4612,16 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "1.5.2"
|
||||
|
||||
@ -18,6 +18,7 @@ path = "src/bin/main.rs"
|
||||
[[bin]]
|
||||
name = "r96util"
|
||||
path = "src/bin/r96util.rs"
|
||||
required-features = ["r96util"]
|
||||
|
||||
[lib]
|
||||
name = "route96"
|
||||
@ -35,6 +36,7 @@ analytics = []
|
||||
void-cat-redirects = ["dep:sqlx-postgres"]
|
||||
ranges = ["dep:http-range-header"]
|
||||
react-ui = []
|
||||
r96util = ["dep:walkdir", "dep:indicatif"]
|
||||
|
||||
[dependencies]
|
||||
log = "0.4.21"
|
||||
@ -57,7 +59,7 @@ clap = { version = "4.5.18", features = ["derive"] }
|
||||
mime2ext = "0.1.53"
|
||||
infer = "0.16.0"
|
||||
tokio-util = { version = "0.7.13", features = ["io", "io-util"] }
|
||||
walkdir = "2.5.0"
|
||||
|
||||
|
||||
libc = { version = "0.2.153", optional = true }
|
||||
ffmpeg-rs-raw = { git = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git", rev = "a63b88ef3c8f58c7c0ac57d361d06ff0bb3ed385", optional = true }
|
||||
@ -68,3 +70,5 @@ sqlx-postgres = { version = "0.8.2", optional = true, features = ["chrono", "uui
|
||||
http-range-header = { version = "0.4.2", optional = true }
|
||||
nostr-cursor = { git = "https://git.v0l.io/Kieran/nostr_backup_proc.git", branch = "main", optional = true }
|
||||
regex = { version = "1.11.1", optional = true }
|
||||
walkdir = { version = "2.5.0", optional = true }
|
||||
indicatif = { version = "0.17.11", optional = true }
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
use anyhow::{Error, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use config::Config;
|
||||
use log::{debug, error, info, warn};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use log::info;
|
||||
use route96::db::{Database, FileUpload};
|
||||
use route96::filesystem::{FileStore, FileSystemResult};
|
||||
use route96::settings::Settings;
|
||||
@ -67,22 +68,23 @@ async fn main() -> Result<(), Error> {
|
||||
Commands::Check { delete } => {
|
||||
info!("Checking files in: {}", settings.storage_dir);
|
||||
let fs = FileStore::new(settings.clone());
|
||||
iter_files(&fs.storage_dir(), |entry| {
|
||||
iter_files(&fs.storage_dir(), |entry, p| {
|
||||
let p = p.clone();
|
||||
Box::pin(async move {
|
||||
let id = if let Some(i) = id_from_path(&entry) {
|
||||
i
|
||||
} else {
|
||||
warn!("Skipping invalid file: {}", &entry.display());
|
||||
p.set_message(format!("Skipping invalid file: {}", &entry.display()));
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let hash = FileStore::hash_file(&entry).await?;
|
||||
if hash != id {
|
||||
if delete.unwrap_or(false) {
|
||||
warn!("Deleting corrupt file: {}", &entry.display());
|
||||
p.set_message(format!("Deleting corrupt file: {}", &entry.display()));
|
||||
tokio::fs::remove_file(&entry).await?;
|
||||
} else {
|
||||
warn!("File is corrupted: {}", &entry.display());
|
||||
p.set_message(format!("File is corrupted: {}", &entry.display()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@ -95,8 +97,9 @@ async fn main() -> Result<(), Error> {
|
||||
let db = Database::new(&settings.database).await?;
|
||||
db.migrate().await?;
|
||||
info!("Importing from: {}", fs.storage_dir().display());
|
||||
iter_files(&from, |entry| {
|
||||
iter_files(&from, |entry, p| {
|
||||
let fs = fs.clone();
|
||||
let p = p.clone();
|
||||
Box::pin(async move {
|
||||
let mime = infer::get_from_path(&entry)?
|
||||
.map(|m| m.mime_type())
|
||||
@ -105,9 +108,11 @@ async fn main() -> Result<(), Error> {
|
||||
let dst = fs.put(file, mime, false).await?;
|
||||
match dst {
|
||||
FileSystemResult::AlreadyExists(_) => {
|
||||
info!("Duplicate file: {}", &entry.display())
|
||||
p.set_message(format!("Duplicate file: {}", &entry.display()));
|
||||
}
|
||||
FileSystemResult::NewFile(_) => {
|
||||
p.set_message(format!("Imported: {}", &entry.display()));
|
||||
}
|
||||
FileSystemResult::NewFile(_) => info!("Imported: {}", &entry.display()),
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
@ -119,19 +124,20 @@ async fn main() -> Result<(), Error> {
|
||||
let db = Database::new(&settings.database).await?;
|
||||
db.migrate().await?;
|
||||
info!("Importing to DB from: {}", fs.storage_dir().display());
|
||||
iter_files(&fs.storage_dir(), |entry| {
|
||||
iter_files(&fs.storage_dir(), |entry, p| {
|
||||
let db = db.clone();
|
||||
let p = p.clone();
|
||||
Box::pin(async move {
|
||||
let id = if let Some(i) = id_from_path(&entry) {
|
||||
i
|
||||
} else {
|
||||
warn!("Skipping invalid file: {}", &entry.display());
|
||||
p.set_message(format!("Skipping invalid file: {}", &entry.display()));
|
||||
return Ok(());
|
||||
};
|
||||
let u = db.get_file(&id).await?;
|
||||
if u.is_none() {
|
||||
if !dry_run.unwrap_or(false) {
|
||||
info!("Importing file: {}", &entry.display());
|
||||
p.set_message(format!("Importing file: {}", &entry.display()));
|
||||
let mime = infer::get_from_path(&entry)?
|
||||
.map(|m| m.mime_type())
|
||||
.unwrap_or("application/octet-stream")
|
||||
@ -155,7 +161,10 @@ async fn main() -> Result<(), Error> {
|
||||
};
|
||||
db.add_file(&entry, None).await?;
|
||||
} else {
|
||||
info!("[DRY-RUN] Importing file: {}", &entry.display());
|
||||
p.set_message(format!(
|
||||
"[DRY-RUN] Importing file: {}",
|
||||
&entry.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@ -173,18 +182,26 @@ fn id_from_path(path: &Path) -> Option<Vec<u8>> {
|
||||
|
||||
async fn iter_files<F>(p: &Path, mut op: F) -> Result<()>
|
||||
where
|
||||
F: FnMut(PathBuf) -> Pin<Box<dyn Future<Output = Result<()>>>>,
|
||||
F: FnMut(PathBuf, &ProgressBar) -> Pin<Box<dyn Future<Output = Result<()>>>>,
|
||||
{
|
||||
info!("Scanning files: {}", p.display());
|
||||
let entries = walkdir::WalkDir::new(p);
|
||||
for entry in entries
|
||||
let dir = entries
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|e| e.file_type().is_file())
|
||||
{
|
||||
debug!("Checking file: {}", entry.path().display());
|
||||
if let Err(e) = op(entry.path().to_path_buf()).await {
|
||||
error!("Error processing file: {} {}", entry.path().display(), e);
|
||||
.collect::<Vec<_>>();
|
||||
let p = ProgressBar::new(dir.len() as u64).with_style(ProgressStyle::with_template(
|
||||
"{spinner} [{pos}/{len}] {msg}",
|
||||
)?);
|
||||
for entry in dir {
|
||||
p.inc(1);
|
||||
if let Err(e) = op(entry.path().to_path_buf(), &p).await {
|
||||
p.set_message(format!(
|
||||
"Error processing file: {} {}",
|
||||
entry.path().display(),
|
||||
e
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user