fix: compression error

This commit is contained in:
kieran 2024-12-21 13:35:40 +00:00
parent 141fa7f77d
commit 98aa3f2021
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941

View File

@ -65,9 +65,9 @@ impl FlatFileWriter {
async fn compress_file(file: PathBuf) -> Result<()> { async fn compress_file(file: PathBuf) -> Result<()> {
let out_path = file.with_extension("jsonl.zstd"); let out_path = file.with_extension("jsonl.zstd");
let mut in_file = File::open(file.clone()).await?; let mut in_file = File::open(file.clone()).await?;
{
let out_file = File::create(out_path.clone()).await?; let out_file = File::create(out_path.clone()).await?;
let mut enc = ZstdEncoder::new(out_file); let mut enc = ZstdEncoder::new(out_file);
let mut buf: [u8; 1024] = [0; 1024]; let mut buf: [u8; 1024] = [0; 1024];
while let Ok(n) = in_file.read(&mut buf).await { while let Ok(n) = in_file.read(&mut buf).await {
if n == 0 { if n == 0 {
@ -75,8 +75,8 @@ impl FlatFileWriter {
} }
enc.write_all(&buf[..n]).await?; enc.write_all(&buf[..n]).await?;
} }
enc.flush().await?; enc.shutdown().await?;
drop(enc); }
let in_size = in_file.metadata().await?.len(); let in_size = in_file.metadata().await?.len();
let out_size = File::open(out_path).await?.metadata().await?.len(); let out_size = File::open(out_path).await?.metadata().await?.len();