From b3be59aec6db0dbe0099d4275bf1cbded6b6b53d Mon Sep 17 00:00:00 2001 From: Kieran Date: Thu, 26 Jan 2023 13:49:38 +0000 Subject: [PATCH] support stripping metadata --- .dockerignore | 3 +- .github/workflows/build.yml | 2 + Dockerfile | 1 + VoidCat/Controllers/UploadController.cs | 16 ++++-- VoidCat/Model/IngressPayload.cs | 2 +- VoidCat/Services/Files/FileStorageStartup.cs | 1 + .../Services/Files/LocalDiskFileStorage.cs | 54 ++++++++++++++++--- VoidCat/Services/Files/StreamFileStore.cs | 4 +- VoidCat/Services/Files/StripMetadata.cs | 51 ++++++++++++++++++ VoidCat/VoidCat.csproj | 1 + 10 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 VoidCat/Services/Files/StripMetadata.cs diff --git a/.dockerignore b/.dockerignore index f343068..0544f28 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,4 +22,5 @@ **/values.dev.yaml LICENSE README.md -**/appsettings.*.json \ No newline at end of file +**/appsettings.*.json +**/data \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 299234a..8edef87 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: docker/login-action@v1 with: registry: ghcr.io diff --git a/Dockerfile b/Dockerfile index 8cd14bc..276e16d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,5 +18,6 @@ RUN dotnet publish -c Release -o out VoidCat/VoidCat.csproj # Build runtime image FROM mcr.microsoft.com/dotnet/aspnet:6.0 WORKDIR /app +RUN apt update && apt install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* COPY --from=build-env /app/out . ENTRYPOINT ["dotnet", "VoidCat.dll"] \ No newline at end of file diff --git a/VoidCat/Controllers/UploadController.cs b/VoidCat/Controllers/UploadController.cs index fcb26ce..acd5f51 100644 --- a/VoidCat/Controllers/UploadController.cs +++ b/VoidCat/Controllers/UploadController.cs @@ -62,6 +62,9 @@ namespace VoidCat.Controllers var uid = HttpContext.GetUserId(); var mime = Request.Headers.GetHeader("V-Content-Type"); var filename = Request.Headers.GetHeader("V-Filename"); + var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata") + ?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false; + if (string.IsNullOrEmpty(mime) && !string.IsNullOrEmpty(filename)) { if (new FileExtensionContentTypeProvider().TryGetContentType(filename, out var contentType)) @@ -87,13 +90,13 @@ namespace VoidCat.Controllers Name = filename, Description = Request.Headers.GetHeader("V-Description"), Digest = Request.Headers.GetHeader("V-Full-Digest"), - Size = (ulong?) Request.ContentLength ?? 0UL, + Size = (ulong?)Request.ContentLength ?? 0UL, Storage = store, EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams") }; var (segment, totalSegments) = ParseSegmentsHeader(); - var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments), + var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata), HttpContext.RequestAborted); // save metadata @@ -102,7 +105,7 @@ namespace VoidCat.Controllers // attach file upload to user if (uid.HasValue) { - await _userUploads.AddFile(uid!.Value, vf.Id); + await _userUploads.AddFile(uid.Value, vf.Id); } if (cli) @@ -156,7 +159,10 @@ namespace VoidCat.Controllers } var editSecret = Request.Headers.GetHeader("V-EditSecret"); - var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments) + var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata") + ?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false; + + var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata) { EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty, Id = gid @@ -348,4 +354,4 @@ namespace VoidCat.Controllers public bool Required { get; init; } } -} \ No newline at end of file +} diff --git a/VoidCat/Model/IngressPayload.cs b/VoidCat/Model/IngressPayload.cs index 8d92153..b0776c6 100644 --- a/VoidCat/Model/IngressPayload.cs +++ b/VoidCat/Model/IngressPayload.cs @@ -1,6 +1,6 @@ namespace VoidCat.Model; -public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments) +public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments, bool ShouldStripMetadata) { public Guid Id { get; init; } = Guid.NewGuid(); public Guid? EditSecret { get; init; } diff --git a/VoidCat/Services/Files/FileStorageStartup.cs b/VoidCat/Services/Files/FileStorageStartup.cs index ba89617..889f446 100644 --- a/VoidCat/Services/Files/FileStorageStartup.cs +++ b/VoidCat/Services/Files/FileStorageStartup.cs @@ -10,6 +10,7 @@ public static class FileStorageStartup { services.AddTransient(); services.AddTransient(); + services.AddTransient(); if (settings.CloudStorage != default) { diff --git a/VoidCat/Services/Files/LocalDiskFileStorage.cs b/VoidCat/Services/Files/LocalDiskFileStorage.cs index 889b671..6554d97 100644 --- a/VoidCat/Services/Files/LocalDiskFileStorage.cs +++ b/VoidCat/Services/Files/LocalDiskFileStorage.cs @@ -1,3 +1,4 @@ +using System.Security.Cryptography; using VoidCat.Model; using VoidCat.Model.Exceptions; using VoidCat.Services.Abstractions; @@ -9,11 +10,13 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore { private const string FilesDir = "files-v1"; private readonly VoidSettings _settings; + private readonly StripMetadata _stripMetadata; - public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats) + public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats, StripMetadata stripMetadata) : base(stats) { _settings = settings; + _stripMetadata = stripMetadata; var dir = Path.Combine(_settings.DataDirectory, FilesDir); if (!Directory.Exists(dir)) @@ -28,7 +31,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore await using var fs = await Open(request, cts); await EgressFromStream(fs, request, outStream, cts); } - + /// public ValueTask StartEgress(EgressRequest request) { @@ -37,14 +40,49 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore /// public string Key => "local-disk"; - + /// public async ValueTask Ingress(IngressPayload payload, CancellationToken cts) { - var fPath = MapPath(payload.Id); - await using var fsTemp = new FileStream(fPath, - payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.Write); - return await IngressToStream(fsTemp, payload, cts); + var finalPath = MapPath(payload.Id); + await using var fsTemp = new FileStream(finalPath, + payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite); + + var vf = await IngressToStream(fsTemp, payload, cts); + + if (payload.ShouldStripMetadata && payload.Segment == payload.TotalSegments) + { + fsTemp.Close(); + var ext = Path.GetExtension(vf.Metadata!.Name); + var srcPath = $"{finalPath}_orig{ext}"; + File.Move(finalPath, srcPath); + + var dstPath = $"{finalPath}_dst{ext}"; + if (await _stripMetadata.TryStripMediaMetadata(srcPath, dstPath, cts)) + { + File.Move(dstPath, finalPath); + File.Delete(srcPath); + + // recompute metadata + var fInfo = new FileInfo(finalPath); + var hash = await SHA256.Create().ComputeHashAsync(fInfo.OpenRead(), cts); + vf = vf with + { + Metadata = vf.Metadata! with + { + Size = (ulong)fInfo.Length, + Digest = hash.ToHex() + } + }; + } + else + { + // move orig file back + File.Move(srcPath, finalPath); + } + } + + return vf; } /// @@ -70,4 +108,4 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore private string MapPath(Guid id) => Path.Join(_settings.DataDirectory, FilesDir, id.ToString()); -} \ No newline at end of file +} diff --git a/VoidCat/Services/Files/StreamFileStore.cs b/VoidCat/Services/Files/StreamFileStore.cs index 3274457..5f2907b 100644 --- a/VoidCat/Services/Files/StreamFileStore.cs +++ b/VoidCat/Services/Files/StreamFileStore.cs @@ -53,14 +53,14 @@ public abstract class StreamFileStore var meta = payload.Meta; if (payload.IsAppend) { - meta = meta! with + meta = meta with { Size = meta.Size + totalSize }; } else { - meta = meta! with + meta = meta with { Uploaded = DateTimeOffset.UtcNow, EditSecret = Guid.NewGuid(), diff --git a/VoidCat/Services/Files/StripMetadata.cs b/VoidCat/Services/Files/StripMetadata.cs new file mode 100644 index 0000000..5652898 --- /dev/null +++ b/VoidCat/Services/Files/StripMetadata.cs @@ -0,0 +1,51 @@ +using FFMpegCore; +using FFMpegCore.Enums; +using FFMpegCore.Pipes; +using Newtonsoft.Json; + +namespace VoidCat.Services.Files; + +/// +/// Service which utilizes ffmpeg to strip metadata from media +/// +public class StripMetadata +{ + private readonly ILogger _logger; + + public StripMetadata(ILogger logger) + { + _logger = logger; + } + + public async Task TryStripMediaMetadata(string input, string output, CancellationToken cts) + { + try + { + var ffprobe = await FFProbe.AnalyseAsync(input, cancellationToken: cts); + if (ffprobe == default) + { + throw new InvalidOperationException("Could not determine media type with ffprobe"); + } + + _logger.LogInformation("Stripping content from {type}", ffprobe.Format.FormatName); + + var ffmpeg = FFMpegArguments + .FromFileInput(input) + .OutputToFile(output, true, o => + { + o.WithoutMetadata(); + o.CopyChannel(); + }) + .CancellableThrough(cts); + + _logger.LogInformation("Running: {command}", ffmpeg.Arguments); + return await ffmpeg.ProcessAsynchronously(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not strip metadata"); + } + + return false; + } +} diff --git a/VoidCat/VoidCat.csproj b/VoidCat/VoidCat.csproj index 674dd9f..1d2cf5b 100644 --- a/VoidCat/VoidCat.csproj +++ b/VoidCat/VoidCat.csproj @@ -16,6 +16,7 @@ +