forked from Kieran/void.cat
support stripping metadata
This commit is contained in:
parent
dbaaf8efe1
commit
b3be59aec6
@ -22,4 +22,5 @@
|
|||||||
**/values.dev.yaml
|
**/values.dev.yaml
|
||||||
LICENSE
|
LICENSE
|
||||||
README.md
|
README.md
|
||||||
**/appsettings.*.json
|
**/appsettings.*.json
|
||||||
|
**/data
|
2
.github/workflows/build.yml
vendored
2
.github/workflows/build.yml
vendored
@ -7,6 +7,8 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
- uses: docker/login-action@v1
|
- uses: docker/login-action@v1
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
|
@ -18,5 +18,6 @@ RUN dotnet publish -c Release -o out VoidCat/VoidCat.csproj
|
|||||||
# Build runtime image
|
# Build runtime image
|
||||||
FROM mcr.microsoft.com/dotnet/aspnet:6.0
|
FROM mcr.microsoft.com/dotnet/aspnet:6.0
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
RUN apt update && apt install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||||
COPY --from=build-env /app/out .
|
COPY --from=build-env /app/out .
|
||||||
ENTRYPOINT ["dotnet", "VoidCat.dll"]
|
ENTRYPOINT ["dotnet", "VoidCat.dll"]
|
@ -62,6 +62,9 @@ namespace VoidCat.Controllers
|
|||||||
var uid = HttpContext.GetUserId();
|
var uid = HttpContext.GetUserId();
|
||||||
var mime = Request.Headers.GetHeader("V-Content-Type");
|
var mime = Request.Headers.GetHeader("V-Content-Type");
|
||||||
var filename = Request.Headers.GetHeader("V-Filename");
|
var filename = Request.Headers.GetHeader("V-Filename");
|
||||||
|
var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata")
|
||||||
|
?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false;
|
||||||
|
|
||||||
if (string.IsNullOrEmpty(mime) && !string.IsNullOrEmpty(filename))
|
if (string.IsNullOrEmpty(mime) && !string.IsNullOrEmpty(filename))
|
||||||
{
|
{
|
||||||
if (new FileExtensionContentTypeProvider().TryGetContentType(filename, out var contentType))
|
if (new FileExtensionContentTypeProvider().TryGetContentType(filename, out var contentType))
|
||||||
@ -87,13 +90,13 @@ namespace VoidCat.Controllers
|
|||||||
Name = filename,
|
Name = filename,
|
||||||
Description = Request.Headers.GetHeader("V-Description"),
|
Description = Request.Headers.GetHeader("V-Description"),
|
||||||
Digest = Request.Headers.GetHeader("V-Full-Digest"),
|
Digest = Request.Headers.GetHeader("V-Full-Digest"),
|
||||||
Size = (ulong?) Request.ContentLength ?? 0UL,
|
Size = (ulong?)Request.ContentLength ?? 0UL,
|
||||||
Storage = store,
|
Storage = store,
|
||||||
EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams")
|
EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams")
|
||||||
};
|
};
|
||||||
|
|
||||||
var (segment, totalSegments) = ParseSegmentsHeader();
|
var (segment, totalSegments) = ParseSegmentsHeader();
|
||||||
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments),
|
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata),
|
||||||
HttpContext.RequestAborted);
|
HttpContext.RequestAborted);
|
||||||
|
|
||||||
// save metadata
|
// save metadata
|
||||||
@ -102,7 +105,7 @@ namespace VoidCat.Controllers
|
|||||||
// attach file upload to user
|
// attach file upload to user
|
||||||
if (uid.HasValue)
|
if (uid.HasValue)
|
||||||
{
|
{
|
||||||
await _userUploads.AddFile(uid!.Value, vf.Id);
|
await _userUploads.AddFile(uid.Value, vf.Id);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cli)
|
if (cli)
|
||||||
@ -156,7 +159,10 @@ namespace VoidCat.Controllers
|
|||||||
}
|
}
|
||||||
|
|
||||||
var editSecret = Request.Headers.GetHeader("V-EditSecret");
|
var editSecret = Request.Headers.GetHeader("V-EditSecret");
|
||||||
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments)
|
var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata")
|
||||||
|
?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false;
|
||||||
|
|
||||||
|
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata)
|
||||||
{
|
{
|
||||||
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
|
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
|
||||||
Id = gid
|
Id = gid
|
||||||
@ -348,4 +354,4 @@ namespace VoidCat.Controllers
|
|||||||
|
|
||||||
public bool Required { get; init; }
|
public bool Required { get; init; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
namespace VoidCat.Model;
|
namespace VoidCat.Model;
|
||||||
|
|
||||||
public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments)
|
public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments, bool ShouldStripMetadata)
|
||||||
{
|
{
|
||||||
public Guid Id { get; init; } = Guid.NewGuid();
|
public Guid Id { get; init; } = Guid.NewGuid();
|
||||||
public Guid? EditSecret { get; init; }
|
public Guid? EditSecret { get; init; }
|
||||||
|
@ -10,6 +10,7 @@ public static class FileStorageStartup
|
|||||||
{
|
{
|
||||||
services.AddTransient<FileInfoManager>();
|
services.AddTransient<FileInfoManager>();
|
||||||
services.AddTransient<FileStoreFactory>();
|
services.AddTransient<FileStoreFactory>();
|
||||||
|
services.AddTransient<StripMetadata>();
|
||||||
|
|
||||||
if (settings.CloudStorage != default)
|
if (settings.CloudStorage != default)
|
||||||
{
|
{
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
using System.Security.Cryptography;
|
||||||
using VoidCat.Model;
|
using VoidCat.Model;
|
||||||
using VoidCat.Model.Exceptions;
|
using VoidCat.Model.Exceptions;
|
||||||
using VoidCat.Services.Abstractions;
|
using VoidCat.Services.Abstractions;
|
||||||
@ -9,11 +10,13 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
{
|
{
|
||||||
private const string FilesDir = "files-v1";
|
private const string FilesDir = "files-v1";
|
||||||
private readonly VoidSettings _settings;
|
private readonly VoidSettings _settings;
|
||||||
|
private readonly StripMetadata _stripMetadata;
|
||||||
|
|
||||||
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
|
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats, StripMetadata stripMetadata)
|
||||||
: base(stats)
|
: base(stats)
|
||||||
{
|
{
|
||||||
_settings = settings;
|
_settings = settings;
|
||||||
|
_stripMetadata = stripMetadata;
|
||||||
|
|
||||||
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
|
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
|
||||||
if (!Directory.Exists(dir))
|
if (!Directory.Exists(dir))
|
||||||
@ -28,7 +31,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
await using var fs = await Open(request, cts);
|
await using var fs = await Open(request, cts);
|
||||||
await EgressFromStream(fs, request, outStream, cts);
|
await EgressFromStream(fs, request, outStream, cts);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public ValueTask<EgressResult> StartEgress(EgressRequest request)
|
public ValueTask<EgressResult> StartEgress(EgressRequest request)
|
||||||
{
|
{
|
||||||
@ -37,14 +40,49 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public string Key => "local-disk";
|
public string Key => "local-disk";
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
|
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
|
||||||
{
|
{
|
||||||
var fPath = MapPath(payload.Id);
|
var finalPath = MapPath(payload.Id);
|
||||||
await using var fsTemp = new FileStream(fPath,
|
await using var fsTemp = new FileStream(finalPath,
|
||||||
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.Write);
|
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite);
|
||||||
return await IngressToStream(fsTemp, payload, cts);
|
|
||||||
|
var vf = await IngressToStream(fsTemp, payload, cts);
|
||||||
|
|
||||||
|
if (payload.ShouldStripMetadata && payload.Segment == payload.TotalSegments)
|
||||||
|
{
|
||||||
|
fsTemp.Close();
|
||||||
|
var ext = Path.GetExtension(vf.Metadata!.Name);
|
||||||
|
var srcPath = $"{finalPath}_orig{ext}";
|
||||||
|
File.Move(finalPath, srcPath);
|
||||||
|
|
||||||
|
var dstPath = $"{finalPath}_dst{ext}";
|
||||||
|
if (await _stripMetadata.TryStripMediaMetadata(srcPath, dstPath, cts))
|
||||||
|
{
|
||||||
|
File.Move(dstPath, finalPath);
|
||||||
|
File.Delete(srcPath);
|
||||||
|
|
||||||
|
// recompute metadata
|
||||||
|
var fInfo = new FileInfo(finalPath);
|
||||||
|
var hash = await SHA256.Create().ComputeHashAsync(fInfo.OpenRead(), cts);
|
||||||
|
vf = vf with
|
||||||
|
{
|
||||||
|
Metadata = vf.Metadata! with
|
||||||
|
{
|
||||||
|
Size = (ulong)fInfo.Length,
|
||||||
|
Digest = hash.ToHex()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// move orig file back
|
||||||
|
File.Move(srcPath, finalPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return vf;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
@ -70,4 +108,4 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
|
|
||||||
private string MapPath(Guid id) =>
|
private string MapPath(Guid id) =>
|
||||||
Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
|
Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
|
||||||
}
|
}
|
||||||
|
@ -53,14 +53,14 @@ public abstract class StreamFileStore
|
|||||||
var meta = payload.Meta;
|
var meta = payload.Meta;
|
||||||
if (payload.IsAppend)
|
if (payload.IsAppend)
|
||||||
{
|
{
|
||||||
meta = meta! with
|
meta = meta with
|
||||||
{
|
{
|
||||||
Size = meta.Size + totalSize
|
Size = meta.Size + totalSize
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
meta = meta! with
|
meta = meta with
|
||||||
{
|
{
|
||||||
Uploaded = DateTimeOffset.UtcNow,
|
Uploaded = DateTimeOffset.UtcNow,
|
||||||
EditSecret = Guid.NewGuid(),
|
EditSecret = Guid.NewGuid(),
|
||||||
|
51
VoidCat/Services/Files/StripMetadata.cs
Normal file
51
VoidCat/Services/Files/StripMetadata.cs
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
using FFMpegCore;
|
||||||
|
using FFMpegCore.Enums;
|
||||||
|
using FFMpegCore.Pipes;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
namespace VoidCat.Services.Files;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service which utilizes ffmpeg to strip metadata from media
|
||||||
|
/// </summary>
|
||||||
|
public class StripMetadata
|
||||||
|
{
|
||||||
|
private readonly ILogger<StripMetadata> _logger;
|
||||||
|
|
||||||
|
public StripMetadata(ILogger<StripMetadata> logger)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<bool> TryStripMediaMetadata(string input, string output, CancellationToken cts)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var ffprobe = await FFProbe.AnalyseAsync(input, cancellationToken: cts);
|
||||||
|
if (ffprobe == default)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Could not determine media type with ffprobe");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("Stripping content from {type}", ffprobe.Format.FormatName);
|
||||||
|
|
||||||
|
var ffmpeg = FFMpegArguments
|
||||||
|
.FromFileInput(input)
|
||||||
|
.OutputToFile(output, true, o =>
|
||||||
|
{
|
||||||
|
o.WithoutMetadata();
|
||||||
|
o.CopyChannel();
|
||||||
|
})
|
||||||
|
.CancellableThrough(cts);
|
||||||
|
|
||||||
|
_logger.LogInformation("Running: {command}", ffmpeg.Arguments);
|
||||||
|
return await ffmpeg.ProcessAsynchronously();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Could not strip metadata");
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
@ -16,6 +16,7 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="AWSSDK.S3" Version="3.7.9.30" />
|
<PackageReference Include="AWSSDK.S3" Version="3.7.9.30" />
|
||||||
<PackageReference Include="Dapper" Version="2.0.123" />
|
<PackageReference Include="Dapper" Version="2.0.123" />
|
||||||
|
<PackageReference Include="FFMpegCore" Version="4.8.0" />
|
||||||
<PackageReference Include="FluentMigrator" Version="3.3.2" />
|
<PackageReference Include="FluentMigrator" Version="3.3.2" />
|
||||||
<PackageReference Include="FluentMigrator.Runner" Version="3.3.2" />
|
<PackageReference Include="FluentMigrator.Runner" Version="3.3.2" />
|
||||||
<PackageReference Include="FluentMigrator.Runner.Postgres" Version="3.3.2" />
|
<PackageReference Include="FluentMigrator.Runner.Postgres" Version="3.3.2" />
|
||||||
|
Loading…
Reference in New Issue
Block a user