laylarr00/voidkitty
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

support stripping metadata

Browse Source
This commit is contained in:
Kieran 2023-01-26 13:49:38 +00:00
parent dbaaf8efe1
commit b3be59aec6
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
10 changed files with 118 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.dockerignore
View File

@ -22,4 +22,5 @@
**/values.dev.yaml
LICENSE
README.md
**/appsettings.*.json
**/appsettings.*.json
**/data

2
.github/workflows/build.yml vendored
View File

@ -7,6 +7,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: docker/login-action@v1
with:
registry: ghcr.io

1
Dockerfile
View File

@ -18,5 +18,6 @@ RUN dotnet publish -c Release -o out VoidCat/VoidCat.csproj
# Build runtime image
FROM mcr.microsoft.com/dotnet/aspnet:6.0
WORKDIR /app
RUN apt update && apt install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
COPY --from=build-env /app/out .
ENTRYPOINT ["dotnet", "VoidCat.dll"]

16
VoidCat/Controllers/UploadController.cs
View File

@ -62,6 +62,9 @@ namespace VoidCat.Controllers
var uid = HttpContext.GetUserId();
var mime = Request.Headers.GetHeader("V-Content-Type");
var filename = Request.Headers.GetHeader("V-Filename");
var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata")
?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false;
if (string.IsNullOrEmpty(mime) && !string.IsNullOrEmpty(filename))
{
if (new FileExtensionContentTypeProvider().TryGetContentType(filename, out var contentType))
@ -87,13 +90,13 @@ namespace VoidCat.Controllers
Name = filename,
Description = Request.Headers.GetHeader("V-Description"),
Digest = Request.Headers.GetHeader("V-Full-Digest"),
Size = (ulong?) Request.ContentLength ?? 0UL,
Size = (ulong?)Request.ContentLength ?? 0UL,
Storage = store,
EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams")
};
var (segment, totalSegments) = ParseSegmentsHeader();
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments),
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata),
HttpContext.RequestAborted);
// save metadata
@ -102,7 +105,7 @@ namespace VoidCat.Controllers
// attach file upload to user
if (uid.HasValue)
{
await _userUploads.AddFile(uid!.Value, vf.Id);
await _userUploads.AddFile(uid.Value, vf.Id);
}
if (cli)
@ -156,7 +159,10 @@ namespace VoidCat.Controllers
}
var editSecret = Request.Headers.GetHeader("V-EditSecret");
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments)
var stripMetadata = Request.Headers.GetHeader("V-Strip-Metadata")
?.Equals("true", StringComparison.InvariantCultureIgnoreCase) ?? false;
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments, stripMetadata)
{
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
Id = gid
@ -348,4 +354,4 @@ namespace VoidCat.Controllers
public bool Required { get; init; }
}
}
}

2
VoidCat/Model/IngressPayload.cs
View File

@ -1,6 +1,6 @@
namespace VoidCat.Model;
public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments)
public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments, bool ShouldStripMetadata)
{
public Guid Id { get; init; } = Guid.NewGuid();
public Guid? EditSecret { get; init; }

1
VoidCat/Services/Files/FileStorageStartup.cs
View File

@ -10,6 +10,7 @@ public static class FileStorageStartup
{
services.AddTransient<FileInfoManager>();
services.AddTransient<FileStoreFactory>();
services.AddTransient<StripMetadata>();
if (settings.CloudStorage != default)
{

54
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@ -1,3 +1,4 @@
using System.Security.Cryptography;
using VoidCat.Model;
using VoidCat.Model.Exceptions;
using VoidCat.Services.Abstractions;
@ -9,11 +10,13 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
{
private const string FilesDir = "files-v1";
private readonly VoidSettings _settings;
private readonly StripMetadata _stripMetadata;
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats, StripMetadata stripMetadata)
: base(stats)
{
_settings = settings;
_stripMetadata = stripMetadata;
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
if (!Directory.Exists(dir))
@ -28,7 +31,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
await using var fs = await Open(request, cts);
await EgressFromStream(fs, request, outStream, cts);
}
/// <inheritdoc />
public ValueTask<EgressResult> StartEgress(EgressRequest request)
{
@ -37,14 +40,49 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
/// <inheritdoc />
public string Key => "local-disk";
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
var fPath = MapPath(payload.Id);
await using var fsTemp = new FileStream(fPath,
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.Write);
return await IngressToStream(fsTemp, payload, cts);
var finalPath = MapPath(payload.Id);
await using var fsTemp = new FileStream(finalPath,
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite);
var vf = await IngressToStream(fsTemp, payload, cts);
if (payload.ShouldStripMetadata && payload.Segment == payload.TotalSegments)
{
fsTemp.Close();
var ext = Path.GetExtension(vf.Metadata!.Name);
var srcPath = $"{finalPath}_orig{ext}";
File.Move(finalPath, srcPath);
var dstPath = $"{finalPath}_dst{ext}";
if (await _stripMetadata.TryStripMediaMetadata(srcPath, dstPath, cts))
{
File.Move(dstPath, finalPath);
File.Delete(srcPath);
// recompute metadata
var fInfo = new FileInfo(finalPath);
var hash = await SHA256.Create().ComputeHashAsync(fInfo.OpenRead(), cts);
vf = vf with
{
Metadata = vf.Metadata! with
{
Size = (ulong)fInfo.Length,
Digest = hash.ToHex()
}
};
}
else
{
// move orig file back
File.Move(srcPath, finalPath);
}
}
return vf;
}
/// <inheritdoc />
@ -70,4 +108,4 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
private string MapPath(Guid id) =>
Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
}
}

4
VoidCat/Services/Files/StreamFileStore.cs
View File

@ -53,14 +53,14 @@ public abstract class StreamFileStore
var meta = payload.Meta;
if (payload.IsAppend)
{
meta = meta! with
meta = meta with
{
Size = meta.Size + totalSize
};
}
else
{
meta = meta! with
meta = meta with
{
Uploaded = DateTimeOffset.UtcNow,
EditSecret = Guid.NewGuid(),

51
VoidCat/Services/Files/StripMetadata.cs Normal file
View File

@ -0,0 +1,51 @@
using FFMpegCore;
using FFMpegCore.Enums;
using FFMpegCore.Pipes;
using Newtonsoft.Json;
namespace VoidCat.Services.Files;
/// <summary>
/// Service which utilizes ffmpeg to strip metadata from media
/// </summary>
public class StripMetadata
{
private readonly ILogger<StripMetadata> _logger;
public StripMetadata(ILogger<StripMetadata> logger)
{
_logger = logger;
}
public async Task<bool> TryStripMediaMetadata(string input, string output, CancellationToken cts)
{
try
{
var ffprobe = await FFProbe.AnalyseAsync(input, cancellationToken: cts);
if (ffprobe == default)
{
throw new InvalidOperationException("Could not determine media type with ffprobe");
}
_logger.LogInformation("Stripping content from {type}", ffprobe.Format.FormatName);
var ffmpeg = FFMpegArguments
.FromFileInput(input)
.OutputToFile(output, true, o =>
{
o.WithoutMetadata();
o.CopyChannel();
})
.CancellableThrough(cts);
_logger.LogInformation("Running: {command}", ffmpeg.Arguments);
return await ffmpeg.ProcessAsynchronously();
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not strip metadata");
}
return false;
}
}

1
VoidCat/VoidCat.csproj
View File

@ -16,6 +16,7 @@
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.9.30" />
<PackageReference Include="Dapper" Version="2.0.123" />
<PackageReference Include="FFMpegCore" Version="4.8.0" />
<PackageReference Include="FluentMigrator" Version="3.3.2" />
<PackageReference Include="FluentMigrator.Runner" Version="3.3.2" />
<PackageReference Include="FluentMigrator.Runner.Postgres" Version="3.3.2" />