267 lines
9.4 KiB
C#
267 lines
9.4 KiB
C#
using System.Buffers;
|
|
using System.Security.Cryptography;
|
|
using VoidCat.Model;
|
|
using VoidCat.Model.Exceptions;
|
|
using VoidCat.Services.Abstractions;
|
|
|
|
namespace VoidCat.Services.Files;
|
|
|
|
public class LocalDiskFileStore : IFileStore
|
|
{
|
|
private const int BufferSize = 1_048_576;
|
|
private readonly ILogger<LocalDiskFileStore> _logger;
|
|
private readonly VoidSettings _settings;
|
|
private readonly IAggregateStatsCollector _stats;
|
|
private readonly IFileMetadataStore _metadataStore;
|
|
private readonly IPaywallStore _paywallStore;
|
|
private readonly IStatsReporter _statsReporter;
|
|
|
|
public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats,
|
|
IFileMetadataStore metadataStore, IPaywallStore paywallStore, IStatsReporter statsReporter)
|
|
{
|
|
_settings = settings;
|
|
_stats = stats;
|
|
_metadataStore = metadataStore;
|
|
_paywallStore = paywallStore;
|
|
_statsReporter = statsReporter;
|
|
_logger = logger;
|
|
|
|
if (!Directory.Exists(_settings.DataDirectory))
|
|
{
|
|
Directory.CreateDirectory(_settings.DataDirectory);
|
|
}
|
|
}
|
|
|
|
public async ValueTask<PublicVoidFile?> Get(Guid id)
|
|
{
|
|
var meta = _metadataStore.GetPublic(id);
|
|
var paywall = _paywallStore.GetConfig(id);
|
|
var bandwidth = _statsReporter.GetBandwidth(id);
|
|
await Task.WhenAll(meta.AsTask(), paywall.AsTask(), bandwidth.AsTask());
|
|
|
|
return new()
|
|
{
|
|
Id = id,
|
|
Metadata = meta.Result,
|
|
Paywall = paywall.Result,
|
|
Bandwidth = bandwidth.Result
|
|
};
|
|
}
|
|
|
|
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
|
|
{
|
|
var path = MapPath(request.Id);
|
|
if (!File.Exists(path)) throw new VoidFileNotFoundException(request.Id);
|
|
|
|
await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
|
|
if (request.Ranges.Any())
|
|
{
|
|
await EgressRanges(request.Id, request.Ranges, fs, outStream, cts);
|
|
}
|
|
else
|
|
{
|
|
await EgressFull(request.Id, fs, outStream, cts);
|
|
}
|
|
}
|
|
|
|
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
|
|
{
|
|
var id = payload.Id ?? Guid.NewGuid();
|
|
var fPath = MapPath(id);
|
|
SecretVoidFileMeta? vf = null;
|
|
if (payload.IsAppend)
|
|
{
|
|
vf = await _metadataStore.Get(payload.Id!.Value);
|
|
if (vf?.EditSecret != null && vf.EditSecret != payload.EditSecret)
|
|
{
|
|
throw new VoidNotAllowedException("Edit secret incorrect!");
|
|
}
|
|
}
|
|
|
|
// open file
|
|
await using var fsTemp = new FileStream(fPath,
|
|
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.Write);
|
|
|
|
var (total, hash) = await IngressInternal(id, payload.InStream, fsTemp, cts);
|
|
|
|
if (!hash.Equals(payload.Hash, StringComparison.InvariantCultureIgnoreCase))
|
|
{
|
|
throw new CryptographicException("Invalid file hash");
|
|
}
|
|
|
|
if (payload.IsAppend)
|
|
{
|
|
vf = vf! with
|
|
{
|
|
Size = vf.Size + total
|
|
};
|
|
}
|
|
else
|
|
{
|
|
vf = new SecretVoidFileMeta()
|
|
{
|
|
Name = payload.Meta.Name,
|
|
Description = payload.Meta.Description,
|
|
Digest = payload.Meta.Digest,
|
|
MimeType = payload.Meta.MimeType,
|
|
Uploader = payload.Meta.Uploader,
|
|
Uploaded = DateTimeOffset.UtcNow,
|
|
EditSecret = Guid.NewGuid(),
|
|
Size = total
|
|
};
|
|
}
|
|
|
|
|
|
await _metadataStore.Set(id, vf);
|
|
return new()
|
|
{
|
|
Id = id,
|
|
Metadata = vf
|
|
};
|
|
}
|
|
|
|
public ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request)
|
|
{
|
|
var files = Directory.EnumerateFiles(_settings.DataDirectory)
|
|
.Where(a => !Path.HasExtension(a));
|
|
|
|
files = (request.SortBy, request.SortOrder) switch
|
|
{
|
|
(PagedSortBy.Id, PageSortOrder.Asc) => files.OrderBy(a =>
|
|
Guid.TryParse(Path.GetFileNameWithoutExtension(a), out var g) ? g : Guid.Empty),
|
|
(PagedSortBy.Id, PageSortOrder.Dsc) => files.OrderByDescending(a =>
|
|
Guid.TryParse(Path.GetFileNameWithoutExtension(a), out var g) ? g : Guid.Empty),
|
|
(PagedSortBy.Name, PageSortOrder.Asc) => files.OrderBy(Path.GetFileNameWithoutExtension),
|
|
(PagedSortBy.Name, PageSortOrder.Dsc) => files.OrderByDescending(Path.GetFileNameWithoutExtension),
|
|
(PagedSortBy.Size, PageSortOrder.Asc) => files.OrderBy(a => new FileInfo(a).Length),
|
|
(PagedSortBy.Size, PageSortOrder.Dsc) => files.OrderByDescending(a => new FileInfo(a).Length),
|
|
(PagedSortBy.Date, PageSortOrder.Asc) => files.OrderBy(File.GetCreationTimeUtc),
|
|
(PagedSortBy.Date, PageSortOrder.Dsc) => files.OrderByDescending(File.GetCreationTimeUtc),
|
|
_ => files
|
|
};
|
|
|
|
async IAsyncEnumerable<PublicVoidFile> EnumeratePage(IEnumerable<string> page)
|
|
{
|
|
foreach (var file in page)
|
|
{
|
|
if (!Guid.TryParse(Path.GetFileNameWithoutExtension(file), out var gid)) continue;
|
|
|
|
var loaded = await Get(gid);
|
|
if (loaded != default)
|
|
{
|
|
yield return loaded;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ValueTask.FromResult(new PagedResult<PublicVoidFile>()
|
|
{
|
|
Page = request.Page,
|
|
PageSize = request.PageSize,
|
|
TotalResults = files.Count(),
|
|
Results = EnumeratePage(files.Skip(request.PageSize * request.Page).Take(request.PageSize))
|
|
});
|
|
}
|
|
|
|
public async ValueTask DeleteFile(Guid id)
|
|
{
|
|
var fp = MapPath(id);
|
|
if (File.Exists(fp))
|
|
{
|
|
_logger.LogInformation("Deleting file: {Path}", fp);
|
|
File.Delete(fp);
|
|
}
|
|
|
|
await _metadataStore.Delete(id);
|
|
}
|
|
|
|
private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream fs, CancellationToken cts)
|
|
{
|
|
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
|
var total = 0UL;
|
|
int readLength = 0, offset = 0;
|
|
var sha = SHA256.Create();
|
|
while ((readLength = await ingress.ReadAsync(buffer.Memory[offset..], cts)) > 0 || offset != 0)
|
|
{
|
|
if (readLength != 0 && offset + readLength < buffer.Memory.Length)
|
|
{
|
|
// read until buffer full
|
|
offset += readLength;
|
|
continue;
|
|
}
|
|
|
|
var totalRead = readLength + offset;
|
|
var buf = buffer.Memory[..totalRead];
|
|
await fs.WriteAsync(buf, cts);
|
|
await _stats.TrackIngress(id, (ulong)buf.Length);
|
|
sha.TransformBlock(buf.ToArray(), 0, buf.Length, null, 0);
|
|
total += (ulong)buf.Length;
|
|
offset = 0;
|
|
}
|
|
|
|
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
|
return (total, BitConverter.ToString(sha.Hash!).Replace("-", string.Empty));
|
|
}
|
|
|
|
private async Task EgressFull(Guid id, FileStream fileStream, Stream outStream,
|
|
CancellationToken cts)
|
|
{
|
|
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
|
int readLength = 0, offset = 0;
|
|
while ((readLength = await fileStream.ReadAsync(buffer.Memory[offset..], cts)) > 0 || offset != 0)
|
|
{
|
|
if (readLength != 0 && offset + readLength < buffer.Memory.Length)
|
|
{
|
|
// read until buffer full
|
|
offset += readLength;
|
|
continue;
|
|
}
|
|
|
|
var fullSize = readLength + offset;
|
|
await outStream.WriteAsync(buffer.Memory[..fullSize], cts);
|
|
await _stats.TrackEgress(id, (ulong)fullSize);
|
|
await outStream.FlushAsync(cts);
|
|
offset = 0;
|
|
}
|
|
}
|
|
|
|
private async Task EgressRanges(Guid id, IEnumerable<RangeRequest> ranges, FileStream fileStream, Stream outStream,
|
|
CancellationToken cts)
|
|
{
|
|
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
|
foreach (var range in ranges)
|
|
{
|
|
fileStream.Seek(range.Start ?? range.End ?? 0L,
|
|
range.Start.HasValue ? SeekOrigin.Begin : SeekOrigin.End);
|
|
|
|
int readLength = 0, offset = 0;
|
|
var dataRemaining = range.Size ?? 0L;
|
|
while ((readLength = await fileStream.ReadAsync(buffer.Memory[offset..], cts)) > 0 || offset != 0)
|
|
{
|
|
if (readLength != 0 && offset + readLength < buffer.Memory.Length)
|
|
{
|
|
// read until buffer full
|
|
offset += readLength;
|
|
continue;
|
|
}
|
|
|
|
var fullSize = readLength + offset;
|
|
var toWrite = Math.Min(fullSize, dataRemaining);
|
|
await outStream.WriteAsync(buffer.Memory[..(int)toWrite], cts);
|
|
await _stats.TrackEgress(id, (ulong)toWrite);
|
|
await outStream.FlushAsync(cts);
|
|
dataRemaining -= toWrite;
|
|
offset = 0;
|
|
|
|
if (dataRemaining == 0)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private string MapPath(Guid id) =>
|
|
Path.Join(_settings.DataDirectory, id.ToString());
|
|
}
|