forked from Kieran/void.cat
Add S3 file storage support
This commit is contained in:
parent
dfeb4d41de
commit
c2c6b92ce6
@ -15,7 +15,8 @@ public class DownloadController : Controller
|
|||||||
private readonly IPaywallStore _paywall;
|
private readonly IPaywallStore _paywall;
|
||||||
private readonly ILogger<DownloadController> _logger;
|
private readonly ILogger<DownloadController> _logger;
|
||||||
|
|
||||||
public DownloadController(IFileStore storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo, IPaywallStore paywall)
|
public DownloadController(IFileStore storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
|
||||||
|
IPaywallStore paywall)
|
||||||
{
|
{
|
||||||
_storage = storage;
|
_storage = storage;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
@ -40,7 +41,7 @@ public class DownloadController : Controller
|
|||||||
var voidFile = await SetupDownload(gid);
|
var voidFile = await SetupDownload(gid);
|
||||||
if (voidFile == default) return;
|
if (voidFile == default) return;
|
||||||
|
|
||||||
var egressReq = new EgressRequest(gid, GetRanges(Request, (long)voidFile!.Metadata!.Size));
|
var egressReq = new EgressRequest(gid, GetRanges(Request, (long) voidFile!.Metadata!.Size));
|
||||||
if (egressReq.Ranges.Count() > 1)
|
if (egressReq.Ranges.Count() > 1)
|
||||||
{
|
{
|
||||||
_logger.LogWarning("Multi-range request not supported!");
|
_logger.LogWarning("Multi-range request not supported!");
|
||||||
@ -52,10 +53,10 @@ public class DownloadController : Controller
|
|||||||
}
|
}
|
||||||
else if (egressReq.Ranges.Count() == 1)
|
else if (egressReq.Ranges.Count() == 1)
|
||||||
{
|
{
|
||||||
Response.StatusCode = (int)HttpStatusCode.PartialContent;
|
Response.StatusCode = (int) HttpStatusCode.PartialContent;
|
||||||
if (egressReq.Ranges.Sum(a => a.Size) == 0)
|
if (egressReq.Ranges.Sum(a => a.Size) == 0)
|
||||||
{
|
{
|
||||||
Response.StatusCode = (int)HttpStatusCode.RequestedRangeNotSatisfiable;
|
Response.StatusCode = (int) HttpStatusCode.RequestedRangeNotSatisfiable;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -91,7 +92,7 @@ public class DownloadController : Controller
|
|||||||
var orderId = Request.Headers.GetHeader("V-OrderId") ?? Request.Query["orderId"];
|
var orderId = Request.Headers.GetHeader("V-OrderId") ?? Request.Query["orderId"];
|
||||||
if (!await IsOrderPaid(orderId))
|
if (!await IsOrderPaid(orderId))
|
||||||
{
|
{
|
||||||
Response.StatusCode = (int)HttpStatusCode.PaymentRequired;
|
Response.StatusCode = (int) HttpStatusCode.PaymentRequired;
|
||||||
return default;
|
return default;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -126,19 +127,9 @@ public class DownloadController : Controller
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var ranges = rangeHeader.Replace("bytes=", string.Empty).Split(",");
|
foreach (var h in RangeRequest.Parse(rangeHeader, totalSize))
|
||||||
foreach (var range in ranges)
|
|
||||||
{
|
{
|
||||||
var rangeValues = range.Split("-");
|
yield return h;
|
||||||
|
|
||||||
long? endByte = null, startByte = 0;
|
|
||||||
if (long.TryParse(rangeValues[1], out var endParsed))
|
|
||||||
endByte = endParsed;
|
|
||||||
|
|
||||||
if (long.TryParse(rangeValues[0], out var startParsed))
|
|
||||||
startByte = startParsed;
|
|
||||||
|
|
||||||
yield return new(totalSize, startByte, endByte);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -41,6 +41,7 @@ namespace VoidCat.Controllers
|
|||||||
Name = Request.Headers.GetHeader("V-Filename"),
|
Name = Request.Headers.GetHeader("V-Filename"),
|
||||||
Description = Request.Headers.GetHeader("V-Description"),
|
Description = Request.Headers.GetHeader("V-Description"),
|
||||||
Digest = Request.Headers.GetHeader("V-Full-Digest"),
|
Digest = Request.Headers.GetHeader("V-Full-Digest"),
|
||||||
|
Size = (ulong?)Request.ContentLength ?? 0UL,
|
||||||
Uploader = uid
|
Uploader = uid
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -84,7 +85,8 @@ namespace VoidCat.Controllers
|
|||||||
{
|
{
|
||||||
Hash = digest,
|
Hash = digest,
|
||||||
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
|
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
|
||||||
Id = gid
|
Id = gid,
|
||||||
|
IsAppend = true
|
||||||
}, HttpContext.RequestAborted);
|
}, HttpContext.RequestAborted);
|
||||||
|
|
||||||
return UploadResult.Success(vf);
|
return UploadResult.Success(vf);
|
||||||
|
@ -2,11 +2,26 @@ using System.IdentityModel.Tokens.Jwt;
|
|||||||
using System.Security.Claims;
|
using System.Security.Claims;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using Amazon;
|
||||||
|
using Amazon.Runtime;
|
||||||
|
using Amazon.S3;
|
||||||
|
|
||||||
namespace VoidCat.Model;
|
namespace VoidCat.Model;
|
||||||
|
|
||||||
public static class Extensions
|
public static class Extensions
|
||||||
{
|
{
|
||||||
|
public static AmazonS3Client CreateClient(this S3BlobConfig c)
|
||||||
|
{
|
||||||
|
return new AmazonS3Client(new BasicAWSCredentials(c.AccessKey, c.SecretKey),
|
||||||
|
new AmazonS3Config
|
||||||
|
{
|
||||||
|
RegionEndpoint = !string.IsNullOrEmpty(c.Region) ? RegionEndpoint.GetBySystemName(c.Region) : null,
|
||||||
|
ServiceURL = c.ServiceUrl?.ToString(),
|
||||||
|
UseHttp = c.ServiceUrl?.Scheme == "http",
|
||||||
|
ForcePathStyle = true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
public static Guid? GetUserId(this HttpContext context)
|
public static Guid? GetUserId(this HttpContext context)
|
||||||
{
|
{
|
||||||
var claimSub = context?.User?.Claims?.FirstOrDefault(a => a.Type == ClaimTypes.NameIdentifier)?.Value;
|
var claimSub = context?.User?.Claims?.FirstOrDefault(a => a.Type == ClaimTypes.NameIdentifier)?.Value;
|
||||||
@ -99,7 +114,7 @@ public static class Extensions
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
private static byte[] FromHex(this string input)
|
public static byte[] FromHex(this string input)
|
||||||
{
|
{
|
||||||
var result = new byte[(input.Length + 1) >> 1];
|
var result = new byte[(input.Length + 1) >> 1];
|
||||||
var lastCell = result.Length - 1;
|
var lastCell = result.Length - 1;
|
||||||
|
@ -6,5 +6,5 @@ public sealed record IngressPayload(Stream InStream, SecretVoidFileMeta Meta)
|
|||||||
public Guid? EditSecret { get; init; }
|
public Guid? EditSecret { get; init; }
|
||||||
public string? Hash { get; init; }
|
public string? Hash { get; init; }
|
||||||
|
|
||||||
public bool IsAppend => EditSecret.HasValue;
|
public bool IsAppend { get; init; }
|
||||||
}
|
}
|
@ -4,6 +4,8 @@ public sealed record RangeRequest(long? TotalSize, long? Start, long? End)
|
|||||||
{
|
{
|
||||||
private const long DefaultBufferSize = 1024L * 512L;
|
private const long DefaultBufferSize = 1024L * 512L;
|
||||||
|
|
||||||
|
public string OriginalString { get; private init; }
|
||||||
|
|
||||||
public long? Size
|
public long? Size
|
||||||
=> Start.HasValue ? (End ?? Math.Min(TotalSize!.Value, Start.Value + DefaultBufferSize)) - Start.Value : End;
|
=> Start.HasValue ? (End ?? Math.Min(TotalSize!.Value, Start.Value + DefaultBufferSize)) - Start.Value : End;
|
||||||
|
|
||||||
@ -16,4 +18,25 @@ public sealed record RangeRequest(long? TotalSize, long? Start, long? End)
|
|||||||
/// <returns></returns>
|
/// <returns></returns>
|
||||||
public string ToContentRange()
|
public string ToContentRange()
|
||||||
=> $"bytes {Start}-{End ?? (Start + Size - 1L)}/{TotalSize?.ToString() ?? "*"}";
|
=> $"bytes {Start}-{End ?? (Start + Size - 1L)}/{TotalSize?.ToString() ?? "*"}";
|
||||||
|
|
||||||
|
public static IEnumerable<RangeRequest> Parse(string header, long totalSize)
|
||||||
|
{
|
||||||
|
var ranges = header.Replace("bytes=", string.Empty).Split(",");
|
||||||
|
foreach (var range in ranges)
|
||||||
|
{
|
||||||
|
var rangeValues = range.Split("-");
|
||||||
|
|
||||||
|
long? endByte = null, startByte = 0;
|
||||||
|
if (long.TryParse(rangeValues[1], out var endParsed))
|
||||||
|
endByte = endParsed;
|
||||||
|
|
||||||
|
if (long.TryParse(rangeValues[0], out var startParsed))
|
||||||
|
startByte = startParsed;
|
||||||
|
|
||||||
|
yield return new(totalSize, startByte, endByte)
|
||||||
|
{
|
||||||
|
OriginalString = range
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
@ -17,11 +17,32 @@ namespace VoidCat.Model
|
|||||||
public SmtpSettings? Smtp { get; init; }
|
public SmtpSettings? Smtp { get; init; }
|
||||||
|
|
||||||
public List<Uri> CorsOrigins { get; init; } = new();
|
public List<Uri> CorsOrigins { get; init; } = new();
|
||||||
|
|
||||||
|
public CloudStorageSettings? CloudStorage { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record TorSettings(Uri TorControl, string PrivateKey, string ControlPassword);
|
public sealed record TorSettings(Uri TorControl, string PrivateKey, string ControlPassword);
|
||||||
|
|
||||||
public sealed record JwtSettings(string Issuer, string Key);
|
public sealed record JwtSettings(string Issuer, string Key);
|
||||||
|
|
||||||
public sealed record SmtpSettings(string Address, string Username, string Password);
|
public sealed record SmtpSettings
|
||||||
|
{
|
||||||
|
public Uri? Server { get; init; }
|
||||||
|
public string? Username { get; init; }
|
||||||
|
public string? Password { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record CloudStorageSettings
|
||||||
|
{
|
||||||
|
public S3BlobConfig? S3 { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record S3BlobConfig
|
||||||
|
{
|
||||||
|
public string? AccessKey { get; init; }
|
||||||
|
public string? SecretKey { get; init; }
|
||||||
|
public Uri? ServiceUrl { get; init; }
|
||||||
|
public string? Region { get; init; }
|
||||||
|
public string? BucketName { get; init; } = "void-cat";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -78,10 +78,7 @@ services.AddAuthorization((opt) => { opt.AddPolicy(Policies.RequireAdmin, (auth)
|
|||||||
services.AddVoidMigrations();
|
services.AddVoidMigrations();
|
||||||
|
|
||||||
// file storage
|
// file storage
|
||||||
services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
|
services.AddStorage(voidSettings);
|
||||||
services.AddTransient<IFileStore, LocalDiskFileStore>();
|
|
||||||
services.AddTransient<IFileInfoManager, FileInfoManager>();
|
|
||||||
services.AddTransient<IUserUploadsStore, UserUploadStore>();
|
|
||||||
|
|
||||||
// stats
|
// stats
|
||||||
services.AddTransient<IAggregateStatsCollector, AggregateStatsCollector>();
|
services.AddTransient<IAggregateStatsCollector, AggregateStatsCollector>();
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
"profiles": {
|
"profiles": {
|
||||||
"VoidCat": {
|
"VoidCat": {
|
||||||
"commandName": "Project",
|
"commandName": "Project",
|
||||||
"launchBrowser": true,
|
"launchBrowser": false,
|
||||||
"environmentVariables": {
|
"environmentVariables": {
|
||||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
},
|
},
|
||||||
|
26
VoidCat/Services/Files/FileStorageStartup.cs
Normal file
26
VoidCat/Services/Files/FileStorageStartup.cs
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
using VoidCat.Model;
|
||||||
|
using VoidCat.Services.Abstractions;
|
||||||
|
using VoidCat.Services.Users;
|
||||||
|
|
||||||
|
namespace VoidCat.Services.Files;
|
||||||
|
|
||||||
|
public static class FileStorageStartup
|
||||||
|
{
|
||||||
|
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
|
||||||
|
{
|
||||||
|
services.AddTransient<IFileInfoManager, FileInfoManager>();
|
||||||
|
services.AddTransient<IUserUploadsStore, UserUploadStore>();
|
||||||
|
|
||||||
|
if (settings.CloudStorage != default)
|
||||||
|
{
|
||||||
|
// cloud storage
|
||||||
|
services.AddSingleton<IFileStore, S3FileStore>();
|
||||||
|
services.AddSingleton<IFileMetadataStore, S3FileMetadataStore>();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
services.AddTransient<IFileStore, LocalDiskFileStore>();
|
||||||
|
services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -6,6 +6,7 @@ namespace VoidCat.Services.Files;
|
|||||||
|
|
||||||
public class LocalDiskFileStore : StreamFileStore, IFileStore
|
public class LocalDiskFileStore : StreamFileStore, IFileStore
|
||||||
{
|
{
|
||||||
|
private const string FilesDir = "files-v1";
|
||||||
private readonly ILogger<LocalDiskFileStore> _logger;
|
private readonly ILogger<LocalDiskFileStore> _logger;
|
||||||
private readonly VoidSettings _settings;
|
private readonly VoidSettings _settings;
|
||||||
private readonly IFileMetadataStore _metadataStore;
|
private readonly IFileMetadataStore _metadataStore;
|
||||||
@ -20,9 +21,10 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
_fileInfo = fileInfo;
|
_fileInfo = fileInfo;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
|
|
||||||
if (!Directory.Exists(_settings.DataDirectory))
|
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
|
||||||
|
if (!Directory.Exists(dir))
|
||||||
{
|
{
|
||||||
Directory.CreateDirectory(_settings.DataDirectory);
|
Directory.CreateDirectory(dir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -99,5 +101,5 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
|||||||
}
|
}
|
||||||
|
|
||||||
private string MapPath(Guid id) =>
|
private string MapPath(Guid id) =>
|
||||||
Path.Join(_settings.DataDirectory, id.ToString());
|
Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
|
||||||
}
|
}
|
62
VoidCat/Services/Files/S3FileMetadataStore.cs
Normal file
62
VoidCat/Services/Files/S3FileMetadataStore.cs
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
using Amazon.S3;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using VoidCat.Model;
|
||||||
|
using VoidCat.Services.Abstractions;
|
||||||
|
|
||||||
|
namespace VoidCat.Services.Files;
|
||||||
|
|
||||||
|
// ReSharper disable once InconsistentNaming
|
||||||
|
public class S3FileMetadataStore : IFileMetadataStore
|
||||||
|
{
|
||||||
|
private readonly ILogger<S3FileMetadataStore> _logger;
|
||||||
|
private readonly AmazonS3Client _client;
|
||||||
|
private readonly S3BlobConfig _config;
|
||||||
|
|
||||||
|
public S3FileMetadataStore(VoidSettings settings, ILogger<S3FileMetadataStore> logger)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_config = settings.CloudStorage!.S3!;
|
||||||
|
_client = _config.CreateClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var obj = await _client.GetObjectAsync(_config.BucketName, ToKey(id));
|
||||||
|
|
||||||
|
using var sr = new StreamReader(obj.ResponseStream);
|
||||||
|
var json = await sr.ReadToEndAsync();
|
||||||
|
return JsonConvert.DeserializeObject<TMeta>(json);
|
||||||
|
}
|
||||||
|
catch (AmazonS3Exception aex)
|
||||||
|
{
|
||||||
|
_logger.LogError(aex, "Failed to get metadata for {Id}, {Error}", id, aex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
return default;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
|
||||||
|
{
|
||||||
|
await _client.PutObjectAsync(new()
|
||||||
|
{
|
||||||
|
BucketName = _config.BucketName,
|
||||||
|
Key = ToKey(id),
|
||||||
|
ContentBody = JsonConvert.SerializeObject(meta),
|
||||||
|
ContentType = "application/json"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask Update(Guid id, SecretVoidFileMeta patch)
|
||||||
|
{
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask Delete(Guid id)
|
||||||
|
{
|
||||||
|
await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ToKey(Guid id) => $"{id}-metadata";
|
||||||
|
}
|
128
VoidCat/Services/Files/S3FileStore.cs
Normal file
128
VoidCat/Services/Files/S3FileStore.cs
Normal file
@ -0,0 +1,128 @@
|
|||||||
|
using Amazon.S3;
|
||||||
|
using Amazon.S3.Model;
|
||||||
|
using VoidCat.Model;
|
||||||
|
using VoidCat.Services.Abstractions;
|
||||||
|
|
||||||
|
namespace VoidCat.Services.Files;
|
||||||
|
|
||||||
|
// ReSharper disable once InconsistentNaming
|
||||||
|
public class S3FileStore : StreamFileStore, IFileStore
|
||||||
|
{
|
||||||
|
private readonly IFileInfoManager _fileInfo;
|
||||||
|
private readonly AmazonS3Client _client;
|
||||||
|
private readonly S3BlobConfig _config;
|
||||||
|
private readonly IAggregateStatsCollector _statsCollector;
|
||||||
|
|
||||||
|
public S3FileStore(VoidSettings settings, IAggregateStatsCollector stats, IFileMetadataStore metadataStore,
|
||||||
|
IUserUploadsStore userUploads, IFileInfoManager fileInfo) : base(stats, metadataStore, userUploads)
|
||||||
|
{
|
||||||
|
_fileInfo = fileInfo;
|
||||||
|
_statsCollector = stats;
|
||||||
|
_config = settings.CloudStorage!.S3!;
|
||||||
|
_client = _config.CreateClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
|
||||||
|
{
|
||||||
|
var req = new PutObjectRequest
|
||||||
|
{
|
||||||
|
BucketName = _config.BucketName,
|
||||||
|
Key = payload.Id.ToString(),
|
||||||
|
InputStream = payload.InStream,
|
||||||
|
ContentType = "application/octet-stream",
|
||||||
|
AutoResetStreamPosition = false,
|
||||||
|
AutoCloseStream = false,
|
||||||
|
ChecksumAlgorithm = ChecksumAlgorithm.SHA256,
|
||||||
|
ChecksumSHA256 = payload.Hash != default ? Convert.ToBase64String(payload.Hash!.FromHex()) : null,
|
||||||
|
StreamTransferProgress = (s, e) =>
|
||||||
|
{
|
||||||
|
_statsCollector.TrackIngress(payload.Id, (ulong) e.IncrementTransferred)
|
||||||
|
.GetAwaiter().GetResult();
|
||||||
|
},
|
||||||
|
Headers =
|
||||||
|
{
|
||||||
|
ContentLength = (long)payload.Meta.Size
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var r = await _client.PutObjectAsync(req, cts);
|
||||||
|
return await HandleCompletedUpload(payload, r.ChecksumSHA256, payload.Meta.Size);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
|
||||||
|
{
|
||||||
|
var req = new GetObjectRequest()
|
||||||
|
{
|
||||||
|
BucketName = _config.BucketName,
|
||||||
|
Key = request.Id.ToString()
|
||||||
|
};
|
||||||
|
if (request.Ranges.Any())
|
||||||
|
{
|
||||||
|
var r = request.Ranges.First();
|
||||||
|
req.ByteRange = new ByteRange(r.OriginalString);
|
||||||
|
}
|
||||||
|
|
||||||
|
var obj = await _client.GetObjectAsync(req, cts);
|
||||||
|
await EgressFull(request.Id, obj.ResponseStream, outStream, cts);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var objs = await _client.ListObjectsV2Async(new ListObjectsV2Request()
|
||||||
|
{
|
||||||
|
BucketName = _config.BucketName,
|
||||||
|
});
|
||||||
|
|
||||||
|
var files = (request.SortBy, request.SortOrder) switch
|
||||||
|
{
|
||||||
|
(PagedSortBy.Date, PageSortOrder.Asc) => objs.S3Objects.OrderBy(a => a.LastModified),
|
||||||
|
(PagedSortBy.Date, PageSortOrder.Dsc) => objs.S3Objects.OrderByDescending(a => a.LastModified),
|
||||||
|
(PagedSortBy.Name, PageSortOrder.Asc) => objs.S3Objects.OrderBy(a => a.Key),
|
||||||
|
(PagedSortBy.Name, PageSortOrder.Dsc) => objs.S3Objects.OrderByDescending(a => a.Key),
|
||||||
|
(PagedSortBy.Size, PageSortOrder.Asc) => objs.S3Objects.OrderBy(a => a.Size),
|
||||||
|
(PagedSortBy.Size, PageSortOrder.Dsc) => objs.S3Objects.OrderByDescending(a => a.Size),
|
||||||
|
_ => objs.S3Objects.AsEnumerable()
|
||||||
|
};
|
||||||
|
|
||||||
|
async IAsyncEnumerable<PublicVoidFile> EnumerateFiles(IEnumerable<S3Object> page)
|
||||||
|
{
|
||||||
|
foreach (var item in page)
|
||||||
|
{
|
||||||
|
if (!Guid.TryParse(item.Key, out var gid)) continue;
|
||||||
|
|
||||||
|
var obj = await _fileInfo.Get(gid);
|
||||||
|
if (obj != default)
|
||||||
|
{
|
||||||
|
yield return obj;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new()
|
||||||
|
{
|
||||||
|
Page = request.Page,
|
||||||
|
PageSize = request.PageSize,
|
||||||
|
TotalResults = files.Count(),
|
||||||
|
Results = EnumerateFiles(files.Skip(request.PageSize * request.Page).Take(request.PageSize))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
catch (AmazonS3Exception aex)
|
||||||
|
{
|
||||||
|
// ignore
|
||||||
|
return new()
|
||||||
|
{
|
||||||
|
Page = request.Page,
|
||||||
|
PageSize = request.PageSize,
|
||||||
|
TotalResults = 0,
|
||||||
|
Results = AsyncEnumerable.Empty<PublicVoidFile>()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask DeleteFile(Guid id)
|
||||||
|
{
|
||||||
|
await _client.DeleteObjectAsync(_config.BucketName, id.ToString());
|
||||||
|
}
|
||||||
|
}
|
@ -34,7 +34,7 @@ public abstract class StreamFileStore
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected async ValueTask<PrivateVoidFile> IngressToStream(Stream stream, IngressPayload payload,
|
protected async ValueTask<PrivateVoidFile> IngressToStream(Stream outStream, IngressPayload payload,
|
||||||
CancellationToken cts)
|
CancellationToken cts)
|
||||||
{
|
{
|
||||||
var id = payload.Id;
|
var id = payload.Id;
|
||||||
@ -47,17 +47,23 @@ public abstract class StreamFileStore
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var (total, hash) = await IngressInternal(id, payload.InStream, stream, cts);
|
var (total, hash) = await IngressInternal(id, payload.InStream, outStream, cts);
|
||||||
if (payload.Hash != null && !hash.Equals(payload.Hash, StringComparison.InvariantCultureIgnoreCase))
|
if (payload.Hash != null && !hash.Equals(payload.Hash, StringComparison.InvariantCultureIgnoreCase))
|
||||||
{
|
{
|
||||||
throw new CryptographicException("Invalid file hash");
|
throw new CryptographicException("Invalid file hash");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return await HandleCompletedUpload(payload, hash, total);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected async Task<PrivateVoidFile> HandleCompletedUpload(IngressPayload payload, string hash, ulong totalSize)
|
||||||
|
{
|
||||||
|
var meta = payload.Meta;
|
||||||
if (payload.IsAppend)
|
if (payload.IsAppend)
|
||||||
{
|
{
|
||||||
meta = meta! with
|
meta = meta! with
|
||||||
{
|
{
|
||||||
Size = meta.Size + total
|
Size = meta.Size + totalSize
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -67,14 +73,14 @@ public abstract class StreamFileStore
|
|||||||
Digest = hash,
|
Digest = hash,
|
||||||
Uploaded = DateTimeOffset.UtcNow,
|
Uploaded = DateTimeOffset.UtcNow,
|
||||||
EditSecret = Guid.NewGuid(),
|
EditSecret = Guid.NewGuid(),
|
||||||
Size = total
|
Size = totalSize
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
await _metadataStore.Set(id, meta);
|
await _metadataStore.Set(payload.Id, meta);
|
||||||
var vf = new PrivateVoidFile()
|
var vf = new PrivateVoidFile()
|
||||||
{
|
{
|
||||||
Id = id,
|
Id = payload.Id,
|
||||||
Metadata = meta
|
Metadata = meta
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -86,7 +92,7 @@ public abstract class StreamFileStore
|
|||||||
return vf;
|
return vf;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream fs, CancellationToken cts)
|
private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream outStream, CancellationToken cts)
|
||||||
{
|
{
|
||||||
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
||||||
var total = 0UL;
|
var total = 0UL;
|
||||||
@ -103,7 +109,7 @@ public abstract class StreamFileStore
|
|||||||
|
|
||||||
var totalRead = readLength + offset;
|
var totalRead = readLength + offset;
|
||||||
var buf = buffer.Memory[..totalRead];
|
var buf = buffer.Memory[..totalRead];
|
||||||
await fs.WriteAsync(buf, cts);
|
await outStream.WriteAsync(buf, cts);
|
||||||
await _stats.TrackIngress(id, (ulong) buf.Length);
|
await _stats.TrackIngress(id, (ulong) buf.Length);
|
||||||
sha.TransformBlock(buf.ToArray(), 0, buf.Length, null, 0);
|
sha.TransformBlock(buf.ToArray(), 0, buf.Length, null, 0);
|
||||||
total += (ulong) buf.Length;
|
total += (ulong) buf.Length;
|
||||||
@ -111,10 +117,10 @@ public abstract class StreamFileStore
|
|||||||
}
|
}
|
||||||
|
|
||||||
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||||
return (total, BitConverter.ToString(sha.Hash!).Replace("-", string.Empty));
|
return (total, sha.Hash!.ToHex());
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task EgressFull(Guid id, Stream inStream, Stream outStream,
|
protected async Task EgressFull(Guid id, Stream inStream, Stream outStream,
|
||||||
CancellationToken cts)
|
CancellationToken cts)
|
||||||
{
|
{
|
||||||
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
|
||||||
|
@ -1,116 +0,0 @@
|
|||||||
using Newtonsoft.Json;
|
|
||||||
using VoidCat.Model;
|
|
||||||
|
|
||||||
namespace VoidCat.Services.Migrations;
|
|
||||||
|
|
||||||
public class MigrateMetadata_20220217 : IMigration
|
|
||||||
{
|
|
||||||
private const string MetadataDir = "metadata";
|
|
||||||
private const string MetadataV2Dir = "metadata-v2";
|
|
||||||
private readonly ILogger<MigrateMetadata_20220217> _logger;
|
|
||||||
private readonly VoidSettings _settings;
|
|
||||||
|
|
||||||
public MigrateMetadata_20220217(VoidSettings settings, ILogger<MigrateMetadata_20220217> log)
|
|
||||||
{
|
|
||||||
_settings = settings;
|
|
||||||
_logger = log;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async ValueTask Migrate()
|
|
||||||
{
|
|
||||||
var newMeta = Path.Combine(_settings.DataDirectory, MetadataV2Dir);
|
|
||||||
if (!Directory.Exists(newMeta))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(newMeta);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var fe in Directory.EnumerateFiles(_settings.DataDirectory))
|
|
||||||
{
|
|
||||||
var filename = Path.GetFileNameWithoutExtension(fe);
|
|
||||||
if (!Guid.TryParse(filename, out var id)) continue;
|
|
||||||
|
|
||||||
var fp = MapMeta(id);
|
|
||||||
if (File.Exists(fp))
|
|
||||||
{
|
|
||||||
_logger.LogInformation("Migrating metadata for {file}", fp);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var oldJson = await File.ReadAllTextAsync(fp);
|
|
||||||
if(!oldJson.Contains("\"Metadata\":")) continue; // old format should contain "Metadata":
|
|
||||||
|
|
||||||
var old = JsonConvert.DeserializeObject<InternalVoidFile>(oldJson);
|
|
||||||
var newObj = new PrivateVoidFile()
|
|
||||||
{
|
|
||||||
Id = old!.Id,
|
|
||||||
Metadata = new()
|
|
||||||
{
|
|
||||||
Name = old.Metadata!.Name,
|
|
||||||
Description = old.Metadata.Description,
|
|
||||||
Uploaded = old.Uploaded,
|
|
||||||
MimeType = old.Metadata.MimeType,
|
|
||||||
EditSecret = old.EditSecret,
|
|
||||||
Size = old.Size
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
await File.WriteAllTextAsync(MapV2Meta(id), JsonConvert.SerializeObject(newObj));
|
|
||||||
|
|
||||||
// delete old metadata
|
|
||||||
File.Delete(fp);
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.LogError(ex, ex.Message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private string MapMeta(Guid id) =>
|
|
||||||
Path.ChangeExtension(Path.Join(_settings.DataDirectory, MetadataDir, id.ToString()), ".json");
|
|
||||||
private string MapV2Meta(Guid id) =>
|
|
||||||
Path.ChangeExtension(Path.Join(_settings.DataDirectory, MetadataV2Dir, id.ToString()), ".json");
|
|
||||||
|
|
||||||
private record VoidFile
|
|
||||||
{
|
|
||||||
[JsonConverter(typeof(Base58GuidConverter))]
|
|
||||||
public Guid Id { get; init; }
|
|
||||||
|
|
||||||
public VoidFileMeta? Metadata { get; set; }
|
|
||||||
|
|
||||||
public ulong Size { get; init; }
|
|
||||||
|
|
||||||
public DateTimeOffset Uploaded { get; init; }
|
|
||||||
}
|
|
||||||
|
|
||||||
private record InternalVoidFile : VoidFile
|
|
||||||
{
|
|
||||||
[JsonConverter(typeof(Base58GuidConverter))]
|
|
||||||
public Guid EditSecret { get; init; }
|
|
||||||
}
|
|
||||||
|
|
||||||
private record VoidFileMeta
|
|
||||||
{
|
|
||||||
public string? Name { get; init; }
|
|
||||||
|
|
||||||
public string? Description { get; init; }
|
|
||||||
|
|
||||||
public string? MimeType { get; init; }
|
|
||||||
}
|
|
||||||
|
|
||||||
private record NewVoidFileMeta
|
|
||||||
{
|
|
||||||
public string? Name { get; init; }
|
|
||||||
public ulong Size { get; init; }
|
|
||||||
public DateTimeOffset Uploaded { get; init; } = DateTimeOffset.UtcNow;
|
|
||||||
public string? Description { get; init; }
|
|
||||||
public string? MimeType { get; init; }
|
|
||||||
public string? Digest { get; init; }
|
|
||||||
}
|
|
||||||
|
|
||||||
private record NewSecretVoidFileMeta : NewVoidFileMeta
|
|
||||||
{
|
|
||||||
[JsonConverter(typeof(Base58GuidConverter))]
|
|
||||||
public Guid EditSecret { get; init; }
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,45 +0,0 @@
|
|||||||
using Newtonsoft.Json;
|
|
||||||
using VoidCat.Model;
|
|
||||||
|
|
||||||
namespace VoidCat.Services.Migrations;
|
|
||||||
|
|
||||||
public class FixMigration_20220218 : MetadataMigrator<WrongFile, WrongMeta>
|
|
||||||
{
|
|
||||||
public FixMigration_20220218(VoidSettings settings, ILogger<MetadataMigrator<WrongFile, WrongMeta>> logger) : base(settings, logger)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override string OldPath => "metadata-v2";
|
|
||||||
protected override string NewPath => "metadata-v3";
|
|
||||||
|
|
||||||
protected override bool ShouldMigrate(string json)
|
|
||||||
{
|
|
||||||
var metaBase = JsonConvert.DeserializeObject<WrongFile>(json);
|
|
||||||
return metaBase?.Metadata?.Version == 2 && metaBase?.Metadata?.Size > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override WrongMeta MigrateModel(WrongFile old)
|
|
||||||
{
|
|
||||||
return old.Metadata!;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public class WrongFile
|
|
||||||
{
|
|
||||||
[JsonConverter(typeof(Base58GuidConverter))]
|
|
||||||
public Guid Id { get; init; }
|
|
||||||
public WrongMeta? Metadata { get; init; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class WrongMeta
|
|
||||||
{
|
|
||||||
public int Version { get; init; } = 2;
|
|
||||||
public string? Name { get; init; }
|
|
||||||
public ulong Size { get; init; }
|
|
||||||
public DateTimeOffset Uploaded { get; init; } = DateTimeOffset.UtcNow;
|
|
||||||
public string? Description { get; init; }
|
|
||||||
public string? MimeType { get; init; }
|
|
||||||
public string? Digest { get; init; }
|
|
||||||
[JsonConverter(typeof(Base58GuidConverter))]
|
|
||||||
public Guid EditSecret { get; init; }
|
|
||||||
}
|
|
@ -7,10 +7,7 @@ public interface IMigration
|
|||||||
|
|
||||||
public static class Migrations
|
public static class Migrations
|
||||||
{
|
{
|
||||||
public static IServiceCollection AddVoidMigrations(this IServiceCollection svc)
|
public static void AddVoidMigrations(this IServiceCollection svc)
|
||||||
{
|
{
|
||||||
svc.AddTransient<IMigration, MigrateMetadata_20220217>();
|
|
||||||
svc.AddTransient<IMigration, FixMigration_20220218>();
|
|
||||||
return svc;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -12,6 +12,7 @@
|
|||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<PackageReference Include="AWSSDK.S3" Version="3.7.8.2" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
|
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
|
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
|
||||||
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
|
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
|
||||||
|
Loading…
Reference in New Issue
Block a user