From e51672352f8875d888f6524ce64fff41f6eaa04b Mon Sep 17 00:00:00 2001 From: Kieran Date: Thu, 24 Aug 2023 10:51:07 +0100 Subject: [PATCH] FilesV2 + Cleanup --- VoidCat/Services/Abstractions/IFileStore.cs | 7 ++ VoidCat/Services/Files/FileSystemFactory.cs | 10 ++- .../Services/Files/LocalDiskFileStorage.cs | 32 +++++--- .../Files/PostgresFileMetadataStore.cs | 18 +++-- VoidCat/Services/Files/S3FileStore.cs | 24 +++++- .../Migrations/CleanupLocalDiskStore.cs | 80 +++++++++++++++++++ VoidCat/Services/Migrations/FileStoreV2.cs | 42 ++++++++++ VoidCat/VoidStartup.cs | 3 + VoidCat/appsettings.json | 6 +- 9 files changed, 195 insertions(+), 27 deletions(-) create mode 100644 VoidCat/Services/Migrations/CleanupLocalDiskStore.cs create mode 100644 VoidCat/Services/Migrations/FileStoreV2.cs diff --git a/VoidCat/Services/Abstractions/IFileStore.cs b/VoidCat/Services/Abstractions/IFileStore.cs index 6326437..8dfc9d8 100644 --- a/VoidCat/Services/Abstractions/IFileStore.cs +++ b/VoidCat/Services/Abstractions/IFileStore.cs @@ -12,6 +12,13 @@ public interface IFileStore /// Return key for named instance /// string? Key { get; } + + /// + /// Check if a file exists in the store + /// + /// + /// + ValueTask Exists(Guid id); /// /// Ingress a file into the system (Upload) diff --git a/VoidCat/Services/Files/FileSystemFactory.cs b/VoidCat/Services/Files/FileSystemFactory.cs index fc79dfb..dcb1962 100644 --- a/VoidCat/Services/Files/FileSystemFactory.cs +++ b/VoidCat/Services/Files/FileSystemFactory.cs @@ -35,6 +35,12 @@ public class FileStoreFactory : IFileStore /// public string? Key => null; + public async ValueTask Exists(Guid id) + { + var store = await GetStore(id); + return await store.Exists(id); + } + /// public ValueTask Ingress(IngressPayload payload, CancellationToken cts) { @@ -43,7 +49,7 @@ public class FileStoreFactory : IFileStore { throw new InvalidOperationException($"Cannot find store '{payload.Meta.Storage}'"); } - + return store.Ingress(payload, cts); } @@ -53,7 +59,7 @@ public class FileStoreFactory : IFileStore var store = await GetStore(request.Id); await store.Egress(request, outStream, cts); } - + /// public async ValueTask StartEgress(EgressRequest request) { diff --git a/VoidCat/Services/Files/LocalDiskFileStorage.cs b/VoidCat/Services/Files/LocalDiskFileStorage.cs index a3c6b8e..62763a1 100644 --- a/VoidCat/Services/Files/LocalDiskFileStorage.cs +++ b/VoidCat/Services/Files/LocalDiskFileStorage.cs @@ -8,7 +8,6 @@ namespace VoidCat.Services.Files; /// public class LocalDiskFileStore : StreamFileStore, IFileStore { - private const string FilesDir = "files-v1"; private readonly VoidSettings _settings; private readonly CompressContent _stripMetadata; @@ -17,12 +16,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore { _settings = settings; _stripMetadata = stripMetadata; - - var dir = Path.Combine(_settings.DataDirectory, FilesDir); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } } /// @@ -41,10 +34,15 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore /// public string Key => "local-disk"; - /// + public ValueTask Exists(Guid id) + { + var path = MapPath(id); + return ValueTask.FromResult(File.Exists(path)); + } + public async ValueTask Ingress(IngressPayload payload, CancellationToken cts) { - var finalPath = MapPath(payload.Id); + var finalPath = MapCreatePath(payload.Id); await using var fsTemp = new FileStream(finalPath, payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite); @@ -97,7 +95,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore return vf; } - /// public ValueTask DeleteFile(Guid id) { var fp = MapPath(id); @@ -109,7 +106,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore return ValueTask.CompletedTask; } - /// public ValueTask Open(EgressRequest request, CancellationToken cts) { var path = MapPath(request.Id); @@ -118,6 +114,18 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore return ValueTask.FromResult(new FileStream(path, FileMode.Open, FileAccess.Read)); } + private string MapCreatePath(Guid id) + { + var path = MapPath(id); + var dir = Path.GetDirectoryName(path); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir!); + } + + return path; + } + private string MapPath(Guid id) => - Path.Join(_settings.DataDirectory, FilesDir, id.ToString()); + Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString()); } diff --git a/VoidCat/Services/Files/PostgresFileMetadataStore.cs b/VoidCat/Services/Files/PostgresFileMetadataStore.cs index dcbcb57..405dcbf 100644 --- a/VoidCat/Services/Files/PostgresFileMetadataStore.cs +++ b/VoidCat/Services/Files/PostgresFileMetadataStore.cs @@ -66,12 +66,8 @@ public class PostgresFileMetadataStore : IFileMetadataStore /// public async ValueTask> ListFiles(PagedRequest request) { - var count = await _db.Files.CountAsync(); - - async IAsyncEnumerable Enumerate() + IQueryable MakeQuery(VoidContext db) { - using var scope = _scopeFactory.CreateScope(); - var db = scope.ServiceProvider.GetRequiredService(); var q = db.Files.AsNoTracking().AsQueryable(); switch (request.SortBy, request.SortOrder) { @@ -101,7 +97,15 @@ public class PostgresFileMetadataStore : IFileMetadataStore break; } - await foreach (var r in q.Skip(request.Page * request.PageSize).Take(request.PageSize).AsAsyncEnumerable()) + return q.Skip(request.Page * request.PageSize).Take(request.PageSize); + } + + async IAsyncEnumerable Enumerate() + { + using var scope = _scopeFactory.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + await foreach (var r in MakeQuery(db).AsAsyncEnumerable()) { yield return r; } @@ -109,7 +113,7 @@ public class PostgresFileMetadataStore : IFileMetadataStore return new() { - TotalResults = count, + TotalResults = await MakeQuery(_db).CountAsync(), PageSize = request.PageSize, Page = request.Page, Results = Enumerate() diff --git a/VoidCat/Services/Files/S3FileStore.cs b/VoidCat/Services/Files/S3FileStore.cs index 1cd8f3d..2dd9ef3 100644 --- a/VoidCat/Services/Files/S3FileStore.cs +++ b/VoidCat/Services/Files/S3FileStore.cs @@ -24,10 +24,26 @@ public class S3FileStore : StreamFileStore, IFileStore _client = _config.CreateClient(); } - /// public string Key => _config.Name; - /// + public async ValueTask Exists(Guid id) + { + try + { + await _client.GetObjectMetadataAsync(new GetObjectMetadataRequest() + { + BucketName = _config.BucketName, + Key = id.ToString() + }); + + return true; + } + catch + { + return false; + } + } + public async ValueTask Ingress(IngressPayload payload, CancellationToken cts) { if (payload.IsMultipart) return await IngressMultipart(payload, cts); @@ -206,7 +222,7 @@ public class S3FileStore : StreamFileStore, IFileStore InputStream = fsTmp, DisablePayloadSigning = _config.DisablePayloadSigning }; - + var bodyResponse = await _client.UploadPartAsync(mBody, cts); if (bodyResponse.HttpStatusCode != HttpStatusCode.OK) { @@ -241,7 +257,7 @@ public class S3FileStore : StreamFileStore, IFileStore throw new Exception("Upload failed"); } } - + return HandleCompletedUpload(payload, segmentLength); } diff --git a/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs b/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs new file mode 100644 index 0000000..1de4f0a --- /dev/null +++ b/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs @@ -0,0 +1,80 @@ +using VoidCat.Model; +using VoidCat.Services.Abstractions; +using VoidCat.Services.Files; + +namespace VoidCat.Services.Migrations; + +public class CleanupLocalDiskStore : IMigration +{ + private readonly VoidSettings _settings; + private readonly IFileMetadataStore _metadataStore; + private readonly IFileStore _fileStore; + private readonly ILogger _logger; + + public CleanupLocalDiskStore(VoidSettings settings, IFileMetadataStore store, ILogger logger, + IFileStore fileStore) + { + _settings = settings; + _metadataStore = store; + _logger = logger; + _fileStore = fileStore; + } + + public int Order => 3; + public async ValueTask Migrate(string[] args) + { + if (_fileStore is not LocalDiskFileStore) + { + return IMigration.MigrationResult.Skipped; + } + + await CleanupDisk(); + await CleanupMetadata(); + + return IMigration.MigrationResult.Completed; + } + + private async Task CleanupDisk() + { + var baseDir = Path.Join(_settings.DataDirectory, "files-v2"); + foreach (var path in Directory.EnumerateFiles(baseDir, "*.*", SearchOption.AllDirectories)) + { + if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id)) + { + continue; + } + + var meta = await _metadataStore.Get(id); + if (meta == default) + { + _logger.LogInformation("Deleting unmapped file {Path}", path); + File.Delete(path); + } + } + } + + private async Task CleanupMetadata() + { + var page = 0; + while (true) + { + var deleting = new List(); + var fileList = await _metadataStore.ListFiles(new(page++, 1000)); + if (fileList.TotalResults == 0) break; + + await foreach (var md in fileList.Results) + { + if (!await _fileStore.Exists(md.Id)) + { + deleting.Add(md.Id); + } + } + + foreach (var toDelete in deleting) + { + _logger.LogInformation("Deleting metadata with missing file {Id}", toDelete); + await _metadataStore.Delete(toDelete); + } + } + } +} diff --git a/VoidCat/Services/Migrations/FileStoreV2.cs b/VoidCat/Services/Migrations/FileStoreV2.cs new file mode 100644 index 0000000..092feb7 --- /dev/null +++ b/VoidCat/Services/Migrations/FileStoreV2.cs @@ -0,0 +1,42 @@ +using VoidCat.Model; + +namespace VoidCat.Services.Migrations; + +public class FileStoreV2 : IMigration +{ + private readonly VoidSettings _settings; + + public FileStoreV2(VoidSettings settings) + { + _settings = settings; + } + + public int Order => 2; + public ValueTask Migrate(string[] args) + { + var baseDir = Path.Join(_settings.DataDirectory, "files-v1"); + foreach (var path in Directory.EnumerateFiles(baseDir)) + { + if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id)) + { + continue; + } + + var dest = MapPathV2(id); + var destDir = Path.GetDirectoryName(dest)!; + if (!Directory.Exists(destDir)) + { + Directory.CreateDirectory(destDir); + } + File.Move(MapPathV1(id), dest); + } + + return ValueTask.FromResult(IMigration.MigrationResult.Completed); + } + + private string MapPathV1(Guid id) => + Path.Join(_settings.DataDirectory, "files-v1", id.ToString()); + + private string MapPathV2(Guid id) => + Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString()); +} diff --git a/VoidCat/VoidStartup.cs b/VoidCat/VoidStartup.cs index 04d71c1..85a0326 100644 --- a/VoidCat/VoidStartup.cs +++ b/VoidCat/VoidStartup.cs @@ -171,6 +171,9 @@ public static class VoidStartup services.AddTransient(); services.AddTransient(); } + + services.AddTransient(); + services.AddTransient(); } public static JsonSerializerSettings ConfigJsonSettings(JsonSerializerSettings s) diff --git a/VoidCat/appsettings.json b/VoidCat/appsettings.json index 72ae732..64b324b 100644 --- a/VoidCat/appsettings.json +++ b/VoidCat/appsettings.json @@ -2,7 +2,8 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore": "Warning" } }, "AllowedHosts": "*", @@ -10,7 +11,8 @@ "SiteUrl": "https://localhost:7195", "DataDirectory": "./data", "CorsOrigins": [ - "http://localhost:3000" + "http://localhost:3000", + "http://localhost:8080" ] } }