FilesV2 + Cleanup

VoidCat/Services/Abstractions/IFileStore.cs

@@ -13,6 +13,13 @@ public interface IFileStore
     /// </summary>
     string? Key { get; }
 
+    /// <summary>
+    /// Check if a file exists in the store
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
+    ValueTask<bool> Exists(Guid id);
+    
     /// <summary>
     /// Ingress a file into the system (Upload)
     /// </summary>

VoidCat/Services/Files/FileSystemFactory.cs

@@ -35,6 +35,12 @@ public class FileStoreFactory : IFileStore
     /// <inheritdoc />
     public string? Key => null;
 
+    public async ValueTask<bool> Exists(Guid id)
+    {
+        var store = await GetStore(id);
+        return await store.Exists(id);
+    }
+
     /// <inheritdoc />
     public ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {

VoidCat/Services/Files/LocalDiskFileStorage.cs

@@ -8,7 +8,6 @@ namespace VoidCat.Services.Files;
 /// <inheritdoc cref="IFileStore"/>
 public class LocalDiskFileStore : StreamFileStore, IFileStore
 {
-    private const string FilesDir = "files-v1";
     private readonly VoidSettings _settings;
     private readonly CompressContent _stripMetadata;
 
@@ -17,12 +16,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
     {
         _settings = settings;
         _stripMetadata = stripMetadata;
-
-        var dir = Path.Combine(_settings.DataDirectory, FilesDir);
-        if (!Directory.Exists(dir))
-        {
-            Directory.CreateDirectory(dir);
-        }
     }
 
     /// <inheritdoc />
@@ -41,10 +34,15 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
     /// <inheritdoc />
     public string Key => "local-disk";
 
-    /// <inheritdoc />
+    public ValueTask<bool> Exists(Guid id)
+    {
+        var path = MapPath(id);
+        return ValueTask.FromResult(File.Exists(path));
+    }
+
     public async ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {
-        var finalPath = MapPath(payload.Id);
+        var finalPath = MapCreatePath(payload.Id);
         await using var fsTemp = new FileStream(finalPath,
             payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite);
 
@@ -97,7 +95,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         return vf;
     }
 
-    /// <inheritdoc />
     public ValueTask DeleteFile(Guid id)
     {
         var fp = MapPath(id);
@@ -109,7 +106,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         return ValueTask.CompletedTask;
     }
 
-    /// <inheritdoc />
     public ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
     {
         var path = MapPath(request.Id);
@@ -118,6 +114,18 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         return ValueTask.FromResult<Stream>(new FileStream(path, FileMode.Open, FileAccess.Read));
     }
 
+    private string MapCreatePath(Guid id)
+    {
+        var path = MapPath(id);
+        var dir = Path.GetDirectoryName(path);
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir!);
+        }
+
+        return path;
+    }
+    
     private string MapPath(Guid id) =>
-        Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
+        Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString());
 }

VoidCat/Services/Files/PostgresFileMetadataStore.cs

@@ -66,12 +66,8 @@ public class PostgresFileMetadataStore : IFileMetadataStore
     /// <inheritdoc />
     public async ValueTask<PagedResult<Database.File>> ListFiles(PagedRequest request)
     {
-        var count = await _db.Files.CountAsync();
+        IQueryable<Database.File> MakeQuery(VoidContext db)
-
-        async IAsyncEnumerable<Database.File> Enumerate()
         {
-            using var scope = _scopeFactory.CreateScope();
-            var db = scope.ServiceProvider.GetRequiredService<VoidContext>();
             var q = db.Files.AsNoTracking().AsQueryable();
             switch (request.SortBy, request.SortOrder)
             {
@@ -101,7 +97,15 @@ public class PostgresFileMetadataStore : IFileMetadataStore
                     break;
             }
 
-            await foreach (var r in q.Skip(request.Page * request.PageSize).Take(request.PageSize).AsAsyncEnumerable())
+            return q.Skip(request.Page * request.PageSize).Take(request.PageSize);
+        }
+        
+        async IAsyncEnumerable<Database.File> Enumerate()
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var db = scope.ServiceProvider.GetRequiredService<VoidContext>();
+            
+            await foreach (var r in MakeQuery(db).AsAsyncEnumerable())
             {
                 yield return r;
             }
@@ -109,7 +113,7 @@ public class PostgresFileMetadataStore : IFileMetadataStore
 
         return new()
         {
-            TotalResults = count,
+            TotalResults = await MakeQuery(_db).CountAsync(),
             PageSize = request.PageSize,
             Page = request.Page,
             Results = Enumerate()

VoidCat/Services/Files/S3FileStore.cs

@@ -24,10 +24,26 @@ public class S3FileStore : StreamFileStore, IFileStore
         _client = _config.CreateClient();
     }
 
-    /// <inheritdoc />
     public string Key => _config.Name;
 
-    /// <inheritdoc />
+    public async ValueTask<bool> Exists(Guid id)
+    {
+        try
+        {
+            await _client.GetObjectMetadataAsync(new GetObjectMetadataRequest()
+            {
+                BucketName = _config.BucketName,
+                Key = id.ToString()
+            });
+
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
     public async ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {
         if (payload.IsMultipart) return await IngressMultipart(payload, cts);

VoidCat/Services/Migrations/CleanupLocalDiskStore.cs

@@ -0,0 +1,80 @@
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+using VoidCat.Services.Files;
+
+namespace VoidCat.Services.Migrations;
+
+public class CleanupLocalDiskStore : IMigration
+{
+    private readonly VoidSettings _settings;
+    private readonly IFileMetadataStore _metadataStore;
+    private readonly IFileStore _fileStore;
+    private readonly ILogger<CleanupLocalDiskStore> _logger;
+
+    public CleanupLocalDiskStore(VoidSettings settings, IFileMetadataStore store, ILogger<CleanupLocalDiskStore> logger,
+        IFileStore fileStore)
+    {
+        _settings = settings;
+        _metadataStore = store;
+        _logger = logger;
+        _fileStore = fileStore;
+    }
+
+    public int Order => 3;
+    public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
+    {
+        if (_fileStore is not LocalDiskFileStore)
+        {
+            return IMigration.MigrationResult.Skipped;
+        }
+
+        await CleanupDisk();
+        await CleanupMetadata();
+
+        return IMigration.MigrationResult.Completed;
+    }
+
+    private async Task CleanupDisk()
+    {
+        var baseDir = Path.Join(_settings.DataDirectory, "files-v2");
+        foreach (var path in Directory.EnumerateFiles(baseDir, "*.*", SearchOption.AllDirectories))
+        {
+            if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id))
+            {
+                continue;
+            }
+
+            var meta = await _metadataStore.Get(id);
+            if (meta == default)
+            {
+                _logger.LogInformation("Deleting unmapped file {Path}", path);
+                File.Delete(path);
+            }
+        }
+    }
+
+    private async Task CleanupMetadata()
+    {
+        var page = 0;
+        while (true)
+        {
+            var deleting = new List<Guid>();
+            var fileList = await _metadataStore.ListFiles(new(page++, 1000));
+            if (fileList.TotalResults == 0) break;
+
+            await foreach (var md in fileList.Results)
+            {
+                if (!await _fileStore.Exists(md.Id))
+                {
+                    deleting.Add(md.Id);
+                }
+            }
+
+            foreach (var toDelete in deleting)
+            {
+                _logger.LogInformation("Deleting metadata with missing file {Id}", toDelete);
+                await _metadataStore.Delete(toDelete);
+            }
+        }
+    }
+}

VoidCat/Services/Migrations/FileStoreV2.cs

@@ -0,0 +1,42 @@
+using VoidCat.Model;
+
+namespace VoidCat.Services.Migrations;
+
+public class FileStoreV2 : IMigration
+{
+    private readonly VoidSettings _settings;
+    
+    public FileStoreV2(VoidSettings settings)
+    {
+        _settings = settings;
+    }
+    
+    public int Order => 2;
+    public ValueTask<IMigration.MigrationResult> Migrate(string[] args)
+    {
+        var baseDir = Path.Join(_settings.DataDirectory, "files-v1");
+        foreach (var path in Directory.EnumerateFiles(baseDir))
+        {
+            if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id))
+            {
+                continue;
+            }
+
+            var dest = MapPathV2(id);
+            var destDir = Path.GetDirectoryName(dest)!;
+            if (!Directory.Exists(destDir))
+            {
+                Directory.CreateDirectory(destDir);
+            }
+            File.Move(MapPathV1(id), dest);
+        }
+
+        return ValueTask.FromResult(IMigration.MigrationResult.Completed);
+    }
+    
+    private string MapPathV1(Guid id) =>
+        Path.Join(_settings.DataDirectory, "files-v1", id.ToString());
+    
+    private string MapPathV2(Guid id) =>
+        Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString());
+}

VoidCat/VoidStartup.cs

@@ -171,6 +171,9 @@ public static class VoidStartup
             services.AddTransient<IMigration, EFMigrationSetup>();
             services.AddTransient<IMigration, EFMigration>();
         }
+
+        services.AddTransient<IMigration, FileStoreV2>();
+        services.AddTransient<IMigration, CleanupLocalDiskStore>();
     }
 
     public static JsonSerializerSettings ConfigJsonSettings(JsonSerializerSettings s)

VoidCat/appsettings.json

@@ -2,7 +2,8 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Warning",
+      "Microsoft.EntityFrameworkCore": "Warning"
     }
   },
   "AllowedHosts": "*",
@@ -10,7 +11,8 @@
     "SiteUrl": "https://localhost:7195",
     "DataDirectory": "./data",
     "CorsOrigins": [
-      "http://localhost:3000"
+      "http://localhost:3000",
+      "http://localhost:8080"
     ]
   }
 }