diff --git a/VoidCat/Services/Abstractions/IFileStore.cs b/VoidCat/Services/Abstractions/IFileStore.cs
index 6326437..8dfc9d8 100644
--- a/VoidCat/Services/Abstractions/IFileStore.cs
+++ b/VoidCat/Services/Abstractions/IFileStore.cs
@@ -12,6 +12,13 @@ public interface IFileStore
/// Return key for named instance
///
string? Key { get; }
+
+ ///
+ /// Check if a file exists in the store
+ ///
+ ///
+ ///
+ ValueTask Exists(Guid id);
///
/// Ingress a file into the system (Upload)
diff --git a/VoidCat/Services/Files/FileSystemFactory.cs b/VoidCat/Services/Files/FileSystemFactory.cs
index fc79dfb..dcb1962 100644
--- a/VoidCat/Services/Files/FileSystemFactory.cs
+++ b/VoidCat/Services/Files/FileSystemFactory.cs
@@ -35,6 +35,12 @@ public class FileStoreFactory : IFileStore
///
public string? Key => null;
+ public async ValueTask Exists(Guid id)
+ {
+ var store = await GetStore(id);
+ return await store.Exists(id);
+ }
+
///
public ValueTask Ingress(IngressPayload payload, CancellationToken cts)
{
@@ -43,7 +49,7 @@ public class FileStoreFactory : IFileStore
{
throw new InvalidOperationException($"Cannot find store '{payload.Meta.Storage}'");
}
-
+
return store.Ingress(payload, cts);
}
@@ -53,7 +59,7 @@ public class FileStoreFactory : IFileStore
var store = await GetStore(request.Id);
await store.Egress(request, outStream, cts);
}
-
+
///
public async ValueTask StartEgress(EgressRequest request)
{
diff --git a/VoidCat/Services/Files/LocalDiskFileStorage.cs b/VoidCat/Services/Files/LocalDiskFileStorage.cs
index a3c6b8e..62763a1 100644
--- a/VoidCat/Services/Files/LocalDiskFileStorage.cs
+++ b/VoidCat/Services/Files/LocalDiskFileStorage.cs
@@ -8,7 +8,6 @@ namespace VoidCat.Services.Files;
///
public class LocalDiskFileStore : StreamFileStore, IFileStore
{
- private const string FilesDir = "files-v1";
private readonly VoidSettings _settings;
private readonly CompressContent _stripMetadata;
@@ -17,12 +16,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
{
_settings = settings;
_stripMetadata = stripMetadata;
-
- var dir = Path.Combine(_settings.DataDirectory, FilesDir);
- if (!Directory.Exists(dir))
- {
- Directory.CreateDirectory(dir);
- }
}
///
@@ -41,10 +34,15 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
///
public string Key => "local-disk";
- ///
+ public ValueTask Exists(Guid id)
+ {
+ var path = MapPath(id);
+ return ValueTask.FromResult(File.Exists(path));
+ }
+
public async ValueTask Ingress(IngressPayload payload, CancellationToken cts)
{
- var finalPath = MapPath(payload.Id);
+ var finalPath = MapCreatePath(payload.Id);
await using var fsTemp = new FileStream(finalPath,
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.ReadWrite);
@@ -97,7 +95,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
return vf;
}
- ///
public ValueTask DeleteFile(Guid id)
{
var fp = MapPath(id);
@@ -109,7 +106,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
return ValueTask.CompletedTask;
}
- ///
public ValueTask Open(EgressRequest request, CancellationToken cts)
{
var path = MapPath(request.Id);
@@ -118,6 +114,18 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
return ValueTask.FromResult(new FileStream(path, FileMode.Open, FileAccess.Read));
}
+ private string MapCreatePath(Guid id)
+ {
+ var path = MapPath(id);
+ var dir = Path.GetDirectoryName(path);
+ if (!Directory.Exists(dir))
+ {
+ Directory.CreateDirectory(dir!);
+ }
+
+ return path;
+ }
+
private string MapPath(Guid id) =>
- Path.Join(_settings.DataDirectory, FilesDir, id.ToString());
+ Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString());
}
diff --git a/VoidCat/Services/Files/PostgresFileMetadataStore.cs b/VoidCat/Services/Files/PostgresFileMetadataStore.cs
index dcbcb57..405dcbf 100644
--- a/VoidCat/Services/Files/PostgresFileMetadataStore.cs
+++ b/VoidCat/Services/Files/PostgresFileMetadataStore.cs
@@ -66,12 +66,8 @@ public class PostgresFileMetadataStore : IFileMetadataStore
///
public async ValueTask> ListFiles(PagedRequest request)
{
- var count = await _db.Files.CountAsync();
-
- async IAsyncEnumerable Enumerate()
+ IQueryable MakeQuery(VoidContext db)
{
- using var scope = _scopeFactory.CreateScope();
- var db = scope.ServiceProvider.GetRequiredService();
var q = db.Files.AsNoTracking().AsQueryable();
switch (request.SortBy, request.SortOrder)
{
@@ -101,7 +97,15 @@ public class PostgresFileMetadataStore : IFileMetadataStore
break;
}
- await foreach (var r in q.Skip(request.Page * request.PageSize).Take(request.PageSize).AsAsyncEnumerable())
+ return q.Skip(request.Page * request.PageSize).Take(request.PageSize);
+ }
+
+ async IAsyncEnumerable Enumerate()
+ {
+ using var scope = _scopeFactory.CreateScope();
+ var db = scope.ServiceProvider.GetRequiredService();
+
+ await foreach (var r in MakeQuery(db).AsAsyncEnumerable())
{
yield return r;
}
@@ -109,7 +113,7 @@ public class PostgresFileMetadataStore : IFileMetadataStore
return new()
{
- TotalResults = count,
+ TotalResults = await MakeQuery(_db).CountAsync(),
PageSize = request.PageSize,
Page = request.Page,
Results = Enumerate()
diff --git a/VoidCat/Services/Files/S3FileStore.cs b/VoidCat/Services/Files/S3FileStore.cs
index 1cd8f3d..2dd9ef3 100644
--- a/VoidCat/Services/Files/S3FileStore.cs
+++ b/VoidCat/Services/Files/S3FileStore.cs
@@ -24,10 +24,26 @@ public class S3FileStore : StreamFileStore, IFileStore
_client = _config.CreateClient();
}
- ///
public string Key => _config.Name;
- ///
+ public async ValueTask Exists(Guid id)
+ {
+ try
+ {
+ await _client.GetObjectMetadataAsync(new GetObjectMetadataRequest()
+ {
+ BucketName = _config.BucketName,
+ Key = id.ToString()
+ });
+
+ return true;
+ }
+ catch
+ {
+ return false;
+ }
+ }
+
public async ValueTask Ingress(IngressPayload payload, CancellationToken cts)
{
if (payload.IsMultipart) return await IngressMultipart(payload, cts);
@@ -206,7 +222,7 @@ public class S3FileStore : StreamFileStore, IFileStore
InputStream = fsTmp,
DisablePayloadSigning = _config.DisablePayloadSigning
};
-
+
var bodyResponse = await _client.UploadPartAsync(mBody, cts);
if (bodyResponse.HttpStatusCode != HttpStatusCode.OK)
{
@@ -241,7 +257,7 @@ public class S3FileStore : StreamFileStore, IFileStore
throw new Exception("Upload failed");
}
}
-
+
return HandleCompletedUpload(payload, segmentLength);
}
diff --git a/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs b/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs
new file mode 100644
index 0000000..1de4f0a
--- /dev/null
+++ b/VoidCat/Services/Migrations/CleanupLocalDiskStore.cs
@@ -0,0 +1,80 @@
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+using VoidCat.Services.Files;
+
+namespace VoidCat.Services.Migrations;
+
+public class CleanupLocalDiskStore : IMigration
+{
+ private readonly VoidSettings _settings;
+ private readonly IFileMetadataStore _metadataStore;
+ private readonly IFileStore _fileStore;
+ private readonly ILogger _logger;
+
+ public CleanupLocalDiskStore(VoidSettings settings, IFileMetadataStore store, ILogger logger,
+ IFileStore fileStore)
+ {
+ _settings = settings;
+ _metadataStore = store;
+ _logger = logger;
+ _fileStore = fileStore;
+ }
+
+ public int Order => 3;
+ public async ValueTask Migrate(string[] args)
+ {
+ if (_fileStore is not LocalDiskFileStore)
+ {
+ return IMigration.MigrationResult.Skipped;
+ }
+
+ await CleanupDisk();
+ await CleanupMetadata();
+
+ return IMigration.MigrationResult.Completed;
+ }
+
+ private async Task CleanupDisk()
+ {
+ var baseDir = Path.Join(_settings.DataDirectory, "files-v2");
+ foreach (var path in Directory.EnumerateFiles(baseDir, "*.*", SearchOption.AllDirectories))
+ {
+ if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id))
+ {
+ continue;
+ }
+
+ var meta = await _metadataStore.Get(id);
+ if (meta == default)
+ {
+ _logger.LogInformation("Deleting unmapped file {Path}", path);
+ File.Delete(path);
+ }
+ }
+ }
+
+ private async Task CleanupMetadata()
+ {
+ var page = 0;
+ while (true)
+ {
+ var deleting = new List();
+ var fileList = await _metadataStore.ListFiles(new(page++, 1000));
+ if (fileList.TotalResults == 0) break;
+
+ await foreach (var md in fileList.Results)
+ {
+ if (!await _fileStore.Exists(md.Id))
+ {
+ deleting.Add(md.Id);
+ }
+ }
+
+ foreach (var toDelete in deleting)
+ {
+ _logger.LogInformation("Deleting metadata with missing file {Id}", toDelete);
+ await _metadataStore.Delete(toDelete);
+ }
+ }
+ }
+}
diff --git a/VoidCat/Services/Migrations/FileStoreV2.cs b/VoidCat/Services/Migrations/FileStoreV2.cs
new file mode 100644
index 0000000..092feb7
--- /dev/null
+++ b/VoidCat/Services/Migrations/FileStoreV2.cs
@@ -0,0 +1,42 @@
+using VoidCat.Model;
+
+namespace VoidCat.Services.Migrations;
+
+public class FileStoreV2 : IMigration
+{
+ private readonly VoidSettings _settings;
+
+ public FileStoreV2(VoidSettings settings)
+ {
+ _settings = settings;
+ }
+
+ public int Order => 2;
+ public ValueTask Migrate(string[] args)
+ {
+ var baseDir = Path.Join(_settings.DataDirectory, "files-v1");
+ foreach (var path in Directory.EnumerateFiles(baseDir))
+ {
+ if (!Guid.TryParse(Path.GetFileNameWithoutExtension(path), out var id))
+ {
+ continue;
+ }
+
+ var dest = MapPathV2(id);
+ var destDir = Path.GetDirectoryName(dest)!;
+ if (!Directory.Exists(destDir))
+ {
+ Directory.CreateDirectory(destDir);
+ }
+ File.Move(MapPathV1(id), dest);
+ }
+
+ return ValueTask.FromResult(IMigration.MigrationResult.Completed);
+ }
+
+ private string MapPathV1(Guid id) =>
+ Path.Join(_settings.DataDirectory, "files-v1", id.ToString());
+
+ private string MapPathV2(Guid id) =>
+ Path.Join(_settings.DataDirectory, "files-v2", id.ToString()[..2], id.ToString()[2..4], id.ToString());
+}
diff --git a/VoidCat/VoidStartup.cs b/VoidCat/VoidStartup.cs
index 04d71c1..85a0326 100644
--- a/VoidCat/VoidStartup.cs
+++ b/VoidCat/VoidStartup.cs
@@ -171,6 +171,9 @@ public static class VoidStartup
services.AddTransient();
services.AddTransient();
}
+
+ services.AddTransient();
+ services.AddTransient();
}
public static JsonSerializerSettings ConfigJsonSettings(JsonSerializerSettings s)
diff --git a/VoidCat/appsettings.json b/VoidCat/appsettings.json
index 72ae732..64b324b 100644
--- a/VoidCat/appsettings.json
+++ b/VoidCat/appsettings.json
@@ -2,7 +2,8 @@
"Logging": {
"LogLevel": {
"Default": "Information",
- "Microsoft.AspNetCore": "Warning"
+ "Microsoft.AspNetCore": "Warning",
+ "Microsoft.EntityFrameworkCore": "Warning"
}
},
"AllowedHosts": "*",
@@ -10,7 +11,8 @@
"SiteUrl": "https://localhost:7195",
"DataDirectory": "./data",
"CorsOrigins": [
- "http://localhost:3000"
+ "http://localhost:3000",
+ "http://localhost:8080"
]
}
}