Delete expired files

VoidCat/Controllers/Admin/AdminController.cs

@@ -12,11 +12,11 @@ public class AdminController : Controller
 {
     private readonly FileStoreFactory _fileStore;
     private readonly IFileMetadataStore _fileMetadata;
-    private readonly IFileInfoManager _fileInfo;
+    private readonly FileInfoManager _fileInfo;
     private readonly IUserStore _userStore;
     private readonly IUserUploadsStore _userUploads;
 
-    public AdminController(FileStoreFactory fileStore, IUserStore userStore, IFileInfoManager fileInfo,
+    public AdminController(FileStoreFactory fileStore, IUserStore userStore, FileInfoManager fileInfo,
         IFileMetadataStore fileMetadata, IUserUploadsStore userUploads)
     {
         _fileStore = fileStore;

VoidCat/Controllers/DownloadController.cs

@@ -11,11 +11,11 @@ namespace VoidCat.Controllers;
 public class DownloadController : Controller
 {
     private readonly FileStoreFactory _storage;
-    private readonly IFileInfoManager _fileInfo;
+    private readonly FileInfoManager _fileInfo;
     private readonly IPaywallOrderStore _paywallOrders;
     private readonly ILogger<DownloadController> _logger;
 
-    public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
+    public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, FileInfoManager fileInfo,
         IPaywallOrderStore paywall)
     {
         _storage = storage;

VoidCat/Controllers/UploadController.cs

@@ -17,14 +17,14 @@ namespace VoidCat.Controllers
         private readonly IFileMetadataStore _metadata;
         private readonly IPaywallStore _paywall;
         private readonly IPaywallFactory _paywallFactory;
-        private readonly IFileInfoManager _fileInfo;
+        private readonly FileInfoManager _fileInfo;
         private readonly IUserUploadsStore _userUploads;
         private readonly IUserStore _userStore;
         private readonly ITimeSeriesStatsReporter _timeSeriesStats;
         private readonly VoidSettings _settings;
 
         public UploadController(FileStoreFactory storage, IFileMetadataStore metadata, IPaywallStore paywall,
-            IPaywallFactory paywallFactory, IFileInfoManager fileInfo, IUserUploadsStore userUploads,
+            IPaywallFactory paywallFactory, FileInfoManager fileInfo, IUserUploadsStore userUploads,
             ITimeSeriesStatsReporter timeSeriesStats, IUserStore userStore, VoidSettings settings)
         {
             _storage = storage;

VoidCat/Controllers/UserController.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.Mvc;
 using VoidCat.Model;
 using VoidCat.Services.Abstractions;
+using VoidCat.Services.Files;
 
 namespace VoidCat.Controllers;
 
@@ -10,10 +11,10 @@ public class UserController : Controller
     private readonly IUserStore _store;
     private readonly IUserUploadsStore _userUploads;
     private readonly IEmailVerification _emailVerification;
-    private readonly IFileInfoManager _fileInfoManager;
+    private readonly FileInfoManager _fileInfoManager;
 
     public UserController(IUserStore store, IUserUploadsStore userUploads, IEmailVerification emailVerification,
-        IFileInfoManager fileInfoManager)
+        FileInfoManager fileInfoManager)
     {
         _store = store;
         _userUploads = userUploads;

VoidCat/Services/Abstractions/IFileInfoManager.cs

@@ -1,38 +0,0 @@
-using VoidCat.Model;
-
-namespace VoidCat.Services.Abstractions;
-
-/// <summary>
-/// Main interface for getting file info to serve to clients.
-/// This interface should wrap all stores and return the combined result
-/// </summary>
-public interface IFileInfoManager
-{
-    /// <summary>
-    /// Get all metadata for a single file
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    ValueTask<PublicVoidFile?> Get(Guid id);
-
-    /// <summary>
-    /// Get all private metadata for a single file
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    ValueTask<PrivateVoidFile?> GetPrivate(Guid id);
-    
-    /// <summary>
-    /// Get all metadata for multiple files
-    /// </summary>
-    /// <param name="ids"></param>
-    /// <returns></returns>
-    ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
-    
-    /// <summary>
-    /// Deletes all file metadata
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    ValueTask Delete(Guid id);
-}

VoidCat/Services/Background/DeleteExpiredFiles.cs

@@ -0,0 +1,52 @@
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+using VoidCat.Services.Files;
+
+namespace VoidCat.Services.Background;
+
+/// <summary>
+/// Delete expired files
+/// </summary>
+public sealed class DeleteExpiredFiles : BackgroundService
+{
+    private readonly ILogger<DeleteExpiredFiles> _logger;
+    private readonly IServiceScopeFactory _scopeFactory;
+
+    public DeleteExpiredFiles(ILogger<DeleteExpiredFiles> logger, IServiceScopeFactory scopeFactory)
+    {
+        _logger = logger;
+        _scopeFactory = scopeFactory;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var metadata = scope.ServiceProvider.GetRequiredService<IFileMetadataStore>();
+            var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
+            var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
+
+            var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
+            await foreach (var f in files.Results.WithCancellation(stoppingToken))
+            {
+                try
+                {
+                    if (f.Expires < DateTime.Now)
+                    {
+                        await fileStoreFactory.DeleteFile(f.Id);
+                        await fileInfoManager.Delete(f.Id);
+
+                        _logger.LogInformation("Deleted file: {Id}", f.Id);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Failed to delete file: {Id}", f.Id);
+                }
+            }
+
+            await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
+        }
+    }
+}

VoidCat/Services/Background/DeleteUnverifiedAccounts.cs

@@ -25,7 +25,7 @@ public class DeleteUnverifiedAccounts : BackgroundService
                 var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
                 var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
                 var fileStore = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
-                var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
+                var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
 
                 var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));
 

VoidCat/Services/Files/FileInfoManager.cs

@@ -3,8 +3,11 @@ using VoidCat.Services.Abstractions;
 
 namespace VoidCat.Services.Files;
 
-/// <inheritdoc />
+/// <summary>
-public class FileInfoManager : IFileInfoManager
+/// Main interface for getting file info to serve to clients.
+/// This interface should wrap all stores and return the combined result
+/// </summary>
+public sealed class FileInfoManager
 {
     private readonly IFileMetadataStore _metadataStore;
     private readonly IPaywallStore _paywallStore;
@@ -24,19 +27,31 @@ public class FileInfoManager : IFileInfoManager
         _userUploadsStore = userUploadsStore;
     }
 
-    /// <inheritdoc />
+    /// <summary>
+    /// Get all metadata for a single file
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
     public ValueTask<PublicVoidFile?> Get(Guid id)
     {
         return Get<PublicVoidFile, VoidFileMeta>(id);
     }
 
-    /// <inheritdoc />
+    /// <summary>
+    /// Get all private metadata for a single file
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
     public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
     {
         return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
     }
 
-    /// <inheritdoc />
+    /// <summary>
+    /// Get all metadata for multiple files
+    /// </summary>
+    /// <param name="ids"></param>
+    /// <returns></returns>
     public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
     {
         var ret = new List<PublicVoidFile>();
@@ -52,7 +67,11 @@ public class FileInfoManager : IFileInfoManager
         return ret;
     }
 
-    /// <inheritdoc />
+    /// <summary>
+    /// Deletes all file metadata
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
     public async ValueTask Delete(Guid id)
     {
         await _metadataStore.Delete(id);

VoidCat/Services/Files/FileStorageStartup.cs

@@ -8,7 +8,7 @@ public static class FileStorageStartup
 {
     public static void AddStorage(this IServiceCollection services, VoidSettings settings)
     {
-        services.AddTransient<IFileInfoManager, FileInfoManager>();
+        services.AddTransient<FileInfoManager>();
         services.AddTransient<FileStoreFactory>();
         
         if (settings.CloudStorage != default)
@@ -19,7 +19,7 @@ public static class FileStorageStartup
                 services.AddTransient<IFileStore>((svc) =>
                     new S3FileStore(s3, 
                         svc.GetRequiredService<IAggregateStatsCollector>(),
-                        svc.GetRequiredService<IFileInfoManager>(),
+                        svc.GetRequiredService<FileInfoManager>(),
                         svc.GetRequiredService<ICache>()));
 
                 if (settings.MetadataStore == s3.Name)

VoidCat/Services/Files/LocalDiskFileMetadataStore.cs

@@ -52,8 +52,8 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
         oldMeta.Description = meta.Description ?? oldMeta.Description;
         oldMeta.Name = meta.Name ?? oldMeta.Name;
         oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
-        oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
         oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
+        oldMeta.Expires = meta.Expires;
 
         await Set(id, oldMeta);
     }

VoidCat/Services/Files/LocalDiskFileStorage.cs

@@ -8,14 +8,12 @@ namespace VoidCat.Services.Files;
 public class LocalDiskFileStore : StreamFileStore, IFileStore
 {
     private const string FilesDir = "files-v1";
-    private readonly ILogger<LocalDiskFileStore> _logger;
     private readonly VoidSettings _settings;
 
-    public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats)
+    public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
         : base(stats)
     {
         _settings = settings;
-        _logger = logger;
 
         var dir = Path.Combine(_settings.DataDirectory, FilesDir);
         if (!Directory.Exists(dir))
@@ -55,7 +53,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         var fp = MapPath(id);
         if (File.Exists(fp))
         {
-            _logger.LogInformation("Deleting file: {Path}", fp);
             File.Delete(fp);
         }
 

VoidCat/Services/Files/PostgresFileMetadataStore.cs

@@ -54,7 +54,7 @@ on conflict (""Id"") do update set
                 mimeType = obj.MimeType,
                 digest = obj.Digest,
                 editSecret = obj.EditSecret,
-                expires = obj.Expires,
+                expires = obj.Expires?.ToUniversalTime(),
                 store = obj.Storage
             });
     }
@@ -91,8 +91,8 @@ on conflict (""Id"") do update set
         oldMeta.Description = meta.Description ?? oldMeta.Description;
         oldMeta.Name = meta.Name ?? oldMeta.Name;
         oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
-        oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
         oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
+        oldMeta.Expires = meta.Expires;
 
         await Set(id, oldMeta);
     }

VoidCat/Services/Files/S3FileMetadataStore.cs

@@ -53,8 +53,8 @@ public class S3FileMetadataStore : IFileMetadataStore
         oldMeta.Description = meta.Description ?? oldMeta.Description;
         oldMeta.Name = meta.Name ?? oldMeta.Name;
         oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
-        oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
         oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
+        oldMeta.Expires = meta.Expires;
 
         await Set(id, oldMeta);
     }

VoidCat/Services/Files/S3FileStore.cs

@@ -9,13 +9,13 @@ namespace VoidCat.Services.Files;
 /// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
 public class S3FileStore : StreamFileStore, IFileStore
 {
-    private readonly IFileInfoManager _fileInfo;
+    private readonly FileInfoManager _fileInfo;
     private readonly AmazonS3Client _client;
     private readonly S3BlobConfig _config;
     private readonly IAggregateStatsCollector _statsCollector;
     private readonly ICache _cache;
 
-    public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo, ICache cache) : base(stats)
+    public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, FileInfoManager fileInfo, ICache cache) : base(stats)
     {
         _fileInfo = fileInfo;
         _cache = cache;

VoidCat/VoidStartup.cs

@@ -155,6 +155,7 @@ public static class VoidStartup
     public static void AddBackgroundServices(this IServiceCollection services, VoidSettings voidSettings)
     {
         services.AddHostedService<DeleteUnverifiedAccounts>();
+        services.AddHostedService<DeleteExpiredFiles>();
 
         if (voidSettings.HasVirusScanner())
         {

VoidCat/spa/src/ApiKeyList.js

@@ -23,7 +23,11 @@ export default function ApiKeyList() {
             setNewApiKey(await rsp.json());
         }
     }
-
+    
+    function openDocs() {
+        window.open("/swagger", "_blank")
+    }
+    
     useEffect(() => {
         if (Api) {
             loadApiKeys();
@@ -38,6 +42,7 @@ export default function ApiKeyList() {
                 </div>
                 <div>
                     <VoidButton onClick={(e) => createApiKey()}>+New</VoidButton>
+                    <VoidButton onClick={(e) => openDocs()}>Docs</VoidButton>
                 </div>
             </div>
             <table>

VoidCat/spa/src/FileEdit.js

@@ -6,6 +6,7 @@ import {useApi} from "./Api";
 import "./FileEdit.css";
 import {useSelector} from "react-redux";
 import {VoidButton} from "./VoidButton";
+import moment from "moment";
 
 export function FileEdit(props) {
     const {Api} = useApi();
@@ -15,9 +16,10 @@ export function FileEdit(props) {
     const [paywall, setPaywall] = useState(file.paywall?.service);
     const [name, setName] = useState(meta?.name);
     const [description, setDescription] = useState(meta?.description);
+    const [expiry, setExpiry] = useState(meta?.expires === undefined || meta?.expires === null ? null : moment(meta?.expires).unix() * 1000);
 
-    const privateFile = file?.uploader?.id && profile?.id === file.uploader.id 
+    const privateFile = file?.uploader?.id && profile?.id === file.uploader.id
-        ? file 
+        ? file
         : JSON.parse(window.localStorage.getItem(file.id));
     if (!privateFile || privateFile?.metadata?.editSecret === null) {
         return null;
@@ -32,11 +34,12 @@ export function FileEdit(props) {
         let meta = {
             name,
             description,
-            editSecret: privateFile?.metadata?.editSecret
+            editSecret: privateFile?.metadata?.editSecret,
+            expires: moment(expiry).toISOString()
         };
-        await Api.updateMetadata(file.id, meta);        
+        await Api.updateMetadata(file.id, meta);
     }
-    
+
     function renderPaywallConfig() {
         switch (paywall) {
             case 0: {
@@ -58,6 +61,22 @@ export function FileEdit(props) {
                     <dd><input type="text" value={name} onChange={(e) => setName(e.target.value)}/></dd>
                     <dt>Description:</dt>
                     <dd><input type="text" value={description} onChange={(e) => setDescription(e.target.value)}/></dd>
+                    <dt>Expiry</dt>
+                    <dd>
+                        <input type="datetime-local"
+                               value={expiry === null ? "" : moment(expiry).toISOString().replace("Z", "")}
+                               max={moment.utc().add(1, "year").toISOString().replace("Z", "")}
+                               min={moment.utc().toISOString().replace("Z", "")}
+                               onChange={(e) => {
+                                   console.log(e.target.value);
+                                   if (e.target.value.length > 0) {
+                                       setExpiry(moment.utc(e.target.value).unix() * 1000);
+                                   } else {
+                                       setExpiry(null);
+                                   }
+
+                               }}/>
+                    </dd>
                 </dl>
                 <VoidButton onClick={(e) => saveMeta()} options={{showSuccess: true}}>Save</VoidButton>
             </div>

VoidCat/spa/src/index.css

@@ -60,7 +60,7 @@ a:hover {
     align-items: center;
 }
 
-input[type="text"], input[type="number"], input[type="password"], select {
+input[type="text"], input[type="number"], input[type="password"], input[type="datetime-local"], select {
     display: inline-block;
     line-height: 1.1;
     border-radius: 10px;