forked from Kieran/void.cat
Delete expired files
This commit is contained in:
parent
ebe5a0e106
commit
1d451aac82
@ -12,11 +12,11 @@ public class AdminController : Controller
|
||||
{
|
||||
private readonly FileStoreFactory _fileStore;
|
||||
private readonly IFileMetadataStore _fileMetadata;
|
||||
private readonly IFileInfoManager _fileInfo;
|
||||
private readonly FileInfoManager _fileInfo;
|
||||
private readonly IUserStore _userStore;
|
||||
private readonly IUserUploadsStore _userUploads;
|
||||
|
||||
public AdminController(FileStoreFactory fileStore, IUserStore userStore, IFileInfoManager fileInfo,
|
||||
public AdminController(FileStoreFactory fileStore, IUserStore userStore, FileInfoManager fileInfo,
|
||||
IFileMetadataStore fileMetadata, IUserUploadsStore userUploads)
|
||||
{
|
||||
_fileStore = fileStore;
|
||||
|
@ -11,11 +11,11 @@ namespace VoidCat.Controllers;
|
||||
public class DownloadController : Controller
|
||||
{
|
||||
private readonly FileStoreFactory _storage;
|
||||
private readonly IFileInfoManager _fileInfo;
|
||||
private readonly FileInfoManager _fileInfo;
|
||||
private readonly IPaywallOrderStore _paywallOrders;
|
||||
private readonly ILogger<DownloadController> _logger;
|
||||
|
||||
public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
|
||||
public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, FileInfoManager fileInfo,
|
||||
IPaywallOrderStore paywall)
|
||||
{
|
||||
_storage = storage;
|
||||
|
@ -17,14 +17,14 @@ namespace VoidCat.Controllers
|
||||
private readonly IFileMetadataStore _metadata;
|
||||
private readonly IPaywallStore _paywall;
|
||||
private readonly IPaywallFactory _paywallFactory;
|
||||
private readonly IFileInfoManager _fileInfo;
|
||||
private readonly FileInfoManager _fileInfo;
|
||||
private readonly IUserUploadsStore _userUploads;
|
||||
private readonly IUserStore _userStore;
|
||||
private readonly ITimeSeriesStatsReporter _timeSeriesStats;
|
||||
private readonly VoidSettings _settings;
|
||||
|
||||
public UploadController(FileStoreFactory storage, IFileMetadataStore metadata, IPaywallStore paywall,
|
||||
IPaywallFactory paywallFactory, IFileInfoManager fileInfo, IUserUploadsStore userUploads,
|
||||
IPaywallFactory paywallFactory, FileInfoManager fileInfo, IUserUploadsStore userUploads,
|
||||
ITimeSeriesStatsReporter timeSeriesStats, IUserStore userStore, VoidSettings settings)
|
||||
{
|
||||
_storage = storage;
|
||||
|
@ -1,6 +1,7 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using VoidCat.Model;
|
||||
using VoidCat.Services.Abstractions;
|
||||
using VoidCat.Services.Files;
|
||||
|
||||
namespace VoidCat.Controllers;
|
||||
|
||||
@ -10,10 +11,10 @@ public class UserController : Controller
|
||||
private readonly IUserStore _store;
|
||||
private readonly IUserUploadsStore _userUploads;
|
||||
private readonly IEmailVerification _emailVerification;
|
||||
private readonly IFileInfoManager _fileInfoManager;
|
||||
private readonly FileInfoManager _fileInfoManager;
|
||||
|
||||
public UserController(IUserStore store, IUserUploadsStore userUploads, IEmailVerification emailVerification,
|
||||
IFileInfoManager fileInfoManager)
|
||||
FileInfoManager fileInfoManager)
|
||||
{
|
||||
_store = store;
|
||||
_userUploads = userUploads;
|
||||
|
@ -1,38 +0,0 @@
|
||||
using VoidCat.Model;
|
||||
|
||||
namespace VoidCat.Services.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Main interface for getting file info to serve to clients.
|
||||
/// This interface should wrap all stores and return the combined result
|
||||
/// </summary>
|
||||
public interface IFileInfoManager
|
||||
{
|
||||
/// <summary>
|
||||
/// Get all metadata for a single file
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
ValueTask<PublicVoidFile?> Get(Guid id);
|
||||
|
||||
/// <summary>
|
||||
/// Get all private metadata for a single file
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
ValueTask<PrivateVoidFile?> GetPrivate(Guid id);
|
||||
|
||||
/// <summary>
|
||||
/// Get all metadata for multiple files
|
||||
/// </summary>
|
||||
/// <param name="ids"></param>
|
||||
/// <returns></returns>
|
||||
ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all file metadata
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
ValueTask Delete(Guid id);
|
||||
}
|
52
VoidCat/Services/Background/DeleteExpiredFiles.cs
Normal file
52
VoidCat/Services/Background/DeleteExpiredFiles.cs
Normal file
@ -0,0 +1,52 @@
|
||||
using VoidCat.Model;
|
||||
using VoidCat.Services.Abstractions;
|
||||
using VoidCat.Services.Files;
|
||||
|
||||
namespace VoidCat.Services.Background;
|
||||
|
||||
/// <summary>
|
||||
/// Delete expired files
|
||||
/// </summary>
|
||||
public sealed class DeleteExpiredFiles : BackgroundService
|
||||
{
|
||||
private readonly ILogger<DeleteExpiredFiles> _logger;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
|
||||
public DeleteExpiredFiles(ILogger<DeleteExpiredFiles> logger, IServiceScopeFactory scopeFactory)
|
||||
{
|
||||
_logger = logger;
|
||||
_scopeFactory = scopeFactory;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var metadata = scope.ServiceProvider.GetRequiredService<IFileMetadataStore>();
|
||||
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
|
||||
var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
|
||||
|
||||
var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
|
||||
await foreach (var f in files.Results.WithCancellation(stoppingToken))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (f.Expires < DateTime.Now)
|
||||
{
|
||||
await fileStoreFactory.DeleteFile(f.Id);
|
||||
await fileInfoManager.Delete(f.Id);
|
||||
|
||||
_logger.LogInformation("Deleted file: {Id}", f.Id);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete file: {Id}", f.Id);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
@ -25,7 +25,7 @@ public class DeleteUnverifiedAccounts : BackgroundService
|
||||
var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
|
||||
var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
|
||||
var fileStore = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
|
||||
var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
|
||||
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
|
||||
|
||||
var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));
|
||||
|
||||
|
@ -3,8 +3,11 @@ using VoidCat.Services.Abstractions;
|
||||
|
||||
namespace VoidCat.Services.Files;
|
||||
|
||||
/// <inheritdoc />
|
||||
public class FileInfoManager : IFileInfoManager
|
||||
/// <summary>
|
||||
/// Main interface for getting file info to serve to clients.
|
||||
/// This interface should wrap all stores and return the combined result
|
||||
/// </summary>
|
||||
public sealed class FileInfoManager
|
||||
{
|
||||
private readonly IFileMetadataStore _metadataStore;
|
||||
private readonly IPaywallStore _paywallStore;
|
||||
@ -24,19 +27,31 @@ public class FileInfoManager : IFileInfoManager
|
||||
_userUploadsStore = userUploadsStore;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Get all metadata for a single file
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
public ValueTask<PublicVoidFile?> Get(Guid id)
|
||||
{
|
||||
return Get<PublicVoidFile, VoidFileMeta>(id);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Get all private metadata for a single file
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
|
||||
{
|
||||
return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Get all metadata for multiple files
|
||||
/// </summary>
|
||||
/// <param name="ids"></param>
|
||||
/// <returns></returns>
|
||||
public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
|
||||
{
|
||||
var ret = new List<PublicVoidFile>();
|
||||
@ -52,7 +67,11 @@ public class FileInfoManager : IFileInfoManager
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Deletes all file metadata
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
public async ValueTask Delete(Guid id)
|
||||
{
|
||||
await _metadataStore.Delete(id);
|
||||
|
@ -8,7 +8,7 @@ public static class FileStorageStartup
|
||||
{
|
||||
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
|
||||
{
|
||||
services.AddTransient<IFileInfoManager, FileInfoManager>();
|
||||
services.AddTransient<FileInfoManager>();
|
||||
services.AddTransient<FileStoreFactory>();
|
||||
|
||||
if (settings.CloudStorage != default)
|
||||
@ -19,7 +19,7 @@ public static class FileStorageStartup
|
||||
services.AddTransient<IFileStore>((svc) =>
|
||||
new S3FileStore(s3,
|
||||
svc.GetRequiredService<IAggregateStatsCollector>(),
|
||||
svc.GetRequiredService<IFileInfoManager>(),
|
||||
svc.GetRequiredService<FileInfoManager>(),
|
||||
svc.GetRequiredService<ICache>()));
|
||||
|
||||
if (settings.MetadataStore == s3.Name)
|
||||
|
@ -52,8 +52,8 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
|
||||
oldMeta.Description = meta.Description ?? oldMeta.Description;
|
||||
oldMeta.Name = meta.Name ?? oldMeta.Name;
|
||||
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
|
||||
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
|
||||
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
|
||||
oldMeta.Expires = meta.Expires;
|
||||
|
||||
await Set(id, oldMeta);
|
||||
}
|
||||
|
@ -8,14 +8,12 @@ namespace VoidCat.Services.Files;
|
||||
public class LocalDiskFileStore : StreamFileStore, IFileStore
|
||||
{
|
||||
private const string FilesDir = "files-v1";
|
||||
private readonly ILogger<LocalDiskFileStore> _logger;
|
||||
private readonly VoidSettings _settings;
|
||||
|
||||
public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats)
|
||||
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
|
||||
: base(stats)
|
||||
{
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
|
||||
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
|
||||
if (!Directory.Exists(dir))
|
||||
@ -55,7 +53,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
|
||||
var fp = MapPath(id);
|
||||
if (File.Exists(fp))
|
||||
{
|
||||
_logger.LogInformation("Deleting file: {Path}", fp);
|
||||
File.Delete(fp);
|
||||
}
|
||||
|
||||
|
@ -54,7 +54,7 @@ on conflict (""Id"") do update set
|
||||
mimeType = obj.MimeType,
|
||||
digest = obj.Digest,
|
||||
editSecret = obj.EditSecret,
|
||||
expires = obj.Expires,
|
||||
expires = obj.Expires?.ToUniversalTime(),
|
||||
store = obj.Storage
|
||||
});
|
||||
}
|
||||
@ -91,8 +91,8 @@ on conflict (""Id"") do update set
|
||||
oldMeta.Description = meta.Description ?? oldMeta.Description;
|
||||
oldMeta.Name = meta.Name ?? oldMeta.Name;
|
||||
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
|
||||
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
|
||||
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
|
||||
oldMeta.Expires = meta.Expires;
|
||||
|
||||
await Set(id, oldMeta);
|
||||
}
|
||||
|
@ -53,8 +53,8 @@ public class S3FileMetadataStore : IFileMetadataStore
|
||||
oldMeta.Description = meta.Description ?? oldMeta.Description;
|
||||
oldMeta.Name = meta.Name ?? oldMeta.Name;
|
||||
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
|
||||
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
|
||||
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
|
||||
oldMeta.Expires = meta.Expires;
|
||||
|
||||
await Set(id, oldMeta);
|
||||
}
|
||||
|
@ -9,13 +9,13 @@ namespace VoidCat.Services.Files;
|
||||
/// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
|
||||
public class S3FileStore : StreamFileStore, IFileStore
|
||||
{
|
||||
private readonly IFileInfoManager _fileInfo;
|
||||
private readonly FileInfoManager _fileInfo;
|
||||
private readonly AmazonS3Client _client;
|
||||
private readonly S3BlobConfig _config;
|
||||
private readonly IAggregateStatsCollector _statsCollector;
|
||||
private readonly ICache _cache;
|
||||
|
||||
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo, ICache cache) : base(stats)
|
||||
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, FileInfoManager fileInfo, ICache cache) : base(stats)
|
||||
{
|
||||
_fileInfo = fileInfo;
|
||||
_cache = cache;
|
||||
|
@ -155,6 +155,7 @@ public static class VoidStartup
|
||||
public static void AddBackgroundServices(this IServiceCollection services, VoidSettings voidSettings)
|
||||
{
|
||||
services.AddHostedService<DeleteUnverifiedAccounts>();
|
||||
services.AddHostedService<DeleteExpiredFiles>();
|
||||
|
||||
if (voidSettings.HasVirusScanner())
|
||||
{
|
||||
|
@ -23,7 +23,11 @@ export default function ApiKeyList() {
|
||||
setNewApiKey(await rsp.json());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function openDocs() {
|
||||
window.open("/swagger", "_blank")
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (Api) {
|
||||
loadApiKeys();
|
||||
@ -38,6 +42,7 @@ export default function ApiKeyList() {
|
||||
</div>
|
||||
<div>
|
||||
<VoidButton onClick={(e) => createApiKey()}>+New</VoidButton>
|
||||
<VoidButton onClick={(e) => openDocs()}>Docs</VoidButton>
|
||||
</div>
|
||||
</div>
|
||||
<table>
|
||||
|
@ -6,6 +6,7 @@ import {useApi} from "./Api";
|
||||
import "./FileEdit.css";
|
||||
import {useSelector} from "react-redux";
|
||||
import {VoidButton} from "./VoidButton";
|
||||
import moment from "moment";
|
||||
|
||||
export function FileEdit(props) {
|
||||
const {Api} = useApi();
|
||||
@ -15,9 +16,10 @@ export function FileEdit(props) {
|
||||
const [paywall, setPaywall] = useState(file.paywall?.service);
|
||||
const [name, setName] = useState(meta?.name);
|
||||
const [description, setDescription] = useState(meta?.description);
|
||||
const [expiry, setExpiry] = useState(meta?.expires === undefined || meta?.expires === null ? null : moment(meta?.expires).unix() * 1000);
|
||||
|
||||
const privateFile = file?.uploader?.id && profile?.id === file.uploader.id
|
||||
? file
|
||||
const privateFile = file?.uploader?.id && profile?.id === file.uploader.id
|
||||
? file
|
||||
: JSON.parse(window.localStorage.getItem(file.id));
|
||||
if (!privateFile || privateFile?.metadata?.editSecret === null) {
|
||||
return null;
|
||||
@ -32,11 +34,12 @@ export function FileEdit(props) {
|
||||
let meta = {
|
||||
name,
|
||||
description,
|
||||
editSecret: privateFile?.metadata?.editSecret
|
||||
editSecret: privateFile?.metadata?.editSecret,
|
||||
expires: moment(expiry).toISOString()
|
||||
};
|
||||
await Api.updateMetadata(file.id, meta);
|
||||
await Api.updateMetadata(file.id, meta);
|
||||
}
|
||||
|
||||
|
||||
function renderPaywallConfig() {
|
||||
switch (paywall) {
|
||||
case 0: {
|
||||
@ -58,6 +61,22 @@ export function FileEdit(props) {
|
||||
<dd><input type="text" value={name} onChange={(e) => setName(e.target.value)}/></dd>
|
||||
<dt>Description:</dt>
|
||||
<dd><input type="text" value={description} onChange={(e) => setDescription(e.target.value)}/></dd>
|
||||
<dt>Expiry</dt>
|
||||
<dd>
|
||||
<input type="datetime-local"
|
||||
value={expiry === null ? "" : moment(expiry).toISOString().replace("Z", "")}
|
||||
max={moment.utc().add(1, "year").toISOString().replace("Z", "")}
|
||||
min={moment.utc().toISOString().replace("Z", "")}
|
||||
onChange={(e) => {
|
||||
console.log(e.target.value);
|
||||
if (e.target.value.length > 0) {
|
||||
setExpiry(moment.utc(e.target.value).unix() * 1000);
|
||||
} else {
|
||||
setExpiry(null);
|
||||
}
|
||||
|
||||
}}/>
|
||||
</dd>
|
||||
</dl>
|
||||
<VoidButton onClick={(e) => saveMeta()} options={{showSuccess: true}}>Save</VoidButton>
|
||||
</div>
|
||||
|
@ -60,7 +60,7 @@ a:hover {
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
input[type="text"], input[type="number"], input[type="password"], select {
|
||||
input[type="text"], input[type="number"], input[type="password"], input[type="datetime-local"], select {
|
||||
display: inline-block;
|
||||
line-height: 1.1;
|
||||
border-radius: 10px;
|
||||
|
Loading…
Reference in New Issue
Block a user