godzhigella/void.cat
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

Delete expired files

Browse Source
This commit is contained in:
Kieran 2022-09-06 22:32:22 +01:00
parent ebe5a0e106
commit 1d451aac82
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
18 changed files with 128 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
VoidCat/Controllers/Admin/AdminController.cs
View File

@ -12,11 +12,11 @@ public class AdminController : Controller
{
private readonly FileStoreFactory _fileStore;
private readonly IFileMetadataStore _fileMetadata;
private readonly IFileInfoManager _fileInfo;
private readonly FileInfoManager _fileInfo;
private readonly IUserStore _userStore;
private readonly IUserUploadsStore _userUploads;
public AdminController(FileStoreFactory fileStore, IUserStore userStore, IFileInfoManager fileInfo,
public AdminController(FileStoreFactory fileStore, IUserStore userStore, FileInfoManager fileInfo,
IFileMetadataStore fileMetadata, IUserUploadsStore userUploads)
{
_fileStore = fileStore;

4
VoidCat/Controllers/DownloadController.cs
View File

@ -11,11 +11,11 @@ namespace VoidCat.Controllers;
public class DownloadController : Controller
{
private readonly FileStoreFactory _storage;
private readonly IFileInfoManager _fileInfo;
private readonly FileInfoManager _fileInfo;
private readonly IPaywallOrderStore _paywallOrders;
private readonly ILogger<DownloadController> _logger;
public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, FileInfoManager fileInfo,
IPaywallOrderStore paywall)
{
_storage = storage;

4
VoidCat/Controllers/UploadController.cs
View File

@ -17,14 +17,14 @@ namespace VoidCat.Controllers
private readonly IFileMetadataStore _metadata;
private readonly IPaywallStore _paywall;
private readonly IPaywallFactory _paywallFactory;
private readonly IFileInfoManager _fileInfo;
private readonly FileInfoManager _fileInfo;
private readonly IUserUploadsStore _userUploads;
private readonly IUserStore _userStore;
private readonly ITimeSeriesStatsReporter _timeSeriesStats;
private readonly VoidSettings _settings;
public UploadController(FileStoreFactory storage, IFileMetadataStore metadata, IPaywallStore paywall,
IPaywallFactory paywallFactory, IFileInfoManager fileInfo, IUserUploadsStore userUploads,
IPaywallFactory paywallFactory, FileInfoManager fileInfo, IUserUploadsStore userUploads,
ITimeSeriesStatsReporter timeSeriesStats, IUserStore userStore, VoidSettings settings)
{
_storage = storage;

5
VoidCat/Controllers/UserController.cs
View File

@ -1,6 +1,7 @@
using Microsoft.AspNetCore.Mvc;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Controllers;
@ -10,10 +11,10 @@ public class UserController : Controller
private readonly IUserStore _store;
private readonly IUserUploadsStore _userUploads;
private readonly IEmailVerification _emailVerification;
private readonly IFileInfoManager _fileInfoManager;
private readonly FileInfoManager _fileInfoManager;
public UserController(IUserStore store, IUserUploadsStore userUploads, IEmailVerification emailVerification,
IFileInfoManager fileInfoManager)
FileInfoManager fileInfoManager)
{
_store = store;
_userUploads = userUploads;

38
VoidCat/Services/Abstractions/IFileInfoManager.cs
View File

@ -1,38 +0,0 @@
using VoidCat.Model;
namespace VoidCat.Services.Abstractions;
/// <summary>
/// Main interface for getting file info to serve to clients.
/// This interface should wrap all stores and return the combined result
/// </summary>
public interface IFileInfoManager
{
/// <summary>
/// Get all metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<PublicVoidFile?> Get(Guid id);
/// <summary>
/// Get all private metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<PrivateVoidFile?> GetPrivate(Guid id);
/// <summary>
/// Get all metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
/// <summary>
/// Deletes all file metadata
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask Delete(Guid id);
}

52
VoidCat/Services/Background/DeleteExpiredFiles.cs Normal file
View File

@ -0,0 +1,52 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Services.Background;
/// <summary>
/// Delete expired files
/// </summary>
public sealed class DeleteExpiredFiles : BackgroundService
{
private readonly ILogger<DeleteExpiredFiles> _logger;
private readonly IServiceScopeFactory _scopeFactory;
public DeleteExpiredFiles(ILogger<DeleteExpiredFiles> logger, IServiceScopeFactory scopeFactory)
{
_logger = logger;
_scopeFactory = scopeFactory;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
using var scope = _scopeFactory.CreateScope();
var metadata = scope.ServiceProvider.GetRequiredService<IFileMetadataStore>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
await foreach (var f in files.Results.WithCancellation(stoppingToken))
{
try
{
if (f.Expires < DateTime.Now)
{
await fileStoreFactory.DeleteFile(f.Id);
await fileInfoManager.Delete(f.Id);
_logger.LogInformation("Deleted file: {Id}", f.Id);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete file: {Id}", f.Id);
}
}
await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
}
}
}

2
VoidCat/Services/Background/DeleteUnverifiedAccounts.cs
View File

@ -25,7 +25,7 @@ public class DeleteUnverifiedAccounts : BackgroundService
var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
var fileStore = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));

31
VoidCat/Services/Files/FileInfoManager.cs
View File

@ -3,8 +3,11 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <inheritdoc />
public class FileInfoManager : IFileInfoManager
/// <summary>
/// Main interface for getting file info to serve to clients.
/// This interface should wrap all stores and return the combined result
/// </summary>
public sealed class FileInfoManager
{
private readonly IFileMetadataStore _metadataStore;
private readonly IPaywallStore _paywallStore;
@ -24,19 +27,31 @@ public class FileInfoManager : IFileInfoManager
_userUploadsStore = userUploadsStore;
}
/// <inheritdoc />
/// <summary>
/// Get all metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ValueTask<PublicVoidFile?> Get(Guid id)
{
return Get<PublicVoidFile, VoidFileMeta>(id);
}
/// <inheritdoc />
/// <summary>
/// Get all private metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
{
return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
}
/// <inheritdoc />
/// <summary>
/// Get all metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
{
var ret = new List<PublicVoidFile>();
@ -52,7 +67,11 @@ public class FileInfoManager : IFileInfoManager
return ret;
}
/// <inheritdoc />
/// <summary>
/// Deletes all file metadata
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public async ValueTask Delete(Guid id)
{
await _metadataStore.Delete(id);

4
VoidCat/Services/Files/FileStorageStartup.cs
View File

@ -8,7 +8,7 @@ public static class FileStorageStartup
{
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IFileInfoManager, FileInfoManager>();
services.AddTransient<FileInfoManager>();
services.AddTransient<FileStoreFactory>();
if (settings.CloudStorage != default)
@ -19,7 +19,7 @@ public static class FileStorageStartup
services.AddTransient<IFileStore>((svc) =>
new S3FileStore(s3,
svc.GetRequiredService<IAggregateStatsCollector>(),
svc.GetRequiredService<IFileInfoManager>(),
svc.GetRequiredService<FileInfoManager>(),
svc.GetRequiredService<ICache>()));
if (settings.MetadataStore == s3.Name)

2
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@ -52,8 +52,8 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

5
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@ -8,14 +8,12 @@ namespace VoidCat.Services.Files;
public class LocalDiskFileStore : StreamFileStore, IFileStore
{
private const string FilesDir = "files-v1";
private readonly ILogger<LocalDiskFileStore> _logger;
private readonly VoidSettings _settings;
public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats)
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
: base(stats)
{
_settings = settings;
_logger = logger;
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
if (!Directory.Exists(dir))
@ -55,7 +53,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
var fp = MapPath(id);
if (File.Exists(fp))
{
_logger.LogInformation("Deleting file: {Path}", fp);
File.Delete(fp);
}

4
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@ -54,7 +54,7 @@ on conflict (""Id"") do update set
mimeType = obj.MimeType,
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires,
expires = obj.Expires?.ToUniversalTime(),
store = obj.Storage
});
}
@ -91,8 +91,8 @@ on conflict (""Id"") do update set
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

2
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@ -53,8 +53,8 @@ public class S3FileMetadataStore : IFileMetadataStore
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

4
VoidCat/Services/Files/S3FileStore.cs
View File

@ -9,13 +9,13 @@ namespace VoidCat.Services.Files;
/// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
public class S3FileStore : StreamFileStore, IFileStore
{
private readonly IFileInfoManager _fileInfo;
private readonly FileInfoManager _fileInfo;
private readonly AmazonS3Client _client;
private readonly S3BlobConfig _config;
private readonly IAggregateStatsCollector _statsCollector;
private readonly ICache _cache;
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo, ICache cache) : base(stats)
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, FileInfoManager fileInfo, ICache cache) : base(stats)
{
_fileInfo = fileInfo;
_cache = cache;

1
VoidCat/VoidStartup.cs
View File

@ -155,6 +155,7 @@ public static class VoidStartup
public static void AddBackgroundServices(this IServiceCollection services, VoidSettings voidSettings)
{
services.AddHostedService<DeleteUnverifiedAccounts>();
services.AddHostedService<DeleteExpiredFiles>();
if (voidSettings.HasVirusScanner())
{

5
VoidCat/spa/src/ApiKeyList.js
View File

@ -24,6 +24,10 @@ export default function ApiKeyList() {
}
}
function openDocs() {
window.open("/swagger", "_blank")
}
useEffect(() => {
if (Api) {
loadApiKeys();
@ -38,6 +42,7 @@ export default function ApiKeyList() {
</div>
<div>
<VoidButton onClick={(e) => createApiKey()}>+New</VoidButton>
<VoidButton onClick={(e) => openDocs()}>Docs</VoidButton>
</div>
</div>
<table>

21
VoidCat/spa/src/FileEdit.js
View File

@ -6,6 +6,7 @@ import {useApi} from "./Api";
import "./FileEdit.css";
import {useSelector} from "react-redux";
import {VoidButton} from "./VoidButton";
import moment from "moment";
export function FileEdit(props) {
const {Api} = useApi();
@ -15,6 +16,7 @@ export function FileEdit(props) {
const [paywall, setPaywall] = useState(file.paywall?.service);
const [name, setName] = useState(meta?.name);
const [description, setDescription] = useState(meta?.description);
const [expiry, setExpiry] = useState(meta?.expires === undefined || meta?.expires === null ? null : moment(meta?.expires).unix() * 1000);
const privateFile = file?.uploader?.id && profile?.id === file.uploader.id
? file
@ -32,7 +34,8 @@ export function FileEdit(props) {
let meta = {
name,
description,
editSecret: privateFile?.metadata?.editSecret
editSecret: privateFile?.metadata?.editSecret,
expires: moment(expiry).toISOString()
};
await Api.updateMetadata(file.id, meta);
}
@ -58,6 +61,22 @@ export function FileEdit(props) {
<dd><input type="text" value={name} onChange={(e) => setName(e.target.value)}/></dd>
<dt>Description:</dt>
<dd><input type="text" value={description} onChange={(e) => setDescription(e.target.value)}/></dd>
<dt>Expiry</dt>
<dd>
<input type="datetime-local"
value={expiry === null ? "" : moment(expiry).toISOString().replace("Z", "")}
max={moment.utc().add(1, "year").toISOString().replace("Z", "")}
min={moment.utc().toISOString().replace("Z", "")}
onChange={(e) => {
console.log(e.target.value);
if (e.target.value.length > 0) {
setExpiry(moment.utc(e.target.value).unix() * 1000);
} else {
setExpiry(null);
}
}}/>
</dd>
</dl>
<VoidButton onClick={(e) => saveMeta()} options={{showSuccess: true}}>Save</VoidButton>
</div>

2
VoidCat/spa/src/index.css
View File

@ -60,7 +60,7 @@ a:hover {
align-items: center;
}
input[type="text"], input[type="number"], input[type="password"], select {
input[type="text"], input[type="number"], input[type="password"], input[type="datetime-local"], select {
display: inline-block;
line-height: 1.1;
border-radius: 10px;