Refactor stores

This commit is contained in:
Kieran 2022-06-08 17:17:53 +01:00
parent f4b1ccfe1d
commit 045399d1a2
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
35 changed files with 635 additions and 236 deletions

View File

@ -10,14 +10,17 @@ namespace VoidCat.Controllers.Admin;
public class AdminController : Controller public class AdminController : Controller
{ {
private readonly IFileStore _fileStore; private readonly IFileStore _fileStore;
private readonly IFileMetadataStore _fileMetadata;
private readonly IFileInfoManager _fileInfo; private readonly IFileInfoManager _fileInfo;
private readonly IUserStore _userStore; private readonly IUserStore _userStore;
public AdminController(IFileStore fileStore, IUserStore userStore, IFileInfoManager fileInfo) public AdminController(IFileStore fileStore, IUserStore userStore, IFileInfoManager fileInfo,
IFileMetadataStore fileMetadata)
{ {
_fileStore = fileStore; _fileStore = fileStore;
_userStore = userStore; _userStore = userStore;
_fileInfo = fileInfo; _fileInfo = fileInfo;
_fileMetadata = fileMetadata;
} }
/// <summary> /// <summary>
@ -29,7 +32,15 @@ public class AdminController : Controller
[Route("file")] [Route("file")]
public async Task<RenderedResults<PublicVoidFile>> ListFiles([FromBody] PagedRequest request) public async Task<RenderedResults<PublicVoidFile>> ListFiles([FromBody] PagedRequest request)
{ {
return await (await _fileStore.ListFiles(request)).GetResults(); var files = await _fileMetadata.ListFiles<VoidFileMeta>(request);
return new()
{
Page = files.Page,
PageSize = files.PageSize,
TotalResults = files.TotalResults,
Results = (await files.Results.SelectAwait(a => _fileInfo.Get(a.Id)).ToListAsync())!
};
} }
/// <summary> /// <summary>

View File

@ -17,15 +17,17 @@ namespace VoidCat.Controllers
private readonly IPaywallStore _paywall; private readonly IPaywallStore _paywall;
private readonly IPaywallFactory _paywallFactory; private readonly IPaywallFactory _paywallFactory;
private readonly IFileInfoManager _fileInfo; private readonly IFileInfoManager _fileInfo;
private readonly IUserUploadsStore _userUploads;
public UploadController(IFileStore storage, IFileMetadataStore metadata, IPaywallStore paywall, public UploadController(IFileStore storage, IFileMetadataStore metadata, IPaywallStore paywall,
IPaywallFactory paywallFactory, IFileInfoManager fileInfo) IPaywallFactory paywallFactory, IFileInfoManager fileInfo, IUserUploadsStore userUploads)
{ {
_storage = storage; _storage = storage;
_metadata = metadata; _metadata = metadata;
_paywall = paywall; _paywall = paywall;
_paywallFactory = paywallFactory; _paywallFactory = paywallFactory;
_fileInfo = fileInfo; _fileInfo = fileInfo;
_userUploads = userUploads;
} }
/// <summary> /// <summary>
@ -76,6 +78,15 @@ namespace VoidCat.Controllers
Hash = digest Hash = digest
}, HttpContext.RequestAborted); }, HttpContext.RequestAborted);
// save metadata
await _metadata.Set(vf.Id, vf.Metadata!);
// attach file upload to user
if (uid.HasValue)
{
await _userUploads.AddFile(uid!.Value, vf);
}
if (cli) if (cli)
{ {
var urlBuilder = new UriBuilder(Request.IsHttps ? "https" : "http", Request.Host.Host, var urlBuilder = new UriBuilder(Request.IsHttps ? "https" : "http", Request.Host.Host,
@ -126,7 +137,9 @@ namespace VoidCat.Controllers
Id = gid, Id = gid,
IsAppend = true IsAppend = true
}, HttpContext.RequestAborted); }, HttpContext.RequestAborted);
// update file size
await _metadata.Set(vf.Id, vf.Metadata!);
return UploadResult.Success(vf); return UploadResult.Success(vf);
} }
catch (Exception ex) catch (Exception ex)

View File

@ -134,7 +134,7 @@ public class UserController : Controller
if (!await _emailVerification.VerifyCode(user, token)) return BadRequest(); if (!await _emailVerification.VerifyCode(user, token)) return BadRequest();
user.Flags |= VoidUserFlags.EmailVerified; user.Flags |= VoidUserFlags.EmailVerified;
await _store.Set(user.Id, user); await _store.UpdateProfile(user.ToPublic());
return Accepted(); return Accepted();
} }

View File

@ -1,8 +1,9 @@
namespace VoidCat.Model; namespace VoidCat.Model;
public class EmailVerificationCode /// <summary>
{ /// Email verification token
public Guid Id { get; init; } = Guid.NewGuid(); /// </summary>
public Guid UserId { get; init; } /// <param name="Id"></param>
public DateTimeOffset Expires { get; init; } /// <param name="User"></param>
} /// <param name="Expires"></param>
public sealed record EmailVerificationCode(Guid User, Guid Code, DateTime Expires);

View File

@ -206,7 +206,7 @@ public static class Extensions
public static bool CheckPassword(this InternalVoidUser vu, string password) public static bool CheckPassword(this InternalVoidUser vu, string password)
{ {
var hashParts = vu.PasswordHash.Split(":"); var hashParts = vu.Password.Split(":");
return vu.PasswordHash == password.Hash(hashParts[0], hashParts.Length == 3 ? hashParts[1] : null); return vu.Password == password.Hash(hashParts[0], hashParts.Length == 3 ? hashParts[1] : null);
} }
} }

View File

@ -25,6 +25,12 @@ public record VoidFileMeta : IVoidFileMeta
/// </summary> /// </summary>
public int Version { get; init; } = IVoidFileMeta.CurrentVersion; public int Version { get; init; } = IVoidFileMeta.CurrentVersion;
/// <summary>
/// Internal Id of the file
/// </summary>
[JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; set; }
/// <summary> /// <summary>
/// Filename /// Filename
/// </summary> /// </summary>

View File

@ -9,16 +9,11 @@ namespace VoidCat.Model;
/// </summary> /// </summary>
public abstract class VoidUser public abstract class VoidUser
{ {
protected VoidUser(Guid id)
{
Id = id;
}
/// <summary> /// <summary>
/// Unique Id of the user /// Unique Id of the user
/// </summary> /// </summary>
[JsonConverter(typeof(Base58GuidConverter))] [JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; } public Guid Id { get; init; }
/// <summary> /// <summary>
/// Roles assigned to this user which grant them extra permissions /// Roles assigned to this user which grant them extra permissions
@ -56,12 +51,14 @@ public abstract class VoidUser
/// <returns></returns> /// <returns></returns>
public PublicVoidUser ToPublic() public PublicVoidUser ToPublic()
{ {
return new(Id) return new()
{ {
Id = Id,
Roles = Roles, Roles = Roles,
Created = Created, Created = Created,
LastLogin = LastLogin, LastLogin = LastLogin,
Avatar = Avatar Avatar = Avatar,
Flags = Flags
}; };
} }
} }
@ -71,24 +68,10 @@ public abstract class VoidUser
/// </summary> /// </summary>
public sealed class InternalVoidUser : PrivateVoidUser public sealed class InternalVoidUser : PrivateVoidUser
{ {
/// <inheritdoc />
public InternalVoidUser(Guid id, string email, string passwordHash) : base(id, email)
{
PasswordHash = passwordHash;
}
/// <inheritdoc />
public InternalVoidUser(Guid Id, string Email, string Password, DateTime Created, DateTime LastLogin,
string Avatar,
string DisplayName, int Flags) : base(Id, Email)
{
PasswordHash = Password;
}
/// <summary> /// <summary>
/// A password hash for the user in the format <see cref="Extensions.HashPassword"/> /// A password hash for the user in the format <see cref="Extensions.HashPassword"/>
/// </summary> /// </summary>
public string PasswordHash { get; } public string Password { get; init; } = null!;
} }
/// <summary> /// <summary>
@ -96,44 +79,15 @@ public sealed class InternalVoidUser : PrivateVoidUser
/// </summary> /// </summary>
public class PrivateVoidUser : VoidUser public class PrivateVoidUser : VoidUser
{ {
/// <inheritdoc />
public PrivateVoidUser(Guid id, string email) : base(id)
{
Email = email;
}
/// <summary> /// <summary>
/// Full constructor for Dapper /// Users email address
/// </summary> /// </summary>
/// <param name="id"></param> public string Email { get; init; } = null!;
/// <param name="email"></param>
/// <param name="password"></param>
/// <param name="created"></param>
/// <param name="last_login"></param>
/// <param name="avatar"></param>
/// <param name="display_name"></param>
/// <param name="flags"></param>
public PrivateVoidUser(Guid Id, String Email, string Password, DateTime Created, DateTime LastLogin, string Avatar,
string DisplayName, int Flags) : base(Id)
{
this.Email = Email;
this.Created = Created;
this.LastLogin = LastLogin;
this.Avatar = Avatar;
this.DisplayName = DisplayName;
this.Flags = (VoidUserFlags) Flags;
}
public string Email { get; }
} }
/// <inheritdoc /> /// <inheritdoc />
public sealed class PublicVoidUser : VoidUser public sealed class PublicVoidUser : VoidUser
{ {
/// <inheritdoc />
public PublicVoidUser(Guid id) : base(id)
{
}
} }
[Flags] [Flags]

View File

@ -1,4 +1,5 @@
@using VoidCat.Model @using VoidCat.Model
@using VoidCat.Services.Users
@model VoidCat.Model.EmailVerificationCode @model VoidCat.Model.EmailVerificationCode
<!DOCTYPE html> <!DOCTYPE html>
@ -30,7 +31,8 @@
<div class="page"> <div class="page">
<h1>void.cat</h1> <h1>void.cat</h1>
<p>Your verification code is below please copy this to complete verification</p> <p>Your verification code is below please copy this to complete verification</p>
<pre>@(Model?.Id.ToBase58() ?? "?????????????")</pre> <pre>@(Model?.Code.ToBase58() ?? "?????????????")</pre>
<p>This code will expire in @BaseEmailVerification.HoursExpire hours</p>
</div> </div>
</body> </body>
</html> </html>

View File

@ -131,6 +131,7 @@ services.AddAuthorization((opt) =>
// void.cat services // void.cat services
// //
services.AddTransient<RazorPartialToStringRenderer>(); services.AddTransient<RazorPartialToStringRenderer>();
services.AddTransient<IMigration, PopulateMetadataId>();
// file storage // file storage
services.AddStorage(voidSettings); services.AddStorage(voidSettings);
@ -167,8 +168,7 @@ if (!string.IsNullOrEmpty(voidSettings.Postgres))
.ConfigureRunner(r => .ConfigureRunner(r =>
r.AddPostgres11_0() r.AddPostgres11_0()
.WithGlobalConnectionString(voidSettings.Postgres) .WithGlobalConnectionString(voidSettings.Postgres)
.ScanIn(typeof(Program).Assembly).For.Migrations()) .ScanIn(typeof(Program).Assembly).For.Migrations());
.AddLogging(l => l.AddFluentMigratorConsole());
} }
if (useRedis) if (useRedis)
@ -196,10 +196,13 @@ var app = builder.Build();
using (var migrationScope = app.Services.CreateScope()) using (var migrationScope = app.Services.CreateScope())
{ {
var migrations = migrationScope.ServiceProvider.GetServices<IMigration>(); var migrations = migrationScope.ServiceProvider.GetServices<IMigration>();
var logger = migrationScope.ServiceProvider.GetRequiredService<ILogger<IMigration>>();
foreach (var migration in migrations) foreach (var migration in migrations)
{ {
await migration.Migrate(args); logger.LogInformation("Running migration: {Migration}", migration.GetType().Name);
if (migration.ExitOnComplete) var res = await migration.Migrate(args);
logger.LogInformation("== Result: {Result}", res.ToString());
if (res == IMigration.MigrationResult.ExitCompleted)
{ {
return; return;
} }

View File

@ -8,7 +8,18 @@ namespace VoidCat.Services.Abstractions;
/// </summary> /// </summary>
public interface IFileInfoManager public interface IFileInfoManager
{ {
/// <summary>
/// Get all metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<PublicVoidFile?> Get(Guid id); ValueTask<PublicVoidFile?> Get(Guid id);
/// <summary>
/// Get all metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids); ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
/// <summary> /// <summary>

View File

@ -2,11 +2,43 @@ using VoidCat.Model;
namespace VoidCat.Services.Abstractions; namespace VoidCat.Services.Abstractions;
/// <summary>
/// File metadata contains all data about a file except for the file data itself
/// </summary>
public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta> public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta>
{ {
/// <summary>
/// Get metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta; ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta;
/// <summary>
/// Get metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta; ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta;
/// <summary>
/// Update file metadata
/// </summary>
/// <param name="id"></param>
/// <param name="meta"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta; ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta;
/// <summary>
/// List all files in the store
/// </summary>
/// <param name="request"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta;
/// <summary> /// <summary>
/// Returns basic stats about the file store /// Returns basic stats about the file store
@ -14,5 +46,10 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <returns></returns> /// <returns></returns>
ValueTask<StoreStats> Stats(); ValueTask<StoreStats> Stats();
/// <summary>
/// Simple stats of the current store
/// </summary>
/// <param name="Files"></param>
/// <param name="Size"></param>
public sealed record StoreStats(long Files, ulong Size); public sealed record StoreStats(long Files, ulong Size);
} }

View File

@ -2,14 +2,28 @@
namespace VoidCat.Services.Abstractions; namespace VoidCat.Services.Abstractions;
/// <summary>
/// File binary data store
/// </summary>
public interface IFileStore public interface IFileStore
{ {
/// <summary>
/// Ingress a file into the system (Upload)
/// </summary>
/// <param name="payload"></param>
/// <param name="cts"></param>
/// <returns></returns>
ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts); ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts);
/// <summary>
/// Egress a file from the system (Download)
/// </summary>
/// <param name="request"></param>
/// <param name="outStream"></param>
/// <param name="cts"></param>
/// <returns></returns>
ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts); ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts);
ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request);
/// <summary> /// <summary>
/// Deletes file data only, metadata must be deleted with <see cref="IFileInfoManager.Delete"/> /// Deletes file data only, metadata must be deleted with <see cref="IFileInfoManager.Delete"/>
/// </summary> /// </summary>
@ -17,5 +31,11 @@ public interface IFileStore
/// <returns></returns> /// <returns></returns>
ValueTask DeleteFile(Guid id); ValueTask DeleteFile(Guid id);
/// <summary>
/// Open a filestream for a file on the system
/// </summary>
/// <param name="request"></param>
/// <param name="cts"></param>
/// <returns></returns>
ValueTask<Stream> Open(EgressRequest request, CancellationToken cts); ValueTask<Stream> Open(EgressRequest request, CancellationToken cts);
} }

View File

@ -1,10 +1,38 @@
namespace VoidCat.Services.Abstractions; namespace VoidCat.Services.Abstractions;
public interface IPublicPrivateStore<TPublic, in TPrivate> /// <summary>
/// Store interface where there is a public and private model
/// </summary>
/// <typeparam name="TPublic"></typeparam>
/// <typeparam name="TPrivate"></typeparam>
public interface IPublicPrivateStore<TPublic, TPrivate>
{ {
/// <summary>
/// Get the public model
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<TPublic?> Get(Guid id); ValueTask<TPublic?> Get(Guid id);
/// <summary>
/// Get the private model (contains sensitive data)
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<TPrivate?> GetPrivate(Guid id);
/// <summary>
/// Set the private obj in the store
/// </summary>
/// <param name="id"></param>
/// <param name="obj"></param>
/// <returns></returns>
ValueTask Set(Guid id, TPrivate obj); ValueTask Set(Guid id, TPrivate obj);
/// <summary>
/// Delete the object from the store
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask Delete(Guid id); ValueTask Delete(Guid id);
} }

View File

@ -2,11 +2,45 @@
namespace VoidCat.Services.Abstractions; namespace VoidCat.Services.Abstractions;
/// <summary>
/// User store
/// </summary>
public interface IUserStore : IPublicPrivateStore<VoidUser, InternalVoidUser> public interface IUserStore : IPublicPrivateStore<VoidUser, InternalVoidUser>
{ {
/// <summary>
/// Get a single user
/// </summary>
/// <param name="id"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
ValueTask<T?> Get<T>(Guid id) where T : VoidUser; ValueTask<T?> Get<T>(Guid id) where T : VoidUser;
/// <summary>
/// Lookup a user by their email address
/// </summary>
/// <param name="email"></param>
/// <returns></returns>
ValueTask<Guid?> LookupUser(string email); ValueTask<Guid?> LookupUser(string email);
/// <summary>
/// List all users in the system
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request); ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request);
/// <summary>
/// Update a users profile
/// </summary>
/// <param name="newUser"></param>
/// <returns></returns>
ValueTask UpdateProfile(PublicVoidUser newUser); ValueTask UpdateProfile(PublicVoidUser newUser);
/// <summary>
/// Updates the last login timestamp for the user
/// </summary>
/// <param name="id"></param>
/// <param name="timestamp"></param>
/// <returns></returns>
ValueTask UpdateLastLogin(Guid id, DateTime timestamp);
} }

View File

@ -1,4 +1,5 @@
using VoidCat.Services.Abstractions; using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.VirusScanner.Exceptions; using VoidCat.Services.VirusScanner.Exceptions;
namespace VoidCat.Services.Background; namespace VoidCat.Services.Background;
@ -7,11 +8,11 @@ public class VirusScannerService : BackgroundService
{ {
private readonly ILogger<VirusScannerService> _logger; private readonly ILogger<VirusScannerService> _logger;
private readonly IVirusScanner _scanner; private readonly IVirusScanner _scanner;
private readonly IFileStore _fileStore; private readonly IFileMetadataStore _fileStore;
private readonly IVirusScanStore _scanStore; private readonly IVirusScanStore _scanStore;
public VirusScannerService(ILogger<VirusScannerService> logger, IVirusScanner scanner, IVirusScanStore scanStore, public VirusScannerService(ILogger<VirusScannerService> logger, IVirusScanner scanner, IVirusScanStore scanStore,
IFileStore fileStore) IFileMetadataStore fileStore)
{ {
_scanner = scanner; _scanner = scanner;
_logger = logger; _logger = logger;
@ -28,7 +29,7 @@ public class VirusScannerService : BackgroundService
var page = 0; var page = 0;
while (true) while (true)
{ {
var files = await _fileStore.ListFiles(new(page, 10)); var files = await _fileStore.ListFiles<VoidFileMeta>(new(page, 10));
if (files.Pages < page) break; if (files.Pages < page) break;
page++; page++;

View File

@ -3,6 +3,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
/// <inheritdoc />
public class FileInfoManager : IFileInfoManager public class FileInfoManager : IFileInfoManager
{ {
private readonly IFileMetadataStore _metadataStore; private readonly IFileMetadataStore _metadataStore;
@ -21,6 +22,7 @@ public class FileInfoManager : IFileInfoManager
_virusScanStore = virusScanStore; _virusScanStore = virusScanStore;
} }
/// <inheritdoc />
public async ValueTask<PublicVoidFile?> Get(Guid id) public async ValueTask<PublicVoidFile?> Get(Guid id)
{ {
var meta = _metadataStore.Get<VoidFileMeta>(id); var meta = _metadataStore.Get<VoidFileMeta>(id);
@ -45,6 +47,7 @@ public class FileInfoManager : IFileInfoManager
}; };
} }
/// <inheritdoc />
public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids) public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
{ {
var ret = new List<PublicVoidFile>(); var ret = new List<PublicVoidFile>();
@ -60,6 +63,7 @@ public class FileInfoManager : IFileInfoManager
return ret; return ret;
} }
/// <inheritdoc />
public async ValueTask Delete(Guid id) public async ValueTask Delete(Guid id)
{ {
await _metadataStore.Delete(id); await _metadataStore.Delete(id);

View File

@ -25,7 +25,7 @@ public static class FileStorageStartup
{ {
services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>(); services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>(); services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, PostgreFileMetadataStore>(); services.AddTransient<IFileMetadataStore, PostgresFileMetadataStore>();
} }
else else
{ {

View File

@ -4,6 +4,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
/// <inheritdoc />
public class LocalDiskFileMetadataStore : IFileMetadataStore public class LocalDiskFileMetadataStore : IFileMetadataStore
{ {
private const string MetadataDir = "metadata-v3"; private const string MetadataDir = "metadata-v3";
@ -22,11 +23,13 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
} }
} }
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{ {
return GetMeta<TMeta>(id); return GetMeta<TMeta>(id);
} }
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{ {
var ret = new List<TMeta>(); var ret = new List<TMeta>();
@ -42,6 +45,7 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
return ret; return ret;
} }
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{ {
var oldMeta = await Get<SecretVoidFileMeta>(id); var oldMeta = await Get<SecretVoidFileMeta>(id);
@ -54,37 +58,69 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
await Set(id, oldMeta); await Set(id, oldMeta);
} }
public async ValueTask<IFileMetadataStore.StoreStats> Stats() /// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
{ {
var count = 0; async IAsyncEnumerable<TMeta> EnumerateFiles()
var size = 0UL;
foreach (var metaFile in Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json"))
{ {
try foreach (var metaFile in
Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json"))
{ {
var json = await File.ReadAllTextAsync(metaFile); var json = await File.ReadAllTextAsync(metaFile);
var meta = JsonConvert.DeserializeObject<VoidFileMeta>(json); var meta = JsonConvert.DeserializeObject<TMeta>(json);
if (meta != null) if (meta != null)
{ {
count++; yield return meta with
size += meta.Size; {
// TODO: remove after migration decay
Id = Guid.Parse(Path.GetFileNameWithoutExtension(metaFile))
};
} }
} }
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load metadata file: {File}", metaFile);
}
} }
return new(count, size); var results = EnumerateFiles();
results = (request.SortBy, request.SortOrder) switch
{
(PagedSortBy.Name, PageSortOrder.Asc) => results.OrderBy(a => a.Name),
(PagedSortBy.Size, PageSortOrder.Asc) => results.OrderBy(a => a.Size),
(PagedSortBy.Date, PageSortOrder.Asc) => results.OrderBy(a => a.Uploaded),
(PagedSortBy.Name, PageSortOrder.Dsc) => results.OrderByDescending(a => a.Name),
(PagedSortBy.Size, PageSortOrder.Dsc) => results.OrderByDescending(a => a.Size),
(PagedSortBy.Date, PageSortOrder.Dsc) => results.OrderByDescending(a => a.Uploaded),
_ => results
};
return ValueTask.FromResult(new PagedResult<TMeta>
{
Page = request.Page,
PageSize = request.PageSize,
Results = results.Take(request.PageSize).Skip(request.Page * request.PageSize)
});
} }
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id) public ValueTask<VoidFileMeta?> Get(Guid id)
{ {
return GetMeta<VoidFileMeta>(id); return GetMeta<VoidFileMeta>(id);
} }
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta) public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
{ {
var path = MapMeta(id); var path = MapMeta(id);
@ -92,6 +128,7 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
await File.WriteAllTextAsync(path, json); await File.WriteAllTextAsync(path, json);
} }
/// <inheritdoc />
public ValueTask Delete(Guid id) public ValueTask Delete(Guid id)
{ {
var path = MapMeta(id); var path = MapMeta(id);

View File

@ -4,21 +4,17 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
/// <inheritdoc cref="IFileStore"/>
public class LocalDiskFileStore : StreamFileStore, IFileStore public class LocalDiskFileStore : StreamFileStore, IFileStore
{ {
private const string FilesDir = "files-v1"; private const string FilesDir = "files-v1";
private readonly ILogger<LocalDiskFileStore> _logger; private readonly ILogger<LocalDiskFileStore> _logger;
private readonly VoidSettings _settings; private readonly VoidSettings _settings;
private readonly IFileMetadataStore _metadataStore;
private readonly IFileInfoManager _fileInfo;
public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats, public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats)
IFileMetadataStore metadataStore, IFileInfoManager fileInfo, IUserUploadsStore userUploads) : base(stats)
: base(stats, metadataStore, userUploads)
{ {
_settings = settings; _settings = settings;
_metadataStore = metadataStore;
_fileInfo = fileInfo;
_logger = logger; _logger = logger;
var dir = Path.Combine(_settings.DataDirectory, FilesDir); var dir = Path.Combine(_settings.DataDirectory, FilesDir);
@ -28,12 +24,14 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
} }
} }
/// <inheritdoc />
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts) public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{ {
await using var fs = await Open(request, cts); await using var fs = await Open(request, cts);
await EgressFromStream(fs, request, outStream, cts); await EgressFromStream(fs, request, outStream, cts);
} }
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts) public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{ {
var fPath = MapPath(payload.Id); var fPath = MapPath(payload.Id);
@ -42,49 +40,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
return await IngressToStream(fsTemp, payload, cts); return await IngressToStream(fsTemp, payload, cts);
} }
public ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request) /// <inheritdoc />
{
var files = Directory.EnumerateFiles(Path.Combine(_settings.DataDirectory, FilesDir))
.Where(a => !Path.HasExtension(a));
files = (request.SortBy, request.SortOrder) switch
{
(PagedSortBy.Id, PageSortOrder.Asc) => files.OrderBy(a =>
Guid.TryParse(Path.GetFileNameWithoutExtension(a), out var g) ? g : Guid.Empty),
(PagedSortBy.Id, PageSortOrder.Dsc) => files.OrderByDescending(a =>
Guid.TryParse(Path.GetFileNameWithoutExtension(a), out var g) ? g : Guid.Empty),
(PagedSortBy.Name, PageSortOrder.Asc) => files.OrderBy(Path.GetFileNameWithoutExtension),
(PagedSortBy.Name, PageSortOrder.Dsc) => files.OrderByDescending(Path.GetFileNameWithoutExtension),
(PagedSortBy.Size, PageSortOrder.Asc) => files.OrderBy(a => new FileInfo(a).Length),
(PagedSortBy.Size, PageSortOrder.Dsc) => files.OrderByDescending(a => new FileInfo(a).Length),
(PagedSortBy.Date, PageSortOrder.Asc) => files.OrderBy(File.GetCreationTimeUtc),
(PagedSortBy.Date, PageSortOrder.Dsc) => files.OrderByDescending(File.GetCreationTimeUtc),
_ => files
};
async IAsyncEnumerable<PublicVoidFile> EnumeratePage(IEnumerable<string> page)
{
foreach (var file in page)
{
if (!Guid.TryParse(Path.GetFileNameWithoutExtension(file), out var gid)) continue;
var loaded = await _fileInfo.Get(gid);
if (loaded != default)
{
yield return loaded;
}
}
}
return ValueTask.FromResult(new PagedResult<PublicVoidFile>()
{
Page = request.Page,
PageSize = request.PageSize,
TotalResults = files.Count(),
Results = EnumeratePage(files.Skip(request.PageSize * request.Page).Take(request.PageSize))
});
}
public ValueTask DeleteFile(Guid id) public ValueTask DeleteFile(Guid id)
{ {
var fp = MapPath(id); var fp = MapPath(id);
@ -93,9 +49,11 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
_logger.LogInformation("Deleting file: {Path}", fp); _logger.LogInformation("Deleting file: {Path}", fp);
File.Delete(fp); File.Delete(fp);
} }
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
/// <inheritdoc />
public ValueTask<Stream> Open(EgressRequest request, CancellationToken cts) public ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
{ {
var path = MapPath(request.Id); var path = MapPath(request.Id);

View File

@ -5,20 +5,29 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
public class PostgreFileMetadataStore : IFileMetadataStore /// <inheritdoc />
public class PostgresFileMetadataStore : IFileMetadataStore
{ {
private readonly NpgsqlConnection _connection; private readonly NpgsqlConnection _connection;
public PostgreFileMetadataStore(NpgsqlConnection connection) public PostgresFileMetadataStore(NpgsqlConnection connection)
{ {
_connection = connection; _connection = connection;
} }
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id) public ValueTask<VoidFileMeta?> Get(Guid id)
{ {
return Get<VoidFileMeta>(id); return Get<VoidFileMeta>(id);
} }
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
{
return Get<SecretVoidFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta obj) public async ValueTask Set(Guid id, SecretVoidFileMeta obj)
{ {
await _connection.ExecuteAsync( await _connection.ExecuteAsync(
@ -38,23 +47,27 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Description"" = :descript
}); });
} }
/// <inheritdoc />
public async ValueTask Delete(Guid id) public async ValueTask Delete(Guid id)
{ {
await _connection.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id}); await _connection.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id});
} }
/// <inheritdoc />
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{ {
return await _connection.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id", return await _connection.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
new {id}); new {id});
} }
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{ {
var ret = await _connection.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids}); var ret = await _connection.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
return ret.ToList(); return ret.ToList();
} }
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{ {
var oldMeta = await Get<SecretVoidFileMeta>(id); var oldMeta = await Get<SecretVoidFileMeta>(id);
@ -67,6 +80,43 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Description"" = :descript
await Set(id, oldMeta); await Set(id, oldMeta);
} }
/// <inheritdoc />
public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
{
var qInner = @"select {0} from ""Files"" order by ""{1}"" {2}";
var orderBy = request.SortBy switch
{
PagedSortBy.Date => "Uploaded",
PagedSortBy.Name => "Name",
PagedSortBy.Size => "Size",
_ => "Id"
};
var orderDirection = request.SortOrder == PageSortOrder.Asc ? "asc" : "desc";
var count = await _connection.ExecuteScalarAsync<int>(string.Format(qInner, "count(*)", orderBy,
orderDirection));
async IAsyncEnumerable<TMeta> Enumerate()
{
var results = await _connection.QueryAsync<TMeta>(
$"{string.Format(qInner, "*", orderBy, orderDirection)} offset @offset limit @limit",
new {offset = request.PageSize * request.Page, limit = request.PageSize});
foreach (var meta in results)
{
yield return meta;
}
}
return new()
{
TotalResults = count,
PageSize = request.PageSize,
Page = request.Page,
Results = Enumerate()
};
}
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats() public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{ {
var v = await _connection.QuerySingleAsync<(long Files, long Size)>( var v = await _connection.QuerySingleAsync<(long Files, long Size)>(

View File

@ -5,6 +5,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
/// <inheritdoc />
public class S3FileMetadataStore : IFileMetadataStore public class S3FileMetadataStore : IFileMetadataStore
{ {
private readonly ILogger<S3FileMetadataStore> _logger; private readonly ILogger<S3FileMetadataStore> _logger;
@ -20,11 +21,13 @@ public class S3FileMetadataStore : IFileMetadataStore
_client = _config.CreateClient(); _client = _config.CreateClient();
} }
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{ {
return GetMeta<TMeta>(id); return GetMeta<TMeta>(id);
} }
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{ {
var ret = new List<TMeta>(); var ret = new List<TMeta>();
@ -40,6 +43,7 @@ public class S3FileMetadataStore : IFileMetadataStore
return ret; return ret;
} }
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{ {
var oldMeta = await GetMeta<SecretVoidFileMeta>(id); var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
@ -52,11 +56,10 @@ public class S3FileMetadataStore : IFileMetadataStore
await Set(id, oldMeta); await Set(id, oldMeta);
} }
public async ValueTask<IFileMetadataStore.StoreStats> Stats() /// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
{ {
var count = 0; async IAsyncEnumerable<TMeta> Enumerate()
var size = 0UL;
try
{ {
var obj = await _client.ListObjectsV2Async(new() var obj = await _client.ListObjectsV2Async(new()
{ {
@ -69,28 +72,45 @@ public class S3FileMetadataStore : IFileMetadataStore
{ {
if (Guid.TryParse(file.Key.Split("metadata_")[1], out var id)) if (Guid.TryParse(file.Key.Split("metadata_")[1], out var id))
{ {
var meta = await GetMeta<VoidFileMeta>(id); var meta = await GetMeta<TMeta>(id);
if (meta != default) if (meta != default)
{ {
count++; yield return meta;
size += meta.Size;
} }
} }
} }
} }
catch (AmazonS3Exception aex)
{
_logger.LogError(aex, "Failed to list files: {Error}", aex.Message);
}
return new(count, size); return ValueTask.FromResult(new PagedResult<TMeta>
{
Page = request.Page,
PageSize = request.PageSize,
Results = Enumerate().Skip(request.PageSize * request.Page).Take(request.PageSize)
});
} }
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id) public ValueTask<VoidFileMeta?> Get(Guid id)
{ {
return GetMeta<VoidFileMeta>(id); return GetMeta<VoidFileMeta>(id);
} }
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta) public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
{ {
await _client.PutObjectAsync(new() await _client.PutObjectAsync(new()
@ -102,6 +122,7 @@ public class S3FileMetadataStore : IFileMetadataStore
}); });
} }
/// <inheritdoc />
public async ValueTask Delete(Guid id) public async ValueTask Delete(Guid id)
{ {
await _client.DeleteObjectAsync(_config.BucketName, ToKey(id)); await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
@ -116,14 +137,18 @@ public class S3FileMetadataStore : IFileMetadataStore
using var sr = new StreamReader(obj.ResponseStream); using var sr = new StreamReader(obj.ResponseStream);
var json = await sr.ReadToEndAsync(); var json = await sr.ReadToEndAsync();
var ret = JsonConvert.DeserializeObject<TMeta>(json); var ret = JsonConvert.DeserializeObject<TMeta>(json);
if (ret != default && _includeUrl) if (ret != default)
{ {
var ub = new UriBuilder(_config.ServiceUrl!) ret.Id = id;
if (_includeUrl)
{ {
Path = $"/{_config.BucketName}/{id}" var ub = new UriBuilder(_config.ServiceUrl!)
}; {
Path = $"/{_config.BucketName}/{id}"
};
ret.Url = ub.Uri; ret.Url = ub.Uri;
}
} }
return ret; return ret;

View File

@ -12,8 +12,7 @@ public class S3FileStore : StreamFileStore, IFileStore
private readonly S3BlobConfig _config; private readonly S3BlobConfig _config;
private readonly IAggregateStatsCollector _statsCollector; private readonly IAggregateStatsCollector _statsCollector;
public S3FileStore(VoidSettings settings, IAggregateStatsCollector stats, IFileMetadataStore metadataStore, public S3FileStore(VoidSettings settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo) : base(stats)
IUserUploadsStore userUploads, IFileInfoManager fileInfo) : base(stats, metadataStore, userUploads)
{ {
_fileInfo = fileInfo; _fileInfo = fileInfo;
_statsCollector = stats; _statsCollector = stats;

View File

@ -6,19 +6,17 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files; namespace VoidCat.Services.Files;
/// <summary>
/// File store based on <see cref="Stream"/> objects
/// </summary>
public abstract class StreamFileStore public abstract class StreamFileStore
{ {
private const int BufferSize = 1_048_576; private const int BufferSize = 1_048_576;
private readonly IAggregateStatsCollector _stats; private readonly IAggregateStatsCollector _stats;
private readonly IFileMetadataStore _metadataStore;
private readonly IUserUploadsStore _userUploads;
protected StreamFileStore(IAggregateStatsCollector stats, IFileMetadataStore metadataStore, protected StreamFileStore(IAggregateStatsCollector stats)
IUserUploadsStore userUploads)
{ {
_stats = stats; _stats = stats;
_metadataStore = metadataStore;
_userUploads = userUploads;
} }
protected async ValueTask EgressFromStream(Stream stream, EgressRequest request, Stream outStream, protected async ValueTask EgressFromStream(Stream stream, EgressRequest request, Stream outStream,
@ -77,22 +75,17 @@ public abstract class StreamFileStore
}; };
} }
await _metadataStore.Set(payload.Id, meta);
var vf = new PrivateVoidFile() var vf = new PrivateVoidFile()
{ {
Id = payload.Id, Id = payload.Id,
Metadata = meta Metadata = meta
}; };
if (meta.Uploader.HasValue)
{
await _userUploads.AddFile(meta.Uploader.Value, vf);
}
return vf; return vf;
} }
private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream outStream, CancellationToken cts) private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream outStream,
CancellationToken cts)
{ {
using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize); using var buffer = MemoryPool<byte>.Shared.Rent(BufferSize);
var total = 0UL; var total = 0UL;

View File

@ -11,11 +11,9 @@ public class FluentMigrationRunner : IMigration
_runner = runner; _runner = runner;
} }
public ValueTask Migrate(string[] args) public ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{ {
_runner.MigrateUp(); _runner.MigrateUp();
return ValueTask.CompletedTask; return ValueTask.FromResult(IMigration.MigrationResult.Completed);
} }
public bool ExitOnComplete => false;
} }

View File

@ -2,6 +2,23 @@
public interface IMigration public interface IMigration
{ {
ValueTask Migrate(string[] args); ValueTask<MigrationResult> Migrate(string[] args);
bool ExitOnComplete { get; }
public enum MigrationResult
{
/// <summary>
/// Migration was not run
/// </summary>
Skipped,
/// <summary>
/// Migration completed successfully, continue to startup
/// </summary>
Completed,
/// <summary>
/// Migration completed Successfully, exit application
/// </summary>
ExitCompleted
}
} }

View File

@ -14,7 +14,7 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
_logger = logger; _logger = logger;
} }
public async ValueTask Migrate(string[] args) public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{ {
var newMeta = Path.Combine(_settings.DataDirectory, OldPath); var newMeta = Path.Combine(_settings.DataDirectory, OldPath);
if (!Directory.Exists(newMeta)) if (!Directory.Exists(newMeta))
@ -51,6 +51,8 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
} }
} }
} }
return IMigration.MigrationResult.Completed;
} }
protected abstract string OldPath { get; } protected abstract string OldPath { get; }
@ -64,6 +66,4 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
private string MapNewMeta(Guid id) => private string MapNewMeta(Guid id) =>
Path.ChangeExtension(Path.Join(_settings.DataDirectory, NewPath, id.ToString()), ".json"); Path.ChangeExtension(Path.Join(_settings.DataDirectory, NewPath, id.ToString()), ".json");
public bool ExitOnComplete => false;
} }

View File

@ -0,0 +1,31 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Migrations;
public class PopulateMetadataId : IMigration
{
private readonly IFileMetadataStore _metadataStore;
public PopulateMetadataId(IFileMetadataStore metadataStore)
{
_metadataStore = metadataStore;
}
public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{
if (!args.Contains("--add-metadata-id"))
{
return IMigration.MigrationResult.Skipped;
}
var files = await _metadataStore.ListFiles<SecretVoidFileMeta>(new(0, Int32.MaxValue));
await foreach (var file in files.Results)
{
// read-write file metadata
await _metadataStore.Set(file.Id, file);
}
return IMigration.MigrationResult.ExitCompleted;
}
}

View File

@ -13,7 +13,7 @@ public class UserLookupKeyHashMigration : IMigration
_database = database; _database = database;
} }
public async ValueTask Migrate(string[] args) public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{ {
var users = await _database.SetMembersAsync("users"); var users = await _database.SetMembersAsync("users");
foreach (var userId in users) foreach (var userId in users)
@ -30,6 +30,8 @@ public class UserLookupKeyHashMigration : IMigration
await _database.StringSetAsync(MapNew(user.Email), $"\"{userId}\""); await _database.StringSetAsync(MapNew(user.Email), $"\"{userId}\"");
} }
} }
return IMigration.MigrationResult.Completed;
} }
private static RedisKey MapOld(string email) => $"user:email:{email}"; private static RedisKey MapOld(string email) => $"user:email:{email}";
@ -41,6 +43,4 @@ public class UserLookupKeyHashMigration : IMigration
public string Email { get; init; } public string Email { get; init; }
} }
public bool ExitOnComplete => false;
} }

View File

@ -5,32 +5,28 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users; namespace VoidCat.Services.Users;
public class EmailVerification : IEmailVerification /// <inheritdoc />
public abstract class BaseEmailVerification : IEmailVerification
{ {
private readonly ICache _cache; public const int HoursExpire = 1;
private readonly VoidSettings _settings; private readonly VoidSettings _settings;
private readonly ILogger<EmailVerification> _logger; private readonly ILogger<BaseEmailVerification> _logger;
private readonly RazorPartialToStringRenderer _renderer; private readonly RazorPartialToStringRenderer _renderer;
public EmailVerification(ICache cache, ILogger<EmailVerification> logger, VoidSettings settings, protected BaseEmailVerification(ILogger<BaseEmailVerification> logger, VoidSettings settings,
RazorPartialToStringRenderer renderer) RazorPartialToStringRenderer renderer)
{ {
_cache = cache;
_logger = logger; _logger = logger;
_settings = settings; _settings = settings;
_renderer = renderer; _renderer = renderer;
} }
/// <inheritdoc />
public async ValueTask<EmailVerificationCode> SendNewCode(PrivateVoidUser user) public async ValueTask<EmailVerificationCode> SendNewCode(PrivateVoidUser user)
{ {
const int codeExpire = 1; var token = new EmailVerificationCode(user.Id, Guid.NewGuid(), DateTime.UtcNow.AddHours(HoursExpire));
var code = new EmailVerificationCode() await SaveToken(token);
{ _logger.LogInformation("Saved email verification token for User={Id} Token={Token}", user.Id, token.Code);
UserId = user.Id,
Expires = DateTimeOffset.UtcNow.AddHours(codeExpire)
};
await _cache.Set(MapToken(code.Id), code, TimeSpan.FromHours(codeExpire));
_logger.LogInformation("Saved email verification token for User={Id} Token={Token}", user.Id, code.Id);
// send email // send email
try try
@ -42,7 +38,7 @@ public class EmailVerification : IEmailVerification
sc.EnableSsl = conf?.Server?.Scheme == "tls"; sc.EnableSsl = conf?.Server?.Scheme == "tls";
sc.Credentials = new NetworkCredential(conf?.Username, conf?.Password); sc.Credentials = new NetworkCredential(conf?.Username, conf?.Password);
var msgContent = await _renderer.RenderPartialToStringAsync("~/Pages/EmailCode.cshtml", code); var msgContent = await _renderer.RenderPartialToStringAsync("~/Pages/EmailCode.cshtml", token);
var msg = new MailMessage(); var msg = new MailMessage();
msg.From = new MailAddress(conf?.Username ?? "no-reply@void.cat"); msg.From = new MailAddress(conf?.Username ?? "no-reply@void.cat");
msg.To.Add(user.Email); msg.To.Add(user.Email);
@ -59,22 +55,26 @@ public class EmailVerification : IEmailVerification
_logger.LogError(ex, "Failed to send email verification code {Error}", ex.Message); _logger.LogError(ex, "Failed to send email verification code {Error}", ex.Message);
} }
return code; return token;
} }
/// <inheritdoc />
public async ValueTask<bool> VerifyCode(PrivateVoidUser user, Guid code) public async ValueTask<bool> VerifyCode(PrivateVoidUser user, Guid code)
{ {
var token = await _cache.Get<EmailVerificationCode>(MapToken(code)); var token = await GetToken(user.Id, code);
if (token == default) return false; if (token == default) return false;
var isValid = user.Id == token.UserId && token.Expires > DateTimeOffset.UtcNow; var isValid = user.Id == token.User &&
DateTime.SpecifyKind(token.Expires, DateTimeKind.Utc) > DateTimeOffset.UtcNow;
if (isValid) if (isValid)
{ {
await _cache.Delete(MapToken(code)); await DeleteToken(user.Id, code);
} }
return isValid; return isValid;
} }
private static string MapToken(Guid id) => $"email-code:{id}"; protected abstract ValueTask SaveToken(EmailVerificationCode code);
protected abstract ValueTask<EmailVerificationCode?> GetToken(Guid user, Guid code);
protected abstract ValueTask DeleteToken(Guid user, Guid code);
} }

View File

@ -0,0 +1,36 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
/// <inheritdoc />
public class CacheEmailVerification : BaseEmailVerification
{
private readonly ICache _cache;
public CacheEmailVerification(ICache cache, ILogger<CacheEmailVerification> logger, VoidSettings settings,
RazorPartialToStringRenderer renderer) : base(logger, settings, renderer)
{
_cache = cache;
}
/// <inheritdoc />
protected override ValueTask SaveToken(EmailVerificationCode code)
{
return _cache.Set(MapToken(code.Code), code, code.Expires - DateTime.UtcNow);
}
/// <inheritdoc />
protected override ValueTask<EmailVerificationCode?> GetToken(Guid user, Guid code)
{
return _cache.Get<EmailVerificationCode>(MapToken(code));
}
/// <inheritdoc />
protected override ValueTask DeleteToken(Guid user, Guid code)
{
return _cache.Delete(MapToken(code));
}
private static string MapToken(Guid id) => $"email-code:{id}";
}

View File

@ -3,28 +3,26 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users; namespace VoidCat.Services.Users;
public class UserStore : IUserStore /// <inheritdoc />
public class CacheUserStore : IUserStore
{ {
private const string UserList = "users"; private const string UserList = "users";
private readonly ILogger<UserStore> _logger; private readonly ILogger<CacheUserStore> _logger;
private readonly ICache _cache; private readonly ICache _cache;
public UserStore(ICache cache, ILogger<UserStore> logger) public CacheUserStore(ICache cache, ILogger<CacheUserStore> logger)
{ {
_cache = cache; _cache = cache;
_logger = logger; _logger = logger;
} }
/// <inheritdoc />
public async ValueTask<Guid?> LookupUser(string email) public async ValueTask<Guid?> LookupUser(string email)
{ {
return await _cache.Get<Guid>(MapKey(email)); return await _cache.Get<Guid>(MapKey(email));
} }
public async ValueTask<VoidUser?> Get(Guid id) /// <inheritdoc />
{
return await Get<PublicVoidUser>(id);
}
public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser
{ {
try try
@ -39,6 +37,19 @@ public class UserStore : IUserStore
return default; return default;
} }
/// <inheritdoc />
public ValueTask<VoidUser?> Get(Guid id)
{
return Get<VoidUser>(id);
}
/// <inheritdoc />
public ValueTask<InternalVoidUser?> GetPrivate(Guid id)
{
return Get<InternalVoidUser>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, InternalVoidUser user) public async ValueTask Set(Guid id, InternalVoidUser user)
{ {
if (id != user.Id) throw new InvalidOperationException(); if (id != user.Id) throw new InvalidOperationException();
@ -48,6 +59,7 @@ public class UserStore : IUserStore
await _cache.Set(MapKey(user.Email), user.Id.ToString()); await _cache.Set(MapKey(user.Email), user.Id.ToString());
} }
/// <inheritdoc />
public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request) public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request)
{ {
var users = (await _cache.GetList(UserList)) var users = (await _cache.GetList(UserList))
@ -81,6 +93,7 @@ public class UserStore : IUserStore
}; };
} }
/// <inheritdoc />
public async ValueTask UpdateProfile(PublicVoidUser newUser) public async ValueTask UpdateProfile(PublicVoidUser newUser)
{ {
var oldUser = await Get<InternalVoidUser>(newUser.Id); var oldUser = await Get<InternalVoidUser>(newUser.Id);
@ -97,6 +110,18 @@ public class UserStore : IUserStore
await Set(newUser.Id, oldUser); await Set(newUser.Id, oldUser);
} }
/// <inheritdoc />
public async ValueTask UpdateLastLogin(Guid id, DateTime timestamp)
{
var user = await Get<InternalVoidUser>(id);
if (user != default)
{
user.LastLogin = timestamp;
await Set(user.Id, user);
}
}
/// <inheritdoc />
public async ValueTask Delete(Guid id) public async ValueTask Delete(Guid id)
{ {
var user = await Get<InternalVoidUser>(id); var user = await Get<InternalVoidUser>(id);
@ -104,7 +129,7 @@ public class UserStore : IUserStore
await Delete(user); await Delete(user);
} }
public async ValueTask Delete(PrivateVoidUser user) private async ValueTask Delete(PrivateVoidUser user)
{ {
await _cache.Delete(MapKey(user.Id)); await _cache.Delete(MapKey(user.Id));
await _cache.RemoveFromList(UserList, user.Id.ToString()); await _cache.RemoveFromList(UserList, user.Id.ToString());

View File

@ -0,0 +1,45 @@
using Dapper;
using Npgsql;
using VoidCat.Model;
namespace VoidCat.Services.Users;
/// <inheritdoc />
public class PostgresEmailVerification : BaseEmailVerification
{
private readonly NpgsqlConnection _connection;
public PostgresEmailVerification(ILogger<BaseEmailVerification> logger, VoidSettings settings,
RazorPartialToStringRenderer renderer, NpgsqlConnection connection) : base(logger, settings, renderer)
{
_connection = connection;
}
/// <inheritdoc />
protected override async ValueTask SaveToken(EmailVerificationCode code)
{
await _connection.ExecuteAsync(
@"insert into ""EmailVerification""(""User"", ""Code"", ""Expires"") values(:user, :code, :expires)",
new
{
user = code.User,
code = code.Code,
expires = code.Expires.ToUniversalTime()
});
}
/// <inheritdoc />
protected override async ValueTask<EmailVerificationCode?> GetToken(Guid user, Guid code)
{
return await _connection.QuerySingleOrDefaultAsync<EmailVerificationCode>(
@"select * from ""EmailVerification"" where ""User"" = :user and ""Code"" = :code",
new {user, code});
}
/// <inheritdoc />
protected override async ValueTask DeleteToken(Guid user, Guid code)
{
await _connection.ExecuteAsync(@"delete from ""EmailVerification"" where ""User"" = :user and ""Code"" = :code",
new {user, code});
}
}

View File

@ -5,6 +5,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users; namespace VoidCat.Services.Users;
/// <inheritdoc />
public class PostgresUserStore : IUserStore public class PostgresUserStore : IUserStore
{ {
private readonly NpgsqlConnection _connection; private readonly NpgsqlConnection _connection;
@ -14,40 +15,70 @@ public class PostgresUserStore : IUserStore
_connection = connection; _connection = connection;
} }
public ValueTask<VoidUser?> Get(Guid id) /// <inheritdoc />
public async ValueTask<VoidUser?> Get(Guid id)
{ {
return Get<VoidUser>(id); return await Get<PublicVoidUser>(id);
} }
/// <inheritdoc />
public async ValueTask<InternalVoidUser?> GetPrivate(Guid id)
{
return await Get<InternalVoidUser>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, InternalVoidUser obj) public async ValueTask Set(Guid id, InternalVoidUser obj)
{ {
await _connection.ExecuteAsync( await _connection.ExecuteAsync(
@"insert into @"insert into
""Users""(""Id"", ""Email"", ""Password"", ""LastLogin"", ""DisplayName"", ""Avatar"", ""Flags"") ""Users""(""Id"", ""Email"", ""Password"", ""LastLogin"", ""DisplayName"", ""Avatar"", ""Flags"")
values(:id, :email, :password, :lastLogin, :displayName, :avatar, :flags) values(:id, :email, :password, :lastLogin, :displayName, :avatar, :flags)",
on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" = :displayName, ""Avatar"" = :avatar, ""Flags"" = :flags",
new new
{ {
Id = id, Id = id,
email = obj.Email, email = obj.Email,
password = obj.PasswordHash, password = obj.Password,
displayName = obj.DisplayName, displayName = obj.DisplayName,
lastLogin = obj.LastLogin, lastLogin = obj.LastLogin.ToUniversalTime(),
avatar = obj.Avatar, avatar = obj.Avatar,
flags = (int) obj.Flags flags = (int) obj.Flags
}); });
if (obj.Roles.Any(a => a != Roles.User))
{
foreach (var r in obj.Roles.Where(a => a != Roles.User))
{
await _connection.ExecuteAsync(@"insert into ""UserRoles""(""User"", ""Role"") values(:user, :role)",
new {user = obj.Id, role = r});
}
}
} }
/// <inheritdoc />
public async ValueTask Delete(Guid id) public async ValueTask Delete(Guid id)
{ {
await _connection.ExecuteAsync(@"delete from ""Users"" where ""Id"" = :id", new {id}); await _connection.ExecuteAsync(@"delete from ""Users"" where ""Id"" = :id", new {id});
} }
/// <inheritdoc />
public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser
{ {
return await _connection.QuerySingleOrDefaultAsync<T?>(@"select * from ""Users"" where ""Id"" = :id", new {id}); var user = await _connection.QuerySingleOrDefaultAsync<T?>(@"select * from ""Users"" where ""Id"" = :id",
new {id});
if (user != default)
{
var roles = await _connection.QueryAsync<string>(@"select ""Role"" from ""UserRoles"" where ""User"" = :id",
new {id});
foreach (var r in roles)
{
user.Roles.Add(r);
}
}
return user;
} }
/// <inheritdoc />
public async ValueTask<Guid?> LookupUser(string email) public async ValueTask<Guid?> LookupUser(string email)
{ {
return await _connection.QuerySingleOrDefaultAsync<Guid?>( return await _connection.QuerySingleOrDefaultAsync<Guid?>(
@ -55,6 +86,7 @@ on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" =
new {email}); new {email});
} }
/// <inheritdoc />
public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request) public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request)
{ {
var orderBy = request.SortBy switch var orderBy = request.SortBy switch
@ -94,10 +126,29 @@ on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" =
}; };
} }
/// <inheritdoc />
public async ValueTask UpdateProfile(PublicVoidUser newUser) public async ValueTask UpdateProfile(PublicVoidUser newUser)
{ {
var oldUser = await Get<InternalVoidUser>(newUser.Id);
if (oldUser == null) return;
var emailFlag = oldUser.Flags.HasFlag(VoidUserFlags.EmailVerified) ? VoidUserFlags.EmailVerified : 0;
await _connection.ExecuteAsync( await _connection.ExecuteAsync(
@"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar where ""Id"" = :id", @"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar, ""Flags"" = :flags where ""Id"" = :id",
new {id = newUser.Id, displayName = newUser.DisplayName, avatar = newUser.Avatar}); new
{
id = newUser.Id,
displayName = newUser.DisplayName,
avatar = newUser.Avatar,
flags = newUser.Flags | emailFlag
});
}
/// <inheritdoc />
public async ValueTask UpdateLastLogin(Guid id, DateTime timestamp)
{
await _connection.ExecuteAsync(@"update ""Users"" set ""LastLogin"" = :timestamp where ""Id"" = :id",
new {id, timestamp});
} }
} }

View File

@ -3,6 +3,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users; namespace VoidCat.Services.Users;
/// <inheritdoc />
public class UserManager : IUserManager public class UserManager : IUserManager
{ {
private readonly IUserStore _store; private readonly IUserStore _store;
@ -15,27 +16,33 @@ public class UserManager : IUserManager
_emailVerification = emailVerification; _emailVerification = emailVerification;
} }
/// <inheritdoc />
public async ValueTask<InternalVoidUser> Login(string email, string password) public async ValueTask<InternalVoidUser> Login(string email, string password)
{ {
var userId = await _store.LookupUser(email); var userId = await _store.LookupUser(email);
if (!userId.HasValue) throw new InvalidOperationException("User does not exist"); if (!userId.HasValue) throw new InvalidOperationException("User does not exist");
var user = await _store.Get<InternalVoidUser>(userId.Value); var user = await _store.GetPrivate(userId.Value);
if (!(user?.CheckPassword(password) ?? false)) throw new InvalidOperationException("User does not exist"); if (!(user?.CheckPassword(password) ?? false)) throw new InvalidOperationException("User does not exist");
user.LastLogin = DateTimeOffset.UtcNow; user.LastLogin = DateTimeOffset.UtcNow;
await _store.Set(user.Id, user); await _store.UpdateLastLogin(user.Id, DateTime.UtcNow);
return user; return user;
} }
/// <inheritdoc />
public async ValueTask<InternalVoidUser> Register(string email, string password) public async ValueTask<InternalVoidUser> Register(string email, string password)
{ {
var existingUser = await _store.LookupUser(email); var existingUser = await _store.LookupUser(email);
if (existingUser != Guid.Empty && existingUser != null) throw new InvalidOperationException("User already exists"); if (existingUser != Guid.Empty && existingUser != null)
throw new InvalidOperationException("User already exists");
var newUser = new InternalVoidUser(Guid.NewGuid(), email, password.HashPassword()) var newUser = new InternalVoidUser
{ {
Id = Guid.NewGuid(),
Email = email,
Password = password.HashPassword(),
Created = DateTimeOffset.UtcNow, Created = DateTimeOffset.UtcNow,
LastLogin = DateTimeOffset.UtcNow LastLogin = DateTimeOffset.UtcNow
}; };

View File

@ -8,14 +8,16 @@ public static class UsersStartup
public static void AddUserServices(this IServiceCollection services, VoidSettings settings) public static void AddUserServices(this IServiceCollection services, VoidSettings settings)
{ {
services.AddTransient<IUserManager, UserManager>(); services.AddTransient<IUserManager, UserManager>();
services.AddTransient<IEmailVerification, EmailVerification>();
if (settings.Postgres != default) if (settings.Postgres != default)
{ {
services.AddTransient<IUserStore, PostgresUserStore>(); services.AddTransient<IUserStore, PostgresUserStore>();
services.AddTransient<IEmailVerification, PostgresEmailVerification>();
} }
else else
{ {
services.AddTransient<IUserStore, UserStore>(); services.AddTransient<IUserStore, CacheUserStore>();
services.AddTransient<IEmailVerification, CacheEmailVerification>();
} }
} }
} }