diff --git a/README.md b/README.md index c50c803..34378be 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,15 @@ curl -X POST \ ``` This command will return the direct download URL only. -To get the json output simply remove the `?cli=true` from the url. \ No newline at end of file +To get the json output simply remove the `?cli=true` from the url. + +### Development +To run postgres in local use: +``` +docker run --rm -it -p 5432:5432 -e POSTGRES_DB=void -e POSTGRES_PASSWORD=postgres postgres -d postgres +``` + +To run MinIO in local use: +``` +docker run --rm -it -p 9000:9000 -p 9001:9001 minio/minio -- server /data --console-address ":9001" +``` diff --git a/VoidCat/Controllers/InfoController.cs b/VoidCat/Controllers/InfoController.cs index 5820017..11e31e0 100644 --- a/VoidCat/Controllers/InfoController.cs +++ b/VoidCat/Controllers/InfoController.cs @@ -8,13 +8,13 @@ namespace VoidCat.Controllers; public class InfoController : Controller { private readonly IStatsReporter _statsReporter; - private readonly IFileStore _fileStore; + private readonly IFileMetadataStore _fileMetadata; private readonly VoidSettings _settings; - public InfoController(IStatsReporter statsReporter, IFileStore fileStore, VoidSettings settings) + public InfoController(IStatsReporter statsReporter, IFileMetadataStore fileMetadata, VoidSettings settings) { _statsReporter = statsReporter; - _fileStore = fileStore; + _fileMetadata = fileMetadata; _settings = settings; } @@ -27,18 +27,11 @@ public class InfoController : Controller public async Task GetGlobalStats() { var bw = await _statsReporter.GetBandwidth(); - var bytes = 0UL; - var count = 0; - var files = await _fileStore.ListFiles(new(0, Int32.MaxValue)); - await foreach (var vf in files.Results) - { - bytes += vf.Metadata?.Size ?? 0; - count++; - } - - return new(bw, bytes, count, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey); + var storeStats = await _fileMetadata.Stats(); + + return new(bw, (ulong)storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey); } - public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, int Count, BuildInfo BuildInfo, + public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo, string? CaptchaSiteKey); } \ No newline at end of file diff --git a/VoidCat/Controllers/UploadController.cs b/VoidCat/Controllers/UploadController.cs index be8740f..6edb1dc 100644 --- a/VoidCat/Controllers/UploadController.cs +++ b/VoidCat/Controllers/UploadController.cs @@ -60,7 +60,7 @@ namespace VoidCat.Controllers } } - var meta = new SecretVoidFileMeta() + var meta = new SecretVoidFileMeta { MimeType = mime, Name = filename, diff --git a/VoidCat/Model/VoidFileMeta.cs b/VoidCat/Model/VoidFileMeta.cs index 5fbb471..eef18ba 100644 --- a/VoidCat/Model/VoidFileMeta.cs +++ b/VoidCat/Model/VoidFileMeta.cs @@ -1,6 +1,8 @@ using Newtonsoft.Json; using VoidCat.Services.Abstractions; +// ReSharper disable InconsistentNaming + namespace VoidCat.Model; /// @@ -9,7 +11,7 @@ namespace VoidCat.Model; public interface IVoidFileMeta { const int CurrentVersion = 3; - + int Version { get; init; } } @@ -22,7 +24,7 @@ public record VoidFileMeta : IVoidFileMeta /// Metadata version /// public int Version { get; init; } = IVoidFileMeta.CurrentVersion; - + /// /// Filename /// @@ -52,12 +54,12 @@ public record VoidFileMeta : IVoidFileMeta /// SHA-256 hash of the file /// public string? Digest { get; init; } - + /// /// Url to download the file /// public Uri? Url { get; set; } - + /// /// User who uploaded the file /// diff --git a/VoidCat/Model/VoidSettings.cs b/VoidCat/Model/VoidSettings.cs index ffa47f8..2561982 100644 --- a/VoidCat/Model/VoidSettings.cs +++ b/VoidCat/Model/VoidSettings.cs @@ -29,6 +29,8 @@ namespace VoidCat.Model public IEnumerable? RequestHeadersLog { get; init; } public CaptchaSettings? CaptchaSettings { get; init; } + + public string? Postgres { get; init; } } public sealed class TorSettings diff --git a/VoidCat/Model/VoidUser.cs b/VoidCat/Model/VoidUser.cs index c21d9ab..f824451 100644 --- a/VoidCat/Model/VoidUser.cs +++ b/VoidCat/Model/VoidUser.cs @@ -1,7 +1,12 @@ using Newtonsoft.Json; +// ReSharper disable InconsistentNaming + namespace VoidCat.Model; +/// +/// The base user object for the system +/// public abstract class VoidUser { protected VoidUser(Guid id) @@ -9,20 +14,32 @@ public abstract class VoidUser Id = id; } + /// + /// Unique Id of the user + /// [JsonConverter(typeof(Base58GuidConverter))] public Guid Id { get; } + /// + /// Roles assigned to this user which grant them extra permissions + /// public HashSet Roles { get; init; } = new() {Model.Roles.User}; + /// + /// When the user account was created + /// public DateTimeOffset Created { get; init; } + /// + /// The last time the user logged in + /// public DateTimeOffset LastLogin { get; set; } /// /// Display avatar for user profile /// public string? Avatar { get; set; } - + /// /// Display name for user profile /// @@ -32,7 +49,11 @@ public abstract class VoidUser /// Profile flags /// public VoidUserFlags Flags { get; set; } = VoidUserFlags.PublicProfile; - + + /// + /// Returns the Public object for this user + /// + /// public PublicVoidUser ToPublic() { return new(Id) @@ -45,28 +66,71 @@ public abstract class VoidUser } } +/// +/// Internal user object used by the system +/// public sealed class InternalVoidUser : PrivateVoidUser { + /// public InternalVoidUser(Guid id, string email, string passwordHash) : base(id, email) { PasswordHash = passwordHash; } + /// + public InternalVoidUser(Guid Id, string Email, string Password, DateTime Created, DateTime LastLogin, + string Avatar, + string DisplayName, int Flags) : base(Id, Email) + { + PasswordHash = Password; + } + + /// + /// A password hash for the user in the format + /// public string PasswordHash { get; } } +/// +/// A user object which includes the Email +/// public class PrivateVoidUser : VoidUser { + /// public PrivateVoidUser(Guid id, string email) : base(id) { Email = email; } + /// + /// Full constructor for Dapper + /// + /// + /// + /// + /// + /// + /// + /// + /// + public PrivateVoidUser(Guid Id, String Email, string Password, DateTime Created, DateTime LastLogin, string Avatar, + string DisplayName, int Flags) : base(Id) + { + this.Email = Email; + this.Created = Created; + this.LastLogin = LastLogin; + this.Avatar = Avatar; + this.DisplayName = DisplayName; + this.Flags = (VoidUserFlags) Flags; + } + public string Email { get; } } +/// public sealed class PublicVoidUser : VoidUser { + /// public PublicVoidUser(Guid id) : base(id) { } @@ -78,4 +142,4 @@ public enum VoidUserFlags PublicProfile = 1, PublicUploads = 2, EmailVerified = 4 -} +} \ No newline at end of file diff --git a/VoidCat/Program.cs b/VoidCat/Program.cs index 0edc0a9..def0b98 100644 --- a/VoidCat/Program.cs +++ b/VoidCat/Program.cs @@ -1,10 +1,13 @@ +using System.Data; using System.Reflection; using System.Text; +using FluentMigrator.Runner; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.HttpLogging; using Microsoft.IdentityModel.Tokens; using Microsoft.OpenApi.Models; using Newtonsoft.Json; +using Npgsql; using Prometheus; using StackExchange.Redis; using VoidCat.Model; @@ -56,10 +59,10 @@ services.AddHttpLogging((o) => o.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders | HttpLoggingFields.ResponsePropertiesAndHeaders; o.RequestBodyLogLimit = 4096; o.ResponseBodyLogLimit = 4096; - + o.MediaTypeOptions.Clear(); o.MediaTypeOptions.AddText("application/json"); - + foreach (var h in voidSettings.RequestHeadersLog ?? Enumerable.Empty()) { o.RequestHeaders.Add(h); @@ -140,9 +143,7 @@ services.AddTransient(); services.AddVoidPaywall(); // users -services.AddTransient(); -services.AddTransient(); -services.AddTransient(); +services.AddUserServices(voidSettings); // background services services.AddHostedService(); @@ -153,13 +154,30 @@ services.AddVirusScanner(voidSettings); // captcha services.AddCaptcha(voidSettings); +// postgres +if (!string.IsNullOrEmpty(voidSettings.Postgres)) +{ + services.AddScoped(); + services.AddScoped((_) => new NpgsqlConnection(voidSettings.Postgres)); + services.AddScoped((svc) => svc.GetRequiredService()); + + // fluent migrations + services.AddTransient(); + services.AddFluentMigratorCore() + .ConfigureRunner(r => + r.AddPostgres11_0() + .WithGlobalConnectionString(voidSettings.Postgres) + .ScanIn(typeof(Program).Assembly).For.Migrations()) + .AddLogging(l => l.AddFluentMigratorConsole()); +} + if (useRedis) { services.AddTransient(); services.AddTransient(); services.AddTransient(svc => svc.GetRequiredService()); services.AddTransient(svc => svc.GetRequiredService()); - + // redis specific migrations services.AddTransient(); } @@ -175,10 +193,17 @@ else var app = builder.Build(); // run migrations -var migrations = app.Services.GetServices(); -foreach (var migration in migrations) +using (var migrationScope = app.Services.CreateScope()) { - await migration.Migrate(); + var migrations = migrationScope.ServiceProvider.GetServices(); + foreach (var migration in migrations) + { + await migration.Migrate(args); + if (migration.ExitOnComplete) + { + return; + } + } } #if HostSPA @@ -193,6 +218,11 @@ app.UseSwaggerUI(); app.UseAuthentication(); app.UseAuthorization(); +if (!string.IsNullOrEmpty(voidSettings.Postgres)) +{ + app.UseMiddleware(); +} + app.UseEndpoints(ep => { ep.MapControllers(); diff --git a/VoidCat/Services/Abstractions/IBasicStore.cs b/VoidCat/Services/Abstractions/IBasicStore.cs index 20eca7d..8fa1f20 100644 --- a/VoidCat/Services/Abstractions/IBasicStore.cs +++ b/VoidCat/Services/Abstractions/IBasicStore.cs @@ -4,6 +4,8 @@ public interface IBasicStore { ValueTask Get(Guid id); + ValueTask> Get(Guid[] ids); + ValueTask Set(Guid id, T obj); ValueTask Delete(Guid id); diff --git a/VoidCat/Services/Abstractions/IFileInfoManager.cs b/VoidCat/Services/Abstractions/IFileInfoManager.cs index e2f5be1..748e282 100644 --- a/VoidCat/Services/Abstractions/IFileInfoManager.cs +++ b/VoidCat/Services/Abstractions/IFileInfoManager.cs @@ -9,5 +9,12 @@ namespace VoidCat.Services.Abstractions; public interface IFileInfoManager { ValueTask Get(Guid id); + ValueTask> Get(Guid[] ids); + + /// + /// Deletes all file metadata + /// + /// + /// ValueTask Delete(Guid id); } diff --git a/VoidCat/Services/Abstractions/IFileMetadataStore.cs b/VoidCat/Services/Abstractions/IFileMetadataStore.cs index a35691f..fe9d3f1 100644 --- a/VoidCat/Services/Abstractions/IFileMetadataStore.cs +++ b/VoidCat/Services/Abstractions/IFileMetadataStore.cs @@ -5,5 +5,14 @@ namespace VoidCat.Services.Abstractions; public interface IFileMetadataStore : IPublicPrivateStore { ValueTask Get(Guid id) where TMeta : VoidFileMeta; + ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta; ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta; -} + + /// + /// Returns basic stats about the file store + /// + /// + ValueTask Stats(); + + public sealed record StoreStats(long Files, ulong Size); +} \ No newline at end of file diff --git a/VoidCat/Services/Abstractions/IFileStore.cs b/VoidCat/Services/Abstractions/IFileStore.cs index 3047ef3..2314764 100644 --- a/VoidCat/Services/Abstractions/IFileStore.cs +++ b/VoidCat/Services/Abstractions/IFileStore.cs @@ -10,6 +10,11 @@ public interface IFileStore ValueTask> ListFiles(PagedRequest request); + /// + /// Deletes file data only, metadata must be deleted with + /// + /// + /// ValueTask DeleteFile(Guid id); ValueTask Open(EgressRequest request, CancellationToken cts); diff --git a/VoidCat/Services/Abstractions/IUserStore.cs b/VoidCat/Services/Abstractions/IUserStore.cs index bed6d6a..08d96e9 100644 --- a/VoidCat/Services/Abstractions/IUserStore.cs +++ b/VoidCat/Services/Abstractions/IUserStore.cs @@ -5,7 +5,6 @@ namespace VoidCat.Services.Abstractions; public interface IUserStore : IPublicPrivateStore { ValueTask Get(Guid id) where T : VoidUser; - ValueTask Delete(PrivateVoidUser user); ValueTask LookupUser(string email); ValueTask> ListUsers(PagedRequest request); diff --git a/VoidCat/Services/Background/DeleteUnverifiedAccounts.cs b/VoidCat/Services/Background/DeleteUnverifiedAccounts.cs index 9a55d96..1266c70 100644 --- a/VoidCat/Services/Background/DeleteUnverifiedAccounts.cs +++ b/VoidCat/Services/Background/DeleteUnverifiedAccounts.cs @@ -6,43 +6,54 @@ namespace VoidCat.Services.Background; public class DeleteUnverifiedAccounts : BackgroundService { private readonly ILogger _logger; - private readonly IUserStore _userStore; - private readonly IUserUploadsStore _userUploads; - private readonly IFileInfoManager _fileInfo; - private readonly IFileStore _fileStore; + private readonly IServiceScopeFactory _scopeFactory; - public DeleteUnverifiedAccounts(ILogger logger, IUserStore userStore, - IUserUploadsStore uploadsStore, IFileInfoManager fileInfo, IFileStore fileStore) + public DeleteUnverifiedAccounts(ILogger logger, IServiceScopeFactory scopeFactory) { - _userStore = userStore; _logger = logger; - _userUploads = uploadsStore; - _fileInfo = fileInfo; - _fileStore = fileStore; + _scopeFactory = scopeFactory; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - var accounts = await _userStore.ListUsers(new(0, Int32.MaxValue)); - - await foreach (var account in accounts.Results.WithCancellation(stoppingToken)) + while (!stoppingToken.IsCancellationRequested) { - if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) && - account.Created.AddDays(7) < DateTimeOffset.UtcNow) + try { - _logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58()); - await _userStore.Delete(account); + using var scope = _scopeFactory.CreateScope(); + var userStore = scope.ServiceProvider.GetRequiredService(); + var userUploads = scope.ServiceProvider.GetRequiredService(); + var fileStore = scope.ServiceProvider.GetRequiredService(); + var fileInfoManager = scope.ServiceProvider.GetRequiredService(); - var files = await _userUploads.ListFiles(account.Id, new(0, Int32.MinValue)); - // ReSharper disable once UseCancellationTokenForIAsyncEnumerable - await foreach (var file in files.Results) + var accounts = await userStore.ListUsers(new(0, Int32.MaxValue)); + + await foreach (var account in accounts.Results.WithCancellation(stoppingToken)) { - await _fileStore.DeleteFile(file.Id); - await _fileInfo.Delete(file.Id); + if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) && + account.Created.AddDays(7) < DateTimeOffset.UtcNow) + { + _logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58()); + await userStore.Delete(account.Id); + + var files = await userUploads.ListFiles(account.Id, new(0, Int32.MinValue)); + // ReSharper disable once UseCancellationTokenForIAsyncEnumerable + await foreach (var file in files.Results) + { + await fileStore.DeleteFile(file.Id); + await fileInfoManager.Delete(file.Id); + } + } } } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete unverified accounts"); + } + finally + { + await Task.Delay(TimeSpan.FromHours(1), stoppingToken); + } } - - await Task.Delay(TimeSpan.FromHours(1), stoppingToken); } } \ No newline at end of file diff --git a/VoidCat/Services/BasicCacheStore.cs b/VoidCat/Services/BasicCacheStore.cs index 908cfe9..c0ef2aa 100644 --- a/VoidCat/Services/BasicCacheStore.cs +++ b/VoidCat/Services/BasicCacheStore.cs @@ -16,6 +16,21 @@ public abstract class BasicCacheStore : IBasicStore return _cache.Get(MapKey(id)); } + public virtual async ValueTask> Get(Guid[] ids) + { + var ret = new List(); + foreach (var id in ids) + { + var r = await _cache.Get(MapKey(id)); + if (r != null) + { + ret.Add(r); + } + } + + return ret; + } + public virtual ValueTask Set(Guid id, TStore obj) { return _cache.Set(MapKey(id), obj); diff --git a/VoidCat/Services/Files/FileInfoManager.cs b/VoidCat/Services/Files/FileInfoManager.cs index 67858a3..b448ca6 100644 --- a/VoidCat/Services/Files/FileInfoManager.cs +++ b/VoidCat/Services/Files/FileInfoManager.cs @@ -30,7 +30,7 @@ public class FileInfoManager : IFileInfoManager await Task.WhenAll(meta.AsTask(), paywall.AsTask(), bandwidth.AsTask(), virusScan.AsTask()); if (meta.Result == default) return default; - + var uploader = meta.Result?.Uploader; var user = uploader.HasValue ? await _userStore.Get(uploader.Value) : null; @@ -45,6 +45,21 @@ public class FileInfoManager : IFileInfoManager }; } + public async ValueTask> Get(Guid[] ids) + { + var ret = new List(); + foreach (var id in ids) + { + var v = await Get(id); + if (v != default) + { + ret.Add(v); + } + } + + return ret; + } + public async ValueTask Delete(Guid id) { await _metadataStore.Delete(id); @@ -52,4 +67,4 @@ public class FileInfoManager : IFileInfoManager await _statsReporter.Delete(id); await _virusScanStore.Delete(id); } -} +} \ No newline at end of file diff --git a/VoidCat/Services/Files/FileStorageStartup.cs b/VoidCat/Services/Files/FileStorageStartup.cs index bd6f9ba..85534fb 100644 --- a/VoidCat/Services/Files/FileStorageStartup.cs +++ b/VoidCat/Services/Files/FileStorageStartup.cs @@ -9,10 +9,11 @@ public static class FileStorageStartup public static void AddStorage(this IServiceCollection services, VoidSettings settings) { services.AddTransient(); - services.AddTransient(); if (settings.CloudStorage != default) { + services.AddTransient(); + // cloud storage if (settings.CloudStorage.S3 != default) { @@ -20,8 +21,15 @@ public static class FileStorageStartup services.AddSingleton(); } } + else if (!string.IsNullOrEmpty(settings.Postgres)) + { + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + } else { + services.AddTransient(); services.AddTransient(); services.AddTransient(); } diff --git a/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs b/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs index c96e09d..bc0b3d5 100644 --- a/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs +++ b/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs @@ -27,18 +27,59 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore return GetMeta(id); } + public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + { + var ret = new List(); + foreach (var id in ids) + { + var r = await GetMeta(id); + if (r != null) + { + ret.Add(r); + } + } + + return ret; + } + public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta { - var oldMeta = await GetMeta(id); + var oldMeta = await Get(id); if (oldMeta == default) return; - + oldMeta.Description = meta.Description ?? oldMeta.Description; oldMeta.Name = meta.Name ?? oldMeta.Name; oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; - + await Set(id, oldMeta); } + public async ValueTask Stats() + { + var count = 0; + var size = 0UL; + foreach (var metaFile in Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json")) + { + try + { + var json = await File.ReadAllTextAsync(metaFile); + var meta = JsonConvert.DeserializeObject(json); + + if (meta != null) + { + count++; + size += meta.Size; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load metadata file: {File}", metaFile); + } + } + + return new(count, size); + } + public ValueTask Get(Guid id) { return GetMeta(id); @@ -74,4 +115,4 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore private string MapMeta(Guid id) => Path.ChangeExtension(Path.Join(_settings.DataDirectory, MetadataDir, id.ToString()), ".json"); -} +} \ No newline at end of file diff --git a/VoidCat/Services/Files/LocalDiskFileStorage.cs b/VoidCat/Services/Files/LocalDiskFileStorage.cs index 6329223..c4329da 100644 --- a/VoidCat/Services/Files/LocalDiskFileStorage.cs +++ b/VoidCat/Services/Files/LocalDiskFileStorage.cs @@ -85,7 +85,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore }); } - public async ValueTask DeleteFile(Guid id) + public ValueTask DeleteFile(Guid id) { var fp = MapPath(id); if (File.Exists(fp)) @@ -93,8 +93,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore _logger.LogInformation("Deleting file: {Path}", fp); File.Delete(fp); } - - await _metadataStore.Delete(id); + return ValueTask.CompletedTask; } public ValueTask Open(EgressRequest request, CancellationToken cts) diff --git a/VoidCat/Services/Files/PostgreFileMetadataStore.cs b/VoidCat/Services/Files/PostgreFileMetadataStore.cs new file mode 100644 index 0000000..213c05a --- /dev/null +++ b/VoidCat/Services/Files/PostgreFileMetadataStore.cs @@ -0,0 +1,76 @@ +using Dapper; +using Npgsql; +using VoidCat.Model; +using VoidCat.Services.Abstractions; + +namespace VoidCat.Services.Files; + +public class PostgreFileMetadataStore : IFileMetadataStore +{ + private readonly NpgsqlConnection _connection; + + public PostgreFileMetadataStore(NpgsqlConnection connection) + { + _connection = connection; + } + + public ValueTask Get(Guid id) + { + return Get(id); + } + + public async ValueTask Set(Guid id, SecretVoidFileMeta obj) + { + await _connection.ExecuteAsync( + @"insert into +""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"") +values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret) +on conflict (""Id"") do update set ""Name"" = :name, ""Description"" = :description, ""MimeType"" = :mimeType", new + { + id, + name = obj.Name, + size = (long) obj.Size, + uploaded = obj.Uploaded.ToUniversalTime(), + description = obj.Description, + mimeType = obj.MimeType, + digest = obj.Digest, + editSecret = obj.EditSecret + }); + } + + public async ValueTask Delete(Guid id) + { + await _connection.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id}); + } + + public async ValueTask Get(Guid id) where TMeta : VoidFileMeta + { + return await _connection.QuerySingleOrDefaultAsync(@"select * from ""Files"" where ""Id"" = :id", + new {id}); + } + + public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + { + var ret = await _connection.QueryAsync("select * from \"Files\" where \"Id\" in :ids", new {ids}); + return ret.ToList(); + } + + public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta + { + var oldMeta = await Get(id); + if (oldMeta == default) return; + + oldMeta.Description = meta.Description ?? oldMeta.Description; + oldMeta.Name = meta.Name ?? oldMeta.Name; + oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; + + await Set(id, oldMeta); + } + + public async ValueTask Stats() + { + var v = await _connection.QuerySingleAsync<(long Files, long Size)>( + @"select count(1) ""Files"", cast(sum(""Size"") as bigint) ""Size"" from ""Files"""); + return new(v.Files, (ulong) v.Size); + } +} \ No newline at end of file diff --git a/VoidCat/Services/Files/S3FileMetadataStore.cs b/VoidCat/Services/Files/S3FileMetadataStore.cs index b2adda1..b6b8af9 100644 --- a/VoidCat/Services/Files/S3FileMetadataStore.cs +++ b/VoidCat/Services/Files/S3FileMetadataStore.cs @@ -25,18 +25,65 @@ public class S3FileMetadataStore : IFileMetadataStore return GetMeta(id); } + public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + { + var ret = new List(); + foreach (var id in ids) + { + var r = await GetMeta(id); + if (r != null) + { + ret.Add(r); + } + } + + return ret; + } + public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta { var oldMeta = await GetMeta(id); if (oldMeta == default) return; - + oldMeta.Description = meta.Description ?? oldMeta.Description; oldMeta.Name = meta.Name ?? oldMeta.Name; oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; - + await Set(id, oldMeta); } + public async ValueTask Stats() + { + var count = 0; + var size = 0UL; + try + { + var obj = await _client.ListObjectsV2Async(new() + { + BucketName = _config.BucketName, + }); + + foreach (var file in obj.S3Objects) + { + if (file.Key.EndsWith("-metadata") && Guid.TryParse(file.Key.Split('-')[0], out var id)) + { + var meta = await GetMeta(id); + if (meta != default) + { + count++; + size += meta.Size; + } + } + } + } + catch (AmazonS3Exception aex) + { + _logger.LogError(aex, "Failed to list files: {Error}", aex.Message); + } + + return new(count, size); + } + public ValueTask Get(Guid id) { return GetMeta(id); @@ -86,6 +133,6 @@ public class S3FileMetadataStore : IFileMetadataStore return default; } - + private static string ToKey(Guid id) => $"{id}-metadata"; -} +} \ No newline at end of file diff --git a/VoidCat/Services/Migrations/Database/00-Init.cs b/VoidCat/Services/Migrations/Database/00-Init.cs new file mode 100644 index 0000000..85e8433 --- /dev/null +++ b/VoidCat/Services/Migrations/Database/00-Init.cs @@ -0,0 +1,57 @@ +using FluentMigrator; +using VoidCat.Model; + +namespace VoidCat.Services.Migrations.Database; + +[Migration(20220604_2232)] +public class Init : Migration { + public override void Up() + { + Create.Table("Users") + .WithColumn("Id").AsGuid().PrimaryKey() + .WithColumn("Email").AsString().NotNullable().Indexed() + .WithColumn("Password").AsString() + .WithColumn("Created").AsDateTime().WithDefault(SystemMethods.CurrentDateTime) + .WithColumn("LastLogin").AsDateTime().Nullable() + .WithColumn("Avatar").AsString().Nullable() + .WithColumn("DisplayName").AsString().WithDefaultValue("void user") + .WithColumn("Flags").AsInt32().WithDefaultValue((int)VoidUserFlags.PublicProfile); + + Create.Table("Files") + .WithColumn("Id").AsGuid().PrimaryKey() + .WithColumn("Name").AsString() + .WithColumn("Size").AsInt64() + .WithColumn("Uploaded").AsDateTime().Indexed().WithDefault(SystemMethods.CurrentDateTime) + .WithColumn("Description").AsString().Nullable() + .WithColumn("MimeType").AsString().WithDefaultValue("application/octet-stream") + .WithColumn("Digest").AsString() + .WithColumn("EditSecret").AsGuid(); + + Create.Table("UserFiles") + .WithColumn("File").AsGuid().ForeignKey("Files", "Id") + .WithColumn("User").AsGuid().ForeignKey("Users", "Id").Indexed(); + + Create.UniqueConstraint() + .OnTable("UserFiles") + .Columns("File", "User"); + + Create.Table("Paywall") + .WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique() + .WithColumn("Type").AsInt16() + .WithColumn("Currency").AsInt16() + .WithColumn("Amount").AsDecimal(); + + Create.Table("PaywallStrike") + .WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique() + .WithColumn("Handle").AsString(); + } + + public override void Down() + { + Delete.Table("Users"); + Delete.Table("Files"); + Delete.Table("UsersFiles"); + Delete.Table("Paywall"); + Delete.Table("PaywallStrike"); + } +} \ No newline at end of file diff --git a/VoidCat/Services/Migrations/FluentMigrationRunner.cs b/VoidCat/Services/Migrations/FluentMigrationRunner.cs new file mode 100644 index 0000000..b07fb9b --- /dev/null +++ b/VoidCat/Services/Migrations/FluentMigrationRunner.cs @@ -0,0 +1,21 @@ +using FluentMigrator.Runner; + +namespace VoidCat.Services.Migrations; + +public class FluentMigrationRunner : IMigration +{ + private readonly IMigrationRunner _runner; + + public FluentMigrationRunner(IMigrationRunner runner) + { + _runner = runner; + } + + public ValueTask Migrate(string[] args) + { + _runner.MigrateUp(); + return ValueTask.CompletedTask; + } + + public bool ExitOnComplete => false; +} \ No newline at end of file diff --git a/VoidCat/Services/Migrations/IMigration.cs b/VoidCat/Services/Migrations/IMigration.cs index a1cf6a7..bfacb65 100644 --- a/VoidCat/Services/Migrations/IMigration.cs +++ b/VoidCat/Services/Migrations/IMigration.cs @@ -2,5 +2,6 @@ public interface IMigration { - ValueTask Migrate(); + ValueTask Migrate(string[] args); + bool ExitOnComplete { get; } } \ No newline at end of file diff --git a/VoidCat/Services/Migrations/MetadataMigrator.cs b/VoidCat/Services/Migrations/MetadataMigrator.cs index e9b9ae7..5e88924 100644 --- a/VoidCat/Services/Migrations/MetadataMigrator.cs +++ b/VoidCat/Services/Migrations/MetadataMigrator.cs @@ -14,7 +14,7 @@ public abstract class MetadataMigrator : IMigration _logger = logger; } - public async ValueTask Migrate() + public async ValueTask Migrate(string[] args) { var newMeta = Path.Combine(_settings.DataDirectory, OldPath); if (!Directory.Exists(newMeta)) @@ -64,4 +64,6 @@ public abstract class MetadataMigrator : IMigration private string MapNewMeta(Guid id) => Path.ChangeExtension(Path.Join(_settings.DataDirectory, NewPath, id.ToString()), ".json"); + + public bool ExitOnComplete => false; } \ No newline at end of file diff --git a/VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs b/VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs index 4eb5541..bc43957 100644 --- a/VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs +++ b/VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs @@ -13,7 +13,7 @@ public class UserLookupKeyHashMigration : IMigration _database = database; } - public async ValueTask Migrate() + public async ValueTask Migrate(string[] args) { var users = await _database.SetMembersAsync("users"); foreach (var userId in users) @@ -41,4 +41,6 @@ public class UserLookupKeyHashMigration : IMigration public string Email { get; init; } } + + public bool ExitOnComplete => false; } \ No newline at end of file diff --git a/VoidCat/Services/OpenDatabase.cs b/VoidCat/Services/OpenDatabase.cs new file mode 100644 index 0000000..0ee2de4 --- /dev/null +++ b/VoidCat/Services/OpenDatabase.cs @@ -0,0 +1,24 @@ +using Npgsql; + +public class OpenDatabase : IMiddleware +{ + private readonly NpgsqlConnection _connection; + + public OpenDatabase(NpgsqlConnection connection) + { + _connection = connection; + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + await _connection.OpenAsync(); + try + { + await next(context); + } + finally + { + await _connection.CloseAsync(); + } + } +} \ No newline at end of file diff --git a/VoidCat/Services/Paywall/PaywallFactory.cs b/VoidCat/Services/Paywall/PaywallFactory.cs index be9d617..fb4044b 100644 --- a/VoidCat/Services/Paywall/PaywallFactory.cs +++ b/VoidCat/Services/Paywall/PaywallFactory.cs @@ -25,16 +25,14 @@ public class PaywallFactory : IPaywallFactory public static class Paywall { - public static IServiceCollection AddVoidPaywall(this IServiceCollection services) + public static void AddVoidPaywall(this IServiceCollection services) { services.AddTransient(); services.AddTransient(); - + // strike services.AddTransient(); services.AddTransient(); services.AddTransient((svc) => svc.GetRequiredService()); - - return services; } } \ No newline at end of file diff --git a/VoidCat/Services/Users/EmailVerification.cs b/VoidCat/Services/Users/EmailVerification.cs index b909c8c..50c7bdc 100644 --- a/VoidCat/Services/Users/EmailVerification.cs +++ b/VoidCat/Services/Users/EmailVerification.cs @@ -12,7 +12,8 @@ public class EmailVerification : IEmailVerification private readonly ILogger _logger; private readonly RazorPartialToStringRenderer _renderer; - public EmailVerification(ICache cache, ILogger logger, VoidSettings settings, RazorPartialToStringRenderer renderer) + public EmailVerification(ICache cache, ILogger logger, VoidSettings settings, + RazorPartialToStringRenderer renderer) { _cache = cache; _logger = logger; @@ -30,7 +31,7 @@ public class EmailVerification : IEmailVerification }; await _cache.Set(MapToken(code.Id), code, TimeSpan.FromHours(codeExpire)); _logger.LogInformation("Saved email verification token for User={Id} Token={Token}", user.Id, code.Id); - + // send email try { @@ -40,7 +41,7 @@ public class EmailVerification : IEmailVerification sc.Port = conf?.Server?.Port ?? 25; sc.EnableSsl = conf?.Server?.Scheme == "tls"; sc.Credentials = new NetworkCredential(conf?.Username, conf?.Password); - + var msgContent = await _renderer.RenderPartialToStringAsync("~/Pages/EmailCode.cshtml", code); var msg = new MailMessage(); msg.From = new MailAddress(conf?.Username ?? "no-reply@void.cat"); @@ -65,7 +66,7 @@ public class EmailVerification : IEmailVerification { var token = await _cache.Get(MapToken(code)); if (token == default) return false; - + var isValid = user.Id == token.UserId && token.Expires > DateTimeOffset.UtcNow; if (isValid) { diff --git a/VoidCat/Services/Users/PostgresUserStore.cs b/VoidCat/Services/Users/PostgresUserStore.cs new file mode 100644 index 0000000..0b7e175 --- /dev/null +++ b/VoidCat/Services/Users/PostgresUserStore.cs @@ -0,0 +1,103 @@ +using Dapper; +using Npgsql; +using VoidCat.Model; +using VoidCat.Services.Abstractions; + +namespace VoidCat.Services.Users; + +public class PostgresUserStore : IUserStore +{ + private readonly NpgsqlConnection _connection; + + public PostgresUserStore(NpgsqlConnection connection) + { + _connection = connection; + } + + public ValueTask Get(Guid id) + { + return Get(id); + } + + public async ValueTask Set(Guid id, InternalVoidUser obj) + { + await _connection.ExecuteAsync( + @"insert into +""Users""(""Id"", ""Email"", ""Password"", ""LastLogin"", ""DisplayName"", ""Avatar"", ""Flags"") +values(:id, :email, :password, :lastLogin, :displayName, :avatar, :flags) +on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" = :displayName, ""Avatar"" = :avatar, ""Flags"" = :flags", + new + { + Id = id, + email = obj.Email, + password = obj.PasswordHash, + displayName = obj.DisplayName, + lastLogin = obj.LastLogin, + avatar = obj.Avatar, + flags = (int) obj.Flags + }); + } + + public async ValueTask Delete(Guid id) + { + await _connection.ExecuteAsync(@"delete from ""Users"" where ""Id"" = :id", new {id}); + } + + public async ValueTask Get(Guid id) where T : VoidUser + { + return await _connection.QuerySingleOrDefaultAsync(@"select * from ""Users"" where ""Id"" = :id", new {id}); + } + + public async ValueTask LookupUser(string email) + { + return await _connection.QuerySingleOrDefaultAsync( + @"select ""Id"" from ""Users"" where ""Email"" = :email", + new {email}); + } + + public async ValueTask> ListUsers(PagedRequest request) + { + var orderBy = request.SortBy switch + { + PagedSortBy.Date => "Created", + PagedSortBy.Name => "DisplayName", + _ => "Id" + }; + var sortBy = request.SortOrder switch + { + PageSortOrder.Dsc => "desc", + _ => "asc" + }; + var totalUsers = await _connection.ExecuteScalarAsync(@"select count(*) from ""Users"""); + var users = await _connection.QueryAsync( + $@"select * from ""Users"" order by ""{orderBy}"" {sortBy} offset :offset limit :limit", + new + { + offset = request.PageSize * request.Page, + limit = request.PageSize + }); + + async IAsyncEnumerable Enumerate() + { + foreach (var u in users ?? Enumerable.Empty()) + { + yield return u; + } + } + + return new() + { + Page = request.Page, + PageSize = request.PageSize, + TotalResults = totalUsers, + Results = Enumerate() + }; + } + + public async ValueTask UpdateProfile(PublicVoidUser newUser) + { + await _connection.ExecuteAsync( + @"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar where ""Id"" = :id", + new {id = newUser.Id, displayName = newUser.DisplayName, avatar = newUser.Avatar}); + } +} \ No newline at end of file diff --git a/VoidCat/Services/Users/PostgresUserUploadStore.cs b/VoidCat/Services/Users/PostgresUserUploadStore.cs new file mode 100644 index 0000000..9addf48 --- /dev/null +++ b/VoidCat/Services/Users/PostgresUserUploadStore.cs @@ -0,0 +1,73 @@ +using Dapper; +using Npgsql; +using VoidCat.Model; +using VoidCat.Services.Abstractions; + +namespace VoidCat.Services.Users; + +public class PostgresUserUploadStore : IUserUploadsStore +{ + private readonly NpgsqlConnection _connection; + private readonly IFileInfoManager _fileInfoManager; + + public PostgresUserUploadStore(NpgsqlConnection connection, IFileInfoManager fileInfoManager) + { + _connection = connection; + _fileInfoManager = fileInfoManager; + } + + public async ValueTask> ListFiles(Guid user, PagedRequest request) + { + var query = @"select {0} +from ""UserFiles"" uf, ""Files"" f +where uf.""User"" = :user +and uf.""File"" = f.""Id"""; + var queryOrder = @"order by f.""{1}"" {2} limit :limit offset :offset"; + + var orderBy = request.SortBy switch + { + PagedSortBy.Name => "Name", + PagedSortBy.Date => "Uploaded", + PagedSortBy.Size => "Size", + _ => "Id" + }; + var sortOrder = request.SortOrder switch + { + PageSortOrder.Dsc => "desc", + _ => "asc" + }; + var count = await _connection.ExecuteScalarAsync(string.Format(query, "count(*)"), new {user}); + var files = await _connection.QueryAsync( + string.Format(query + queryOrder, "uf.\"File\"", orderBy, sortOrder), + new {user, offset = request.Page * request.PageSize, limit = request.PageSize}); + + async IAsyncEnumerable EnumerateFiles() + { + foreach (var file in files ?? Enumerable.Empty()) + { + var v = await _fileInfoManager.Get(file); + if (v != default) + { + yield return v; + } + } + } + + return new() + { + Page = request.Page, + PageSize = request.PageSize, + TotalResults = count, + Results = EnumerateFiles() + }; + } + + public async ValueTask AddFile(Guid user, PrivateVoidFile voidFile) + { + await _connection.ExecuteAsync(@"insert into ""UserFiles""(""File"", ""User"") values(:file, :user)", new + { + file = voidFile.Id, + user + }); + } +} \ No newline at end of file diff --git a/VoidCat/Services/Users/UserManager.cs b/VoidCat/Services/Users/UserManager.cs index 6172fc6..022f168 100644 --- a/VoidCat/Services/Users/UserManager.cs +++ b/VoidCat/Services/Users/UserManager.cs @@ -32,7 +32,7 @@ public class UserManager : IUserManager public async ValueTask Register(string email, string password) { var existingUser = await _store.LookupUser(email); - if (existingUser != Guid.Empty) throw new InvalidOperationException("User already exists"); + if (existingUser != Guid.Empty && existingUser != null) throw new InvalidOperationException("User already exists"); var newUser = new InternalVoidUser(Guid.NewGuid(), email, password.HashPassword()) { @@ -50,9 +50,9 @@ public class UserManager : IUserManager newUser.Roles.Add(Roles.Admin); } } - + await _store.Set(newUser.Id, newUser); await _emailVerification.SendNewCode(newUser); return newUser; } -} +} \ No newline at end of file diff --git a/VoidCat/Services/Users/UserUploadStore.cs b/VoidCat/Services/Users/UserUploadStore.cs index 09acf27..e13d0ed 100644 --- a/VoidCat/Services/Users/UserUploadStore.cs +++ b/VoidCat/Services/Users/UserUploadStore.cs @@ -51,4 +51,4 @@ public class UserUploadStore : IUserUploadsStore } private static string MapKey(Guid id) => $"user:{id}:uploads"; -} +} \ No newline at end of file diff --git a/VoidCat/Services/Users/UsersStartup.cs b/VoidCat/Services/Users/UsersStartup.cs new file mode 100644 index 0000000..b0ef569 --- /dev/null +++ b/VoidCat/Services/Users/UsersStartup.cs @@ -0,0 +1,21 @@ +using VoidCat.Model; +using VoidCat.Services.Abstractions; + +namespace VoidCat.Services.Users; + +public static class UsersStartup +{ + public static void AddUserServices(this IServiceCollection services, VoidSettings settings) + { + services.AddTransient(); + services.AddTransient(); + if (settings.Postgres != default) + { + services.AddTransient(); + } + else + { + services.AddTransient(); + } + } +} \ No newline at end of file diff --git a/VoidCat/VoidCat.csproj b/VoidCat/VoidCat.csproj index 8de45bd..9b6c40c 100644 --- a/VoidCat/VoidCat.csproj +++ b/VoidCat/VoidCat.csproj @@ -15,6 +15,10 @@ + + + + @@ -24,6 +28,7 @@ + diff --git a/VoidCat/spa/src/Admin/UserList.js b/VoidCat/spa/src/Admin/UserList.js index da82f42..80523b8 100644 --- a/VoidCat/spa/src/Admin/UserList.js +++ b/VoidCat/spa/src/Admin/UserList.js @@ -18,7 +18,7 @@ export function UserList() { let pageReq = { page: page, pageSize, - sortBy: PagedSortBy.Id, + sortBy: PagedSortBy.Date, sortOrder: PageSortOrder.Asc }; let req = await AdminApi.userList(pageReq); @@ -33,7 +33,7 @@ export function UserList() { function renderUser(u) { return ( - + {u.id.substring(0, 4)}.. {moment(u.created).fromNow()} {moment(u.lastLogin).fromNow()}