godzhigella/void.cat
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

Merge pull request #50 from v0l/minor4v1

Browse Source 
Minor4v1
This commit is contained in:
Kieran 2022-07-26 13:35:39 +01:00 committed by GitHub
commit ab917b0413
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
49 changed files with 971 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
VoidCat/Controllers/Admin/AdminController.cs
View File

@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Controllers.Admin;
@ -9,13 +10,13 @@ namespace VoidCat.Controllers.Admin;
[Authorize(Policy = Policies.RequireAdmin)]
public class AdminController : Controller
{
private readonly IFileStore _fileStore;
private readonly FileStoreFactory _fileStore;
private readonly IFileMetadataStore _fileMetadata;
private readonly IFileInfoManager _fileInfo;
private readonly IUserStore _userStore;
private readonly IUserUploadsStore _userUploads;
public AdminController(IFileStore fileStore, IUserStore userStore, IFileInfoManager fileInfo,
public AdminController(FileStoreFactory fileStore, IUserStore userStore, IFileInfoManager fileInfo,
IFileMetadataStore fileMetadata, IUserUploadsStore userUploads)
{
_fileStore = fileStore;
@ -64,7 +65,7 @@ public class AdminController : Controller
/// <param name="request">Page request</param>
/// <returns></returns>
[HttpPost]
[Route("user")]
[Route("users")]
public async Task<RenderedResults<AdminListedUser>> ListUsers([FromBody] PagedRequest request)
{
var result = await _userStore.ListUsers(request);
@ -74,6 +75,7 @@ public class AdminController : Controller
var uploads = await _userUploads.ListFiles(a.Id, new(0, int.MaxValue));
return new AdminListedUser(a, uploads.TotalResults);
}).ToListAsync();
return new()
{
PageSize = request.PageSize,
@ -83,5 +85,21 @@ public class AdminController : Controller
};
}
/// <summary>
/// Admin update user account
/// </summary>
/// <param name="user"></param>
/// <returns></returns>
[HttpPost]
[Route("update-user")]
public async Task<IActionResult> UpdateUser([FromBody] PrivateVoidUser user)
{
var oldUser = await _userStore.Get(user.Id);
if (oldUser == default) return BadRequest();
await _userStore.AdminUpdateUser(user);
return Ok();
}
public record AdminListedUser(PrivateVoidUser User, int Uploads);
}

69
VoidCat/Controllers/AuthController.cs
View File

@ -15,12 +15,17 @@ public class AuthController : Controller
private readonly IUserManager _manager;
private readonly VoidSettings _settings;
private readonly ICaptchaVerifier _captchaVerifier;
private readonly IApiKeyStore _apiKeyStore;
private readonly IUserStore _userStore;
public AuthController(IUserManager userStore, VoidSettings settings, ICaptchaVerifier captchaVerifier)
public AuthController(IUserManager userStore, VoidSettings settings, ICaptchaVerifier captchaVerifier, IApiKeyStore apiKeyStore,
IUserStore userStore1)
{
_manager = userStore;
_settings = settings;
_captchaVerifier = captchaVerifier;
_apiKeyStore = apiKeyStore;
_userStore = userStore1;
}
/// <summary>
@ -47,7 +52,7 @@ public class AuthController : Controller
}
var user = await _manager.Login(req.Username, req.Password);
var token = CreateToken(user);
var token = CreateToken(user, DateTime.UtcNow.AddHours(12));
var tokenWriter = new JwtSecurityTokenHandler();
return new(tokenWriter.WriteToken(token), Profile: user.ToPublic());
}
@ -81,7 +86,7 @@ public class AuthController : Controller
}
var newUser = await _manager.Register(req.Username, req.Password);
var token = CreateToken(newUser);
var token = CreateToken(newUser, DateTime.UtcNow.AddHours(12));
var tokenWriter = new JwtSecurityTokenHandler();
return new(tokenWriter.WriteToken(token), Profile: newUser.ToPublic());
}
@ -91,7 +96,56 @@ public class AuthController : Controller
}
}
private JwtSecurityToken CreateToken(VoidUser user)
/// <summary>
/// List api keys for the user
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
[HttpGet]
[Route("api-key")]
public async Task<IActionResult> ListApiKeys()
{
var uid = HttpContext.GetUserId();
if (uid == default) return Unauthorized();
return Json(await _apiKeyStore.ListKeys(uid.Value));
}
/// <summary>
/// Create a new API key for the logged in user
/// </summary>
/// <param name="id"></param>
/// <param name="request"></param>
/// <returns></returns>
[HttpPost]
[Route("api-key")]
public async Task<IActionResult> CreateApiKey([FromBody] CreateApiKeyRequest request)
{
var uid = HttpContext.GetUserId();
if (uid == default) return Unauthorized();
var user = await _userStore.Get(uid.Value);
if (user == default) return Unauthorized();
var expiry = DateTime.SpecifyKind(request.Expiry, DateTimeKind.Utc);
if (expiry > DateTime.UtcNow.AddYears(1))
{
return BadRequest();
}
var key = new ApiKey()
{
Id = Guid.NewGuid(),
UserId = user.Id,
Token = new JwtSecurityTokenHandler().WriteToken(CreateToken(user, expiry)),
Expiry = expiry
};
await _apiKeyStore.Add(key.Id, key);
return Json(key);
}
private JwtSecurityToken CreateToken(VoidUser user, DateTime expiry)
{
var securityKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(_settings.JwtSettings.Key));
var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.HmacSha256);
@ -99,16 +153,16 @@ public class AuthController : Controller
var claims = new List<Claim>()
{
new(ClaimTypes.NameIdentifier, user.Id.ToString()),
new(JwtRegisteredClaimNames.Exp, DateTimeOffset.UtcNow.AddHours(6).ToUnixTimeSeconds().ToString()),
new(JwtRegisteredClaimNames.Exp, new DateTimeOffset(expiry).ToUnixTimeSeconds().ToString()),
new(JwtRegisteredClaimNames.Iat, DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString())
};
claims.AddRange(user.Roles.Select(a => new Claim(ClaimTypes.Role, a)));
return new JwtSecurityToken(_settings.JwtSettings.Issuer, claims: claims,
signingCredentials: credentials);
}
public sealed class LoginRequest
{
public LoginRequest(string username, string password)
@ -129,4 +183,7 @@ public class AuthController : Controller
}
public sealed record LoginResponse(string? Jwt, string? Error = null, VoidUser? Profile = null);
public sealed record CreateApiKeyRequest(DateTime Expiry);
}

22
VoidCat/Controllers/DownloadController.cs
View File

@ -3,18 +3,19 @@ using Microsoft.AspNetCore.Mvc;
using VoidCat.Model;
using VoidCat.Model.Paywall;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Controllers;
[Route("d")]
public class DownloadController : Controller
{
private readonly IFileStore _storage;
private readonly FileStoreFactory _storage;
private readonly IFileInfoManager _fileInfo;
private readonly IPaywallOrderStore _paywallOrders;
private readonly ILogger<DownloadController> _logger;
public DownloadController(IFileStore storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
public DownloadController(FileStoreFactory storage, ILogger<DownloadController> logger, IFileInfoManager fileInfo,
IPaywallOrderStore paywall)
{
_storage = storage;
@ -44,7 +45,7 @@ public class DownloadController : Controller
var voidFile = await SetupDownload(gid);
if (voidFile == default) return;
var egressReq = new EgressRequest(gid, GetRanges(Request, (long) voidFile!.Metadata!.Size));
var egressReq = new EgressRequest(gid, GetRanges(Request, (long)voidFile!.Metadata!.Size));
if (egressReq.Ranges.Count() > 1)
{
_logger.LogWarning("Multi-range request not supported!");
@ -56,10 +57,10 @@ public class DownloadController : Controller
}
else if (egressReq.Ranges.Count() == 1)
{
Response.StatusCode = (int) HttpStatusCode.PartialContent;
Response.StatusCode = (int)HttpStatusCode.PartialContent;
if (egressReq.Ranges.Sum(a => a.Size) == 0)
{
Response.StatusCode = (int) HttpStatusCode.RequestedRangeNotSatisfiable;
Response.StatusCode = (int)HttpStatusCode.RequestedRangeNotSatisfiable;
return;
}
}
@ -74,6 +75,15 @@ public class DownloadController : Controller
Response.ContentLength = range.Size;
}
var preResult = await _storage.StartEgress(egressReq);
if (preResult.Redirect != null)
{
Response.StatusCode = (int)HttpStatusCode.Redirect;
Response.Headers.Location = preResult.Redirect.ToString();
Response.ContentLength = 0;
return;
}
var cts = HttpContext.RequestAborted;
await Response.StartAsync(cts);
await _storage.Egress(egressReq, Response.Body, cts);
@ -95,7 +105,7 @@ public class DownloadController : Controller
var orderId = Request.Headers.GetHeader("V-OrderId") ?? Request.Query["orderId"];
if (!await IsOrderPaid(orderId))
{
Response.StatusCode = (int) HttpStatusCode.PaymentRequired;
Response.StatusCode = (int)HttpStatusCode.PaymentRequired;
return default;
}
}

9
VoidCat/Controllers/InfoController.cs
View File

@ -11,14 +11,16 @@ public class InfoController : Controller
private readonly IFileMetadataStore _fileMetadata;
private readonly VoidSettings _settings;
private readonly ITimeSeriesStatsReporter _timeSeriesStats;
private readonly IEnumerable<string?> _fileStores;
public InfoController(IStatsReporter statsReporter, IFileMetadataStore fileMetadata, VoidSettings settings,
ITimeSeriesStatsReporter stats)
ITimeSeriesStatsReporter stats, IEnumerable<IFileStore> fileStores)
{
_statsReporter = statsReporter;
_fileMetadata = fileMetadata;
_settings = settings;
_timeSeriesStats = stats;
_fileStores = fileStores.Select(a => a.Key);
}
/// <summary>
@ -34,9 +36,10 @@ public class InfoController : Controller
return new(bw, storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(),
_settings.CaptchaSettings?.SiteKey,
await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow));
await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow),
_fileStores);
}
public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo,
string? CaptchaSiteKey, IEnumerable<BandwidthPoint> TimeSeriesMetrics);
string? CaptchaSiteKey, IEnumerable<BandwidthPoint> TimeSeriesMetrics, IEnumerable<string?> FileStores);
}

70
VoidCat/Controllers/UploadController.cs
View File

@ -6,23 +6,26 @@ using Newtonsoft.Json;
using VoidCat.Model;
using VoidCat.Model.Paywall;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Controllers
{
[Route("upload")]
public class UploadController : Controller
{
private readonly IFileStore _storage;
private readonly FileStoreFactory _storage;
private readonly IFileMetadataStore _metadata;
private readonly IPaywallStore _paywall;
private readonly IPaywallFactory _paywallFactory;
private readonly IFileInfoManager _fileInfo;
private readonly IUserUploadsStore _userUploads;
private readonly IUserStore _userStore;
private readonly ITimeSeriesStatsReporter _timeSeriesStats;
private readonly VoidSettings _settings;
public UploadController(IFileStore storage, IFileMetadataStore metadata, IPaywallStore paywall,
public UploadController(FileStoreFactory storage, IFileMetadataStore metadata, IPaywallStore paywall,
IPaywallFactory paywallFactory, IFileInfoManager fileInfo, IUserUploadsStore userUploads,
ITimeSeriesStatsReporter timeSeriesStats)
ITimeSeriesStatsReporter timeSeriesStats, IUserStore userStore, VoidSettings settings)
{
_storage = storage;
_metadata = metadata;
@ -31,6 +34,8 @@ namespace VoidCat.Controllers
_fileInfo = fileInfo;
_userUploads = userUploads;
_timeSeriesStats = timeSeriesStats;
_userStore = userStore;
_settings = settings;
}
/// <summary>
@ -65,20 +70,29 @@ namespace VoidCat.Controllers
}
}
// detect store for ingress
var store = _settings.DefaultFileStore;
if (uid.HasValue)
{
var user = await _userStore.Get<InternalVoidUser>(uid.Value);
if (user?.Storage != default)
{
store = user.Storage!;
}
}
var meta = new SecretVoidFileMeta
{
MimeType = mime,
Name = filename,
Description = Request.Headers.GetHeader("V-Description"),
Digest = Request.Headers.GetHeader("V-Full-Digest"),
Size = (ulong?) Request.ContentLength ?? 0UL
Size = (ulong?)Request.ContentLength ?? 0UL,
Storage = store
};
var digest = Request.Headers.GetHeader("V-Digest");
var vf = await _storage.Ingress(new(Request.Body, meta)
{
Hash = digest
}, HttpContext.RequestAborted);
var (segment, totalSegments) = ParseSegmentsHeader();
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments), HttpContext.RequestAborted);
// save metadata
await _metadata.Set(vf.Id, vf.Metadata!);
@ -130,14 +144,20 @@ namespace VoidCat.Controllers
var meta = await _metadata.Get<SecretVoidFileMeta>(gid);
if (meta == default) return UploadResult.Error("File not found");
var editSecret = Request.Headers.GetHeader("V-EditSecret");
var digest = Request.Headers.GetHeader("V-Digest");
var vf = await _storage.Ingress(new(Request.Body, meta)
// Parse V-Segment header
var (segment, totalSegments) = ParseSegmentsHeader();
// sanity check for append operations
if (segment <= 1 || totalSegments <= 1)
{
return UploadResult.Error("Malformed request, segment must be > 1 for append");
}
var editSecret = Request.Headers.GetHeader("V-EditSecret");
var vf = await _storage.Ingress(new(Request.Body, meta, segment, totalSegments)
{
Hash = digest,
EditSecret = editSecret?.FromBase58Guid() ?? Guid.Empty,
Id = gid,
IsAppend = true
Id = gid
}, HttpContext.RequestAborted);
// update file size
@ -160,6 +180,7 @@ namespace VoidCat.Controllers
public async Task<IActionResult> GetInfo([FromRoute] string id)
{
if (!id.TryFromBase58Guid(out var fid)) return StatusCode(404);
var uid = HttpContext.GetUserId();
var isOwner = uid.HasValue && await _userUploads.Uploader(fid) == uid;
@ -240,6 +261,7 @@ namespace VoidCat.Controllers
Handle = req.Strike.Handle,
Cost = req.Strike.Cost
});
return Ok();
}
@ -269,6 +291,24 @@ namespace VoidCat.Controllers
await _metadata.Update(gid, fileMeta);
return Ok();
}
private (int Segment, int TotalSegments) ParseSegmentsHeader()
{
// Parse V-Segment header
int segment = 1, totalSegments = 1;
var segmentHeader = Request.Headers.GetHeader("V-Segment");
if (!string.IsNullOrEmpty(segmentHeader))
{
var split = segmentHeader.Split("/");
if (split.Length == 2 && int.TryParse(split[0], out var a) && int.TryParse(split[1], out var b))
{
segment = a;
totalSegments = b;
}
}
return (segment, totalSegments);
}
}
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]

4
VoidCat/Controllers/UserController.cs
View File

@ -12,7 +12,8 @@ public class UserController : Controller
private readonly IEmailVerification _emailVerification;
private readonly IFileInfoManager _fileInfoManager;
public UserController(IUserStore store, IUserUploadsStore userUploads, IEmailVerification emailVerification, IFileInfoManager fileInfoManager)
public UserController(IUserStore store, IUserUploadsStore userUploads, IEmailVerification emailVerification,
IFileInfoManager fileInfoManager)
{
_store = store;
_userUploads = userUploads;
@ -42,6 +43,7 @@ public class UserController : Controller
{
var pUser = await _store.Get<PrivateVoidUser>(requestedId);
if (pUser == default) return NotFound();
return Json(pUser);
}

18
VoidCat/Model/ApiKey.cs Normal file
View File

@ -0,0 +1,18 @@
using Newtonsoft.Json;
namespace VoidCat.Model;
public sealed class ApiKey
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; init; }
[JsonConverter(typeof(Base58GuidConverter))]
public Guid UserId { get; init; }
public string Token { get; init; }
public DateTime Expiry { get; init; }
public DateTime Created { get; init; }
}

8
VoidCat/Model/EgressRequest.cs
View File

@ -1,7 +1,5 @@
using VoidCat.Services.Abstractions;
namespace VoidCat.Model;
public sealed record EgressRequest(Guid Id, IEnumerable<RangeRequest> Ranges)
{
}
public sealed record EgressRequest(Guid Id, IEnumerable<RangeRequest> Ranges);
public sealed record EgressResult(Uri? Redirect = null);

7
VoidCat/Model/IngressPayload.cs
View File

@ -1,10 +1,11 @@
namespace VoidCat.Model;
public sealed record IngressPayload(Stream InStream, SecretVoidFileMeta Meta)
public sealed record IngressPayload(Stream InStream, SecretVoidFileMeta Meta, int Segment, int TotalSegments)
{
public Guid Id { get; init; } = Guid.NewGuid();
public Guid? EditSecret { get; init; }
public string? Hash { get; init; }
public bool IsAppend { get; init; }
public bool IsAppend => Segment > 1 && IsMultipart;
public bool IsMultipart => TotalSegments > 1;
}

5
VoidCat/Model/VoidFileMeta.cs
View File

@ -70,6 +70,11 @@ public record VoidFileMeta : IVoidFileMeta
/// Time when the file will expire and be deleted
/// </summary>
public DateTimeOffset? Expires { get; set; }
/// <summary>
/// What storage system the file is on
/// </summary>
public string? Storage { get; set; }
}
/// <summary>

15
VoidCat/Model/VoidSettings.cs
View File

@ -75,6 +75,16 @@ namespace VoidCat.Model
/// Prometheus server for querying metrics
/// </summary>
public Uri? Prometheus { get; init; }
/// <summary>
/// Select where to store metadata, if not set "local-disk" will be used
/// </summary>
public string MetadataStore { get; init; } = "local-disk";
/// <summary>
/// Select which store to use for files storage, if not set "local-disk" will be used
/// </summary>
public string DefaultFileStore { get; init; } = "local-disk";
}
public sealed class TorSettings
@ -99,17 +109,18 @@ namespace VoidCat.Model
public sealed class CloudStorageSettings
{
public bool ServeFromCloud { get; init; }
public S3BlobConfig? S3 { get; set; }
public S3BlobConfig[]? S3 { get; init; }
}
public sealed class S3BlobConfig
{
public string Name { get; init; } = null!;
public string? AccessKey { get; init; }
public string? SecretKey { get; init; }
public Uri? ServiceUrl { get; init; }
public string? Region { get; init; }
public string? BucketName { get; init; } = "void-cat";
public bool Direct { get; init; }
}
public sealed class VirusScannerSettings

7
VoidCat/Model/VoidUser.cs
View File

@ -82,7 +82,12 @@ public class PrivateVoidUser : VoidUser
/// <summary>
/// Users email address
/// </summary>
public string Email { get; init; } = null!;
public string Email { get; set; } = null!;
/// <summary>
/// Users storage system for new uploads
/// </summary>
public string? Storage { get; set; }
}
/// <inheritdoc />

16
VoidCat/Services/Abstractions/IApiKeyStore.cs Normal file
View File

@ -0,0 +1,16 @@
using VoidCat.Model;
namespace VoidCat.Services.Abstractions;
/// <summary>
/// Api key store
/// </summary>
public interface IApiKeyStore : IBasicStore<ApiKey>
{
/// <summary>
/// Return a list of Api keys for a given user
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<IReadOnlyList<ApiKey>> ListKeys(Guid id);
}

5
VoidCat/Services/Abstractions/ICaptchaVerifier.cs
View File

@ -2,5 +2,10 @@
public interface ICaptchaVerifier
{
/// <summary>
/// Verify captcha token is valid
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
ValueTask<bool> Verify(string? token);
}

12
VoidCat/Services/Abstractions/IFileStore.cs
View File

@ -7,6 +7,11 @@ namespace VoidCat.Services.Abstractions;
/// </summary>
public interface IFileStore
{
/// <summary>
/// Return key for named instance
/// </summary>
string? Key { get; }
/// <summary>
/// Ingress a file into the system (Upload)
/// </summary>
@ -24,6 +29,13 @@ public interface IFileStore
/// <returns></returns>
ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts);
/// <summary>
/// Pre-Egress checks
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
ValueTask<EgressResult> StartEgress(EgressRequest request);
/// <summary>
/// Deletes file data only, metadata must be deleted with <see cref="IFileInfoManager.Delete"/>
/// </summary>

7
VoidCat/Services/Abstractions/IUserStore.cs
View File

@ -43,4 +43,11 @@ public interface IUserStore : IPublicPrivateStore<VoidUser, InternalVoidUser>
/// <param name="timestamp"></param>
/// <returns></returns>
ValueTask UpdateLastLogin(Guid id, DateTime timestamp);
/// <summary>
/// Update user account for admin
/// </summary>
/// <param name="user"></param>
/// <returns></returns>
ValueTask AdminUpdateUser(PrivateVoidUser user);
}

3
VoidCat/Services/Background/DeleteUnverifiedAccounts.cs
View File

@ -1,5 +1,6 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Services.Background;
@ -23,7 +24,7 @@ public class DeleteUnverifiedAccounts : BackgroundService
using var scope = _scopeFactory.CreateScope();
var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
var fileStore = scope.ServiceProvider.GetRequiredService<IFileStore>();
var fileStore = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));

6
VoidCat/Services/Captcha/NoOpVerifier.cs
View File

@ -2,10 +2,14 @@
namespace VoidCat.Services.Captcha;
/// <summary>
/// No captcha system is configured
/// </summary>
public class NoOpVerifier : ICaptchaVerifier
{
/// <inheritdoc />
public ValueTask<bool> Verify(string? token)
{
return ValueTask.FromResult(token == null);
return ValueTask.FromResult(true);
}
}

33
VoidCat/Services/Files/FileStorageStartup.cs
View File

@ -9,29 +9,44 @@ public static class FileStorageStartup
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IFileInfoManager, FileInfoManager>();
services.AddTransient<FileStoreFactory>();
if (settings.CloudStorage != default)
{
services.AddTransient<IUserUploadsStore, CacheUserUploadStore>();
// cloud storage
if (settings.CloudStorage.S3 != default)
// S3 storage
foreach (var s3 in settings.CloudStorage.S3 ?? Array.Empty<S3BlobConfig>())
{
services.AddSingleton<IFileStore, S3FileStore>();
services.AddSingleton<IFileMetadataStore, S3FileMetadataStore>();
services.AddTransient<IFileStore>((svc) =>
new S3FileStore(s3,
svc.GetRequiredService<IAggregateStatsCollector>(),
svc.GetRequiredService<IFileInfoManager>(),
svc.GetRequiredService<ICache>()));
if (settings.MetadataStore == s3.Name)
{
services.AddSingleton<IFileMetadataStore>((svc) =>
new S3FileMetadataStore(s3, svc.GetRequiredService<ILogger<S3FileMetadataStore>>()));
}
}
else if (!string.IsNullOrEmpty(settings.Postgres))
}
if (!string.IsNullOrEmpty(settings.Postgres))
{
services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, PostgresFileMetadataStore>();
if (settings.MetadataStore == "postgres")
{
services.AddSingleton<IFileMetadataStore, PostgresFileMetadataStore>();
}
}
else
{
services.AddTransient<IUserUploadsStore, CacheUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
if (settings.MetadataStore == "local-disk")
{
services.AddSingleton<IFileMetadataStore, LocalDiskFileMetadataStore>();
}
}
}
}

96
VoidCat/Services/Files/FileSystemFactory.cs Normal file
View File

@ -0,0 +1,96 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <summary>
/// Primary class for accessing <see cref="IFileStore"/> implementations
/// </summary>
public class FileStoreFactory : IFileStore
{
private readonly IFileMetadataStore _metadataStore;
private readonly IEnumerable<IFileStore> _fileStores;
public FileStoreFactory(IEnumerable<IFileStore> fileStores, IFileMetadataStore metadataStore)
{
_fileStores = fileStores;
_metadataStore = metadataStore;
}
/// <summary>
/// Get files store interface by key
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public IFileStore? GetFileStore(string? key)
{
if (key == default && _fileStores.Count() == 1)
{
return _fileStores.First();
}
return _fileStores.FirstOrDefault(a => a.Key == key);
}
/// <inheritdoc />
public string? Key => null;
/// <inheritdoc />
public ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
var store = GetFileStore(payload.Meta.Storage!);
if (store == default)
{
throw new InvalidOperationException($"Cannot find store '{payload.Meta.Storage}'");
}
return store.Ingress(payload, cts);
}
/// <inheritdoc />
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{
var store = await GetStore(request.Id);
await store.Egress(request, outStream, cts);
}
/// <inheritdoc />
public async ValueTask<EgressResult> StartEgress(EgressRequest request)
{
var store = await GetStore(request.Id);
return await store.StartEgress(request);
}
/// <inheritdoc />
public async ValueTask DeleteFile(Guid id)
{
var store = await GetStore(id);
await store.DeleteFile(id);
}
/// <inheritdoc />
public async ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
{
var store = await GetStore(request.Id);
return await store.Open(request, cts);
}
/// <summary>
/// Get file store for a file by id
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
private async Task<IFileStore> GetStore(Guid id)
{
var meta = await _metadataStore.Get(id);
var store = GetFileStore(meta?.Storage);
if (store == default)
{
throw new InvalidOperationException($"Cannot find store '{meta?.Storage}'");
}
return store;
}
}

1
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@ -53,6 +53,7 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}

9
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@ -31,6 +31,15 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
await EgressFromStream(fs, request, outStream, cts);
}
/// <inheritdoc />
public ValueTask<EgressResult> StartEgress(EgressRequest request)
{
return ValueTask.FromResult(new EgressResult());
}
/// <inheritdoc />
public string Key => "local-disk";
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{

19
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@ -14,6 +14,9 @@ public class PostgresFileMetadataStore : IFileMetadataStore
_connection = connection;
}
/// <inheritdoc />
public string? Key => "postgres";
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
{
@ -32,9 +35,15 @@ public class PostgresFileMetadataStore : IFileMetadataStore
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"insert into
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires)
on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Description"" = :description, ""MimeType"" = :mimeType, ""Expires"" = :expires",
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store)
on conflict (""Id"") do update set
""Name"" = :name,
""Size"" = :size,
""Description"" = :description,
""MimeType"" = :mimeType,
""Expires"" = :expires,
""Storage"" = :store",
new
{
id,
@ -45,7 +54,8 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Descrip
mimeType = obj.MimeType,
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires
expires = obj.Expires,
store = obj.Storage
});
}
@ -82,6 +92,7 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Descrip
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}

19
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@ -11,16 +11,17 @@ public class S3FileMetadataStore : IFileMetadataStore
private readonly ILogger<S3FileMetadataStore> _logger;
private readonly AmazonS3Client _client;
private readonly S3BlobConfig _config;
private readonly bool _includeUrl;
public S3FileMetadataStore(VoidSettings settings, ILogger<S3FileMetadataStore> logger)
public S3FileMetadataStore(S3BlobConfig settings, ILogger<S3FileMetadataStore> logger)
{
_logger = logger;
_includeUrl = settings.CloudStorage?.ServeFromCloud ?? false;
_config = settings.CloudStorage!.S3!;
_config = settings;
_client = _config.CreateClient();
}
/// <inheritdoc />
public string? Key => _config.Name;
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{
@ -53,6 +54,7 @@ public class S3FileMetadataStore : IFileMetadataStore
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}
@ -141,15 +143,6 @@ public class S3FileMetadataStore : IFileMetadataStore
if (ret != default)
{
ret.Id = id;
if (_includeUrl)
{
var ub = new UriBuilder(_config.ServiceUrl!)
{
Path = $"/{_config.BucketName}/{id}"
};
ret.Url = ub.Uri;
}
}
return ret;

134
VoidCat/Services/Files/S3FileStore.cs
View File

@ -1,27 +1,37 @@
using Amazon.S3;
using System.Net;
using Amazon.S3;
using Amazon.S3.Model;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
public class S3FileStore : StreamFileStore, IFileStore
{
private readonly IFileInfoManager _fileInfo;
private readonly AmazonS3Client _client;
private readonly S3BlobConfig _config;
private readonly IAggregateStatsCollector _statsCollector;
private readonly ICache _cache;
public S3FileStore(VoidSettings settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo) : base(stats)
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo, ICache cache) : base(stats)
{
_fileInfo = fileInfo;
_cache = cache;
_statsCollector = stats;
_config = settings.CloudStorage!.S3!;
_config = settings;
_client = _config.CreateClient();
}
/// <inheritdoc />
public string Key => _config.Name;
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
if (payload.IsMultipart) return await IngressMultipart(payload, cts);
var req = new PutObjectRequest
{
BucketName = _config.BucketName,
@ -31,28 +41,46 @@ public class S3FileStore : StreamFileStore, IFileStore
AutoResetStreamPosition = false,
AutoCloseStream = false,
ChecksumAlgorithm = ChecksumAlgorithm.SHA256,
ChecksumSHA256 = payload.Hash != default ? Convert.ToBase64String(payload.Hash!.FromHex()) : null,
StreamTransferProgress = (s, e) =>
{
_statsCollector.TrackIngress(payload.Id, (ulong) e.IncrementTransferred)
.GetAwaiter().GetResult();
},
ChecksumSHA256 = payload.Meta.Digest != default ? Convert.ToBase64String(payload.Meta.Digest!.FromHex()) : null,
Headers =
{
ContentLength = (long) payload.Meta.Size
ContentLength = (long)payload.Meta.Size
}
};
await _client.PutObjectAsync(req, cts);
await _statsCollector.TrackIngress(payload.Id, payload.Meta.Size);
return HandleCompletedUpload(payload, payload.Meta.Size);
}
/// <inheritdoc />
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{
await using var stream = await Open(request, cts);
await EgressFull(request.Id, stream, outStream, cts);
}
/// <inheritdoc />
public async ValueTask<EgressResult> StartEgress(EgressRequest request)
{
if (!_config.Direct) return new();
var meta = await _fileInfo.Get(request.Id);
var url = _client.GetPreSignedURL(new()
{
BucketName = _config.BucketName,
Expires = DateTime.UtcNow.AddHours(1),
Key = request.Id.ToString(),
ResponseHeaderOverrides = new()
{
ContentDisposition = $"inline; filename=\"{meta?.Metadata?.Name}\"",
ContentType = meta?.Metadata?.MimeType
}
});
return new(new Uri(url));
}
public async ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request)
{
try
@ -108,11 +136,13 @@ public class S3FileStore : StreamFileStore, IFileStore
}
}
/// <inheritdoc />
public async ValueTask DeleteFile(Guid id)
{
await _client.DeleteObjectAsync(_config.BucketName, id.ToString());
}
/// <inheritdoc />
public async ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
{
var req = new GetObjectRequest()
@ -120,6 +150,7 @@ public class S3FileStore : StreamFileStore, IFileStore
BucketName = _config.BucketName,
Key = request.Id.ToString()
};
if (request.Ranges.Any())
{
var r = request.Ranges.First();
@ -129,4 +160,87 @@ public class S3FileStore : StreamFileStore, IFileStore
var obj = await _client.GetObjectAsync(req, cts);
return obj.ResponseStream;
}
private async Task<PrivateVoidFile> IngressMultipart(IngressPayload payload, CancellationToken cts)
{
string? uploadId;
var cacheKey = $"s3:{_config.Name}:multipart-upload-id:{payload.Id}";
var partsCacheKey = $"s3:{_config.Name}:multipart-upload:{payload.Id}";
if (payload.Segment == 1)
{
var mStart = new InitiateMultipartUploadRequest()
{
BucketName = _config.BucketName,
Key = payload.Id.ToString(),
ContentType = "application/octet-stream",
ChecksumAlgorithm = ChecksumAlgorithm.SHA256
};
var mStartResult = await _client.InitiateMultipartUploadAsync(mStart, cts);
uploadId = mStartResult.UploadId;
await _cache.Set(cacheKey, uploadId, TimeSpan.FromHours(1));
}
else
{
uploadId = await _cache.Get<string>(cacheKey);
}
// sadly it seems like we need a tmp file here
var tmpFile = Path.GetTempFileName();
await using var fsTmp = new FileStream(tmpFile, FileMode.Create, FileAccess.ReadWrite);
await payload.InStream.CopyToAsync(fsTmp, cts);
fsTmp.Seek(0, SeekOrigin.Begin);
var segmentLength = (ulong)fsTmp.Length;
var mbody = new UploadPartRequest()
{
UploadId = uploadId,
BucketName = _config.BucketName,
PartNumber = payload.Segment,
Key = payload.Id.ToString(),
InputStream = fsTmp
};
var bodyResponse = await _client.UploadPartAsync(mbody, cts);
if (bodyResponse.HttpStatusCode != HttpStatusCode.OK)
{
await _client.AbortMultipartUploadAsync(new()
{
BucketName = _config.BucketName,
UploadId = uploadId
}, cts);
throw new Exception("Upload aborted");
}
await _statsCollector.TrackIngress(payload.Id, segmentLength);
await _cache.AddToList(partsCacheKey, $"{payload.Segment}|{bodyResponse.ETag.Replace("\"", string.Empty)}");
if (payload.Segment == payload.TotalSegments)
{
var parts = await _cache.GetList(partsCacheKey);
var completeResponse = await _client.CompleteMultipartUploadAsync(new()
{
BucketName = _config.BucketName,
Key = payload.Id.ToString(),
UploadId = uploadId,
PartETags = parts.Select(a =>
{
var pSplit = a.Split('|');
return new PartETag()
{
PartNumber = int.Parse(pSplit[0]),
ETag = pSplit[1]
};
}).ToList()
}, cts);
if (completeResponse.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception("Upload failed");
}
}
return HandleCompletedUpload(payload, segmentLength);
}
}

37
VoidCat/Services/Migrations/Database/02-MinorVersion1.cs Normal file
View File

@ -0,0 +1,37 @@
using System.Data;
using FluentMigrator;
namespace VoidCat.Services.Migrations.Database;
[Migration(20220725_1137)]
public class MinorVersion1 : Migration
{
public override void Up()
{
Create.Table("ApiKey")
.WithColumn("Id").AsGuid().PrimaryKey()
.WithColumn("UserId").AsGuid().ForeignKey("Users", "Id").OnDelete(Rule.Cascade).Indexed()
.WithColumn("Token").AsString()
.WithColumn("Expiry").AsDateTimeOffset()
.WithColumn("Created").AsDateTimeOffset().WithDefault(SystemMethods.CurrentUTCDateTime);
Create.Column("Storage")
.OnTable("Files")
.AsString().WithDefaultValue("local-disk");
Create.Column("Storage")
.OnTable("Users")
.AsString().WithDefaultValue("local-disk");
}
public override void Down()
{
Delete.Table("ApiKey");
Delete.Column("Storage")
.FromTable("Files");
Delete.Column("Storage")
.FromTable("Users");
}
}

17
VoidCat/Services/Migrations/FixSize.cs
View File

@ -1,5 +1,6 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Services.Migrations;
@ -8,9 +9,9 @@ public class FixSize : IMigration
{
private readonly ILogger<FixSize> _logger;
private readonly IFileMetadataStore _fileMetadata;
private readonly IFileStore _fileStore;
private readonly FileStoreFactory _fileStore;
public FixSize(ILogger<FixSize> logger, IFileMetadataStore fileMetadata, IFileStore fileStore)
public FixSize(ILogger<FixSize> logger, IFileMetadataStore fileMetadata, FileStoreFactory fileStore)
{
_logger = logger;
_fileMetadata = fileMetadata;
@ -25,18 +26,26 @@ public class FixSize : IMigration
{
var files = await _fileMetadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
await foreach (var file in files.Results)
{
try
{
var fs = await _fileStore.Open(new(file.Id, Enumerable.Empty<RangeRequest>()), CancellationToken.None);
if (file.Size != (ulong) fs.Length)
if (file.Size != (ulong)fs.Length)
{
_logger.LogInformation("Updating file size {Id} to {Size}", file.Id, fs.Length);
var newFile = file with
{
Size = (ulong) fs.Length
Size = (ulong)fs.Length
};
await _fileMetadata.Set(newFile.Id, newFile);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to fix file {id}", file.Id);
}
}
return IMigration.MigrationResult.Completed;
}

6
VoidCat/Services/Migrations/MigrateToPostgres.cs
View File

@ -18,11 +18,11 @@ public class MigrateToPostgres : IMigration
private readonly IPaywallStore _paywallStore;
private readonly IUserStore _userStore;
private readonly IUserUploadsStore _userUploads;
private readonly IFileStore _fileStore;
private readonly FileStoreFactory _fileStore;
public MigrateToPostgres(VoidSettings settings, ILogger<MigrateToPostgres> logger, IFileMetadataStore fileMetadata,
ICache cache, IPaywallStore paywallStore, IUserStore userStore, IUserUploadsStore userUploads,
IFileStore fileStore)
FileStoreFactory fileStore)
{
_logger = logger;
_settings = settings;
@ -75,6 +75,7 @@ public class MigrateToPostgres : IMigration
{
var fs = await _fileStore.Open(new(file.Id, Enumerable.Empty<RangeRequest>()),
CancellationToken.None);
var hash = await SHA256.Create().ComputeHashAsync(fs);
file.Digest = hash.ToHex();
}
@ -143,6 +144,7 @@ public class MigrateToPostgres : IMigration
Password = privateUser.Password!,
Roles = privateUser.Roles
});
_logger.LogInformation("Migrated user {USer}", user.Id);
}
catch (Exception ex)

20
VoidCat/Services/Users/CacheApiKeyStore.cs Normal file
View File

@ -0,0 +1,20 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
/// <inheritdoc cref="VoidCat.Services.Abstractions.IApiKeyStore" />
public class CacheApiKeyStore : BasicCacheStore<ApiKey>, IApiKeyStore
{
public CacheApiKeyStore(ICache cache) : base(cache)
{
}
/// <inheritdoc />
public ValueTask<IReadOnlyList<ApiKey>> ListKeys(Guid id)
{
throw new NotImplementedException();
}
protected override string MapKey(Guid id) => $"api-key:{id}";
}

12
VoidCat/Services/Users/CacheUserStore.cs
View File

@ -110,6 +110,18 @@ public class CacheUserStore : IUserStore
}
}
/// <inheritdoc />
public async ValueTask AdminUpdateUser(PrivateVoidUser user)
{
var oldUser = await Get<InternalVoidUser>(user.Id);
if (oldUser == null) return;
oldUser.Email = user.Email;
oldUser.Storage = user.Storage;
await Set(oldUser.Id, oldUser);
}
/// <inheritdoc />
public async ValueTask Delete(Guid id)
{

59
VoidCat/Services/Users/PostgresApiKeyStore.cs Normal file
View File

@ -0,0 +1,59 @@
using Dapper;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
/// <inheritdoc />
public class PostgresApiKeyStore : IApiKeyStore
{
private readonly PostgresConnectionFactory _factory;
public PostgresApiKeyStore(PostgresConnectionFactory factory)
{
_factory = factory;
}
/// <inheritdoc />
public async ValueTask<ApiKey?> Get(Guid id)
{
await using var conn = await _factory.Get();
return await conn.QuerySingleOrDefaultAsync<ApiKey>(@"select * from ""ApiKey"" where ""Id"" = :id", new {id});
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<ApiKey>> Get(Guid[] ids)
{
await using var conn = await _factory.Get();
return (await conn.QueryAsync<ApiKey>(@"select * from ""ApiKey"" where ""Id"" in :ids", new {ids})).ToList();
}
/// <inheritdoc />
public async ValueTask Add(Guid id, ApiKey obj)
{
await using var conn = await _factory.Get();
await conn.ExecuteAsync(@"insert into ""ApiKey""(""Id"", ""UserId"", ""Token"", ""Expiry"")
values(:id, :userId, :token, :expiry)", new
{
id = obj.Id,
userId = obj.UserId,
token = obj.Token,
expiry = obj.Expiry.ToUniversalTime()
});
}
/// <inheritdoc />
public async ValueTask Delete(Guid id)
{
await using var conn = await _factory.Get();
await conn.ExecuteAsync(@"delete from ""ApiKey"" where ""Id"" = :id", new {id});
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<ApiKey>> ListKeys(Guid id)
{
await using var conn = await _factory.Get();
return (await conn.QueryAsync<ApiKey>(@"select ""Id"", ""UserId"", ""Expiry"", ""Created"" from ""ApiKey"" where ""UserId"" = :id", new {id}))
.ToList();
}
}

24
VoidCat/Services/Users/PostgresUserStore.cs
View File

@ -43,8 +43,9 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
displayName = obj.DisplayName,
lastLogin = obj.LastLogin.ToUniversalTime(),
avatar = obj.Avatar,
flags = (int) obj.Flags
flags = (int)obj.Flags
});
if (obj.Roles.Any(a => a != Roles.User))
{
foreach (var r in obj.Roles.Where(a => a != Roles.User))
@ -69,11 +70,13 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
await using var conn = await _connection.Get();
var user = await conn.QuerySingleOrDefaultAsync<T?>(@"select * from ""Users"" where ""Id"" = :id",
new {id});
if (user != default)
{
var roles = await conn.QueryAsync<string>(
@"select ""Role"" from ""UserRoles"" where ""User"" = :id",
new {id});
foreach (var r in roles)
{
user.Roles.Add(r);
@ -106,11 +109,13 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
PagedSortBy.Name => "DisplayName",
_ => "Id"
};
var sortBy = request.SortOrder switch
{
PageSortOrder.Dsc => "desc",
_ => "asc"
};
await using var iconn = await _connection.Get();
var users = await iconn.ExecuteReaderAsync(
$@"select * from ""Users"" order by ""{orderBy}"" {sortBy} offset :offset limit :limit",
@ -119,6 +124,7 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
offset = request.PageSize * request.Page,
limit = request.PageSize
});
var rowParser = users.GetRowParser<PrivateVoidUser>();
while (await users.ReadAsync())
{
@ -144,7 +150,7 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
var emailFlag = oldUser.Flags.HasFlag(VoidUserFlags.EmailVerified) ? VoidUserFlags.EmailVerified : 0;
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar, ""Flags"" = :flags where ""Id"" = :id",
@"update ""Users"" set ""DisplayName"" = :displayName, ""Avatar"" = :avatar, ""Flags"" = :flags where ""Id"" = :id",
new
{
id = newUser.Id,
@ -161,4 +167,18 @@ values(:id, :email, :password, :created, :lastLogin, :displayName, :avatar, :fla
await conn.ExecuteAsync(@"update ""Users"" set ""LastLogin"" = :timestamp where ""Id"" = :id",
new {id, timestamp});
}
/// <inheritdoc />
public async ValueTask AdminUpdateUser(PrivateVoidUser user)
{
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"update ""Users"" set ""Email"" = :email, ""Storage"" = :storage where ""Id"" = :id",
new
{
id = user.Id,
email = user.Email,
storage = user.Storage
});
}
}

4
VoidCat/Services/Users/UsersStartup.cs
View File

@ -9,15 +9,17 @@ public static class UsersStartup
{
services.AddTransient<IUserManager, UserManager>();
if (settings.Postgres != default)
if (settings.HasPostgres())
{
services.AddTransient<IUserStore, PostgresUserStore>();
services.AddTransient<IEmailVerification, PostgresEmailVerification>();
services.AddTransient<IApiKeyStore, PostgresApiKeyStore>();
}
else
{
services.AddTransient<IUserStore, CacheUserStore>();
services.AddTransient<IEmailVerification, CacheEmailVerification>();
services.AddTransient<IApiKeyStore, CacheApiKeyStore>();
}
}
}

9
VoidCat/Services/VirusScanner/ClamAvScanner.cs
View File

@ -1,6 +1,7 @@
using nClam;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Services.VirusScanner;
@ -11,13 +12,13 @@ public class ClamAvScanner : IVirusScanner
{
private readonly ILogger<ClamAvScanner> _logger;
private readonly IClamClient _clam;
private readonly IFileStore _store;
private readonly FileStoreFactory _fileSystemFactory;
public ClamAvScanner(ILogger<ClamAvScanner> logger, IClamClient clam, IFileStore store)
public ClamAvScanner(ILogger<ClamAvScanner> logger, IClamClient clam, FileStoreFactory fileSystemFactory)
{
_logger = logger;
_clam = clam;
_store = store;
_fileSystemFactory = fileSystemFactory;
}
/// <inheritdoc />
@ -25,7 +26,7 @@ public class ClamAvScanner : IVirusScanner
{
_logger.LogInformation("Starting scan of {Filename}", id);
await using var fs = await _store.Open(new(id, Enumerable.Empty<RangeRequest>()), cts);
await using var fs = await _fileSystemFactory.Open(new(id, Enumerable.Empty<RangeRequest>()), cts);
var result = await _clam.SendAndScanFileAsync(fs, cts);
if (result.Result == ClamScanResults.Error)

2
VoidCat/VoidCat.csproj
View File

@ -10,7 +10,7 @@
<HostSPA>True</HostSPA>
<DefineConstants Condition="'$(HostSPA)' == 'True'">$(DefineConstants);HostSPA</DefineConstants>
<DocumentationFile>$(AssemblyName).xml</DocumentationFile>
<Version>4.0.0</Version>
<Version>4.1.0</Version>
</PropertyGroup>
<ItemGroup>

17
VoidCat/spa/src/Admin/Admin.css
View File

@ -9,23 +9,6 @@
padding: 10px;
}
.admin table {
width: 100%;
word-break: keep-all;
text-overflow: ellipsis;
white-space: nowrap;
border-collapse: collapse;
}
.admin table th {
background-color: #222;
text-align: start;
}
.admin table tr:nth-child(2n) {
background-color: #111;
}
.admin .btn {
padding: 5px 8px;
border-radius: 3px;

15
VoidCat/spa/src/Admin/Admin.js
View File

@ -5,11 +5,14 @@ import {UserList} from "./UserList";
import {Navigate} from "react-router-dom";
import {useApi} from "../Api";
import {VoidButton} from "../VoidButton";
import {useState} from "react";
import VoidModal from "../VoidModal";
import EditUser from "./EditUser";
export function Admin() {
const auth = useSelector((state) => state.login.jwt);
const {AdminApi} = useApi();
const [editUser, setEditUser] = useState(null);
async function deleteFile(e, id) {
if (window.confirm(`Are you sure you want to delete: ${id}?`)) {
@ -28,7 +31,10 @@ export function Admin() {
return (
<div className="admin">
<h2>Users</h2>
<UserList/>
<UserList actions={(i) => [
<VoidButton key={`delete-${i.id}`}>Delete</VoidButton>,
<VoidButton key={`edit-${i.id}`} onClick={(e) => setEditUser(i)}>Edit</VoidButton>
]}/>
<h2>Files</h2>
<FileList loadPage={AdminApi.fileList} actions={(i) => {
@ -36,6 +42,11 @@ export function Admin() {
<VoidButton onClick={(e) => deleteFile(e, i.id)}>Delete</VoidButton>
</td>
}}/>
{editUser !== null ?
<VoidModal title="Edit user">
<EditUser user={editUser} onClose={() => setEditUser(null)}/>
</VoidModal> : null}
</div>
);
}

46
VoidCat/spa/src/Admin/EditUser.js Normal file
View File

@ -0,0 +1,46 @@
import {VoidButton} from "../VoidButton";
import {useState} from "react";
import {useSelector} from "react-redux";
import {useApi} from "../Api";
export default function EditUser(props) {
const user = props.user;
const onClose = props.onClose;
const adminApi = useApi().AdminApi;
const fileStores = useSelector((state) => state.info?.stats?.fileStores ?? ["local-disk"])
const [storage, setStorage] = useState(user.storage);
const [email, setEmail] = useState(user.email);
async function updateUser() {
await adminApi.updateUser({
id: user.id,
email,
storage
});
onClose();
}
return (
<>
Editing user '{user.displayName}' ({user.id})
<dl>
<dt>Email:</dt>
<dd><input type="text" value={email} onChange={(e) => setEmail(e.target.value)}/></dd>
<dt>File storage:</dt>
<dd>
<select value={storage} onChange={(e) => setStorage(e.target.value)}>
<option disabled={true}>Current: {storage}</option>
{fileStores.map(e => <option key={e}>{e}</option>)}
</select>
</dd>
<dt>Roles:</dt>
<dd>{user.roles.map(e => <span className="btn" key={e}>{e}</span>)}</dd>
</dl>
<VoidButton onClick={(e) => updateUser()}>Save</VoidButton>
<VoidButton onClick={(e) => onClose()}>Cancel</VoidButton>
</>
);
}

9
VoidCat/spa/src/Admin/UserList.js
View File

@ -5,15 +5,15 @@ import {useApi} from "../Api";
import {logout} from "../LoginState";
import {PageSelector} from "../PageSelector";
import moment from "moment";
import {VoidButton} from "../VoidButton";
export function UserList() {
export function UserList(props) {
const {AdminApi} = useApi();
const dispatch = useDispatch();
const [users, setUsers] = useState();
const [page, setPage] = useState(0);
const pageSize = 10;
const [accessDenied, setAccessDenied] = useState();
const actions = props.actions;
async function loadUserList() {
let pageReq = {
@ -40,10 +40,7 @@ export function UserList() {
<td>{moment(user.created).fromNow()}</td>
<td>{moment(user.lastLogin).fromNow()}</td>
<td>{obj.uploads}</td>
<td>
<VoidButton>Delete</VoidButton>
<VoidButton>SetRoles</VoidButton>
</td>
<td>{actions(user)}</td>
</tr>
);
}

7
VoidCat/spa/src/Api.js
View File

@ -27,7 +27,8 @@ export function useApi() {
AdminApi: {
fileList: (pageReq) => getJson("POST", "/admin/file", pageReq, auth),
deleteFile: (id) => getJson("DELETE", `/admin/file/${id}`, undefined, auth),
userList: (pageReq) => getJson("POST", `/admin/user`, pageReq, auth)
userList: (pageReq) => getJson("POST", `/admin/users`, pageReq, auth),
updateUser: (user) => getJson("POST", `/admin/update-user`, user, auth)
},
Api: {
info: () => getJson("GET", "/info"),
@ -42,7 +43,9 @@ export function useApi() {
listUserFiles: (uid, pageReq) => getJson("POST", `/user/${uid}/files`, pageReq, auth),
submitVerifyCode: (uid, code) => getJson("POST", `/user/${uid}/verify`, code, auth),
sendNewCode: (uid) => getJson("GET", `/user/${uid}/verify`, undefined, auth),
updateMetadata: (id, meta) => getJson("POST", `/upload/${id}/meta`, meta, auth)
updateMetadata: (id, meta) => getJson("POST", `/upload/${id}/meta`, meta, auth),
listApiKeys: () => getJson("GET", `/auth/api-key`, undefined, auth),
createApiKey: (req) => getJson("POST", `/auth/api-key`, req, auth)
}
};
}

71
VoidCat/spa/src/ApiKeyList.js Normal file
View File

@ -0,0 +1,71 @@
import {useApi} from "./Api";
import {useEffect, useState} from "react";
import {VoidButton} from "./VoidButton";
import moment from "moment";
import VoidModal from "./VoidModal";
export default function ApiKeyList() {
const {Api} = useApi();
const [apiKeys, setApiKeys] = useState([]);
const [newApiKey, setNewApiKey] = useState();
const DefaultExpiry = 1000 * 60 * 60 * 24 * 90;
async function loadApiKeys() {
let keys = await Api.listApiKeys();
setApiKeys(await keys.json());
}
async function createApiKey() {
let rsp = await Api.createApiKey({
expiry: new Date(new Date().getTime() + DefaultExpiry)
});
if (rsp.ok) {
setNewApiKey(await rsp.json());
}
}
useEffect(() => {
if (Api) {
loadApiKeys();
}
}, []);
return (
<>
<div className="flex flex-center">
<div className="flx-grow">
<h1>API Keys</h1>
</div>
<div>
<VoidButton onClick={(e) => createApiKey()}>+New</VoidButton>
</div>
</div>
<table>
<thead>
<tr>
<th>Id</th>
<th>Created</th>
<th>Expiry</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{apiKeys.map(e => <tr key={e.id}>
<td>{e.id}</td>
<td>{moment(e.created).fromNow()}</td>
<td>{moment(e.expiry).fromNow()}</td>
<td>
<VoidButton>Delete</VoidButton>
</td>
</tr>)}
</tbody>
</table>
{newApiKey ?
<VoidModal title="New Api Key" style={{maxWidth: "50vw"}}>
Please save this now as it will not be shown again:
<pre className="copy">{newApiKey.token}</pre>
<VoidButton onClick={(e) => setNewApiKey(undefined)}>Close</VoidButton>
</VoidModal> : null}
</>
);
}

16
VoidCat/spa/src/FileList.css
View File

@ -1,16 +0,0 @@
table.file-list {
width: 100%;
word-break: keep-all;
text-overflow: ellipsis;
white-space: nowrap;
border-collapse: collapse;
}
table.file-list tr:nth-child(2n) {
background-color: #111;
}
table.file-list th {
background-color: #222;
text-align: start;
}

3
VoidCat/spa/src/FileList.js
View File

@ -1,4 +1,3 @@
import "./FileList.css";
import moment from "moment";
import {Link} from "react-router-dom";
import {useDispatch} from "react-redux";
@ -59,7 +58,7 @@ export function FileList(props) {
}
return (
<table className="file-list">
<table>
<thead>
<tr>
<th>Id</th>

16
VoidCat/spa/src/FileUpload.js
View File

@ -87,11 +87,11 @@ export function FileUpload(props) {
* @param id {string}
* @param editSecret {string?}
* @param fullDigest {string?} Full file hash
* @param part {int?} Segment number
* @param partOf {int?} Total number of segments
* @returns {Promise<any>}
*/
async function xhrSegment(segment, id, editSecret, fullDigest) {
setUState(UploadState.Hashing);
const digest = await crypto.subtle.digest(DigestAlgo, segment);
async function xhrSegment(segment, id, editSecret, fullDigest, part, partOf) {
setUState(UploadState.Uploading);
return await new Promise((resolve, reject) => {
@ -114,10 +114,10 @@ export function FileUpload(props) {
req.upload.onprogress = handleProgress;
req.open("POST", typeof (id) === "string" ? `${ApiHost}/upload/${id}` : `${ApiHost}/upload`);
req.setRequestHeader("Content-Type", "application/octet-stream");
req.setRequestHeader("V-Content-Type", props.file.type);
req.setRequestHeader("V-Content-Type", props.file.type.length === 0 ? "application/octet-stream" : props.file.type);
req.setRequestHeader("V-Filename", props.file.name);
req.setRequestHeader("V-Digest", buf2hex(digest));
req.setRequestHeader("V-Full-Digest", fullDigest);
req.setRequestHeader("V-Segment", `${part}/${partOf}`)
if (auth) {
req.setRequestHeader("Authorization", `Bearer ${auth}`);
}
@ -136,14 +136,16 @@ export function FileUpload(props) {
// upload file in segments of 50MB
const UploadSize = 50_000_000;
setUState(UploadState.Hashing);
let digest = await crypto.subtle.digest(DigestAlgo, await props.file.arrayBuffer());
let xhr = null;
const segments = props.file.size / UploadSize;
const segments = Math.ceil(props.file.size / UploadSize);
for (let s = 0; s < segments; s++) {
calc.ResetLastLoaded();
let offset = s * UploadSize;
let slice = props.file.slice(offset, offset + UploadSize, props.file.type);
let segment = await slice.arrayBuffer();
xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, buf2hex(digest));
xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, buf2hex(digest), s + 1, segments);
if (!xhr.ok) {
break;
}

2
VoidCat/spa/src/Profile.js
View File

@ -10,6 +10,7 @@ import {buf2hex, hasFlag} from "./Util";
import moment from "moment";
import {FileList} from "./FileList";
import {VoidButton} from "./VoidButton";
import ApiKeyList from "./ApiKeyList";
export function Profile() {
const [profile, setProfile] = useState();
@ -210,6 +211,7 @@ export function Profile() {
{needsEmailVerify ? renderEmailVerify() : null}
<h1>Uploads</h1>
<FileList loadPage={(req) => Api.listUserFiles(profile.id, req)}/>
{cantEditProfile ? <ApiKeyList/> : null}
</div>
</div>
);

4
VoidCat/spa/src/RateCalculator.js
View File

@ -4,6 +4,10 @@ export class RateCalculator {
this.lastLoaded = 0;
}
ResetLastLoaded() {
this.lastLoaded = 0;
}
ReportProgress(amount) {
this.reports.push({
time: new Date().getTime(),

35
VoidCat/spa/src/VoidModal.css Normal file
View File

@ -0,0 +1,35 @@
.modal-bg {
position: absolute;
top: 0;
left: 0;
width: 100vw;
height: 100vh;
background-color: rgba(0, 0, 0, 0.6);
display: flex;
align-items: center;
justify-content: center;
}
.modal-bg .modal {
min-height: 100px;
min-width: 300px;
background-color: #bbb;
color: #000;
border-radius: 10px;
overflow: hidden;
}
.modal-bg .modal .modal-header {
text-align: center;
border-bottom: 1px solid;
margin: 0;
line-height: 2em;
background-color: #222;
color: #bbb;
font-weight: bold;
text-transform: uppercase;
}
.modal-bg .modal .modal-body {
padding: 10px;
}

19
VoidCat/spa/src/VoidModal.js Normal file
View File

@ -0,0 +1,19 @@
import "./VoidModal.css";
export default function VoidModal(props) {
const title = props.title;
const style = props.style;
return (
<div className="modal-bg">
<div className="modal" style={style}>
<div className="modal-header">
{title ?? "Unknown modal"}
</div>
<div className="modal-body">
{props.children ?? "Missing body"}
</div>
</div>
</div>
)
}

25
VoidCat/spa/src/index.css
View File

@ -68,3 +68,28 @@ input[type="text"], input[type="number"], input[type="password"], select {
margin: 5px;
border: 0;
}
table {
width: 100%;
word-break: keep-all;
text-overflow: ellipsis;
white-space: nowrap;
border-collapse: collapse;
}
table tr:nth-child(2n) {
background-color: #111;
}
table th {
background-color: #222;
text-align: start;
}
pre.copy {
user-select: all;
width: fit-content;
border-radius: 4px;
border: 1px solid;
padding: 5px;
}