godzhigella/void.cat
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

Refactor

Browse Source
This commit is contained in:
Kieran 2022-02-16 16:33:00 +00:00
parent 74df427842
commit f300bbc197
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
22 changed files with 213 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VoidCat/Controllers/Admin/AdminController.cs
View File

@ -1,8 +1,10 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace VoidCat.Controllers.Admin;
[Route("admin")]
[Authorize(Policy = "Admin")]
public class AdminController : Controller
{

1
VoidCat/Controllers/DownloadController.cs
View File

@ -2,6 +2,7 @@ using System.Net;
using Microsoft.AspNetCore.Mvc;
using VoidCat.Model;
using VoidCat.Services;
using VoidCat.Services.Abstractions;
namespace VoidCat.Controllers;

14
VoidCat/Controllers/StatsController.cs
View File

@ -1,6 +1,7 @@
using Microsoft.AspNetCore.Mvc;
using VoidCat.Model;
using VoidCat.Services;
using VoidCat.Services.Abstractions;
namespace VoidCat.Controllers
{
@ -8,17 +9,24 @@ namespace VoidCat.Controllers
public class StatsController : Controller
{
private readonly IStatsCollector _statsCollector;
private readonly IFileStore _fileStore;
public StatsController(IStatsCollector statsCollector)
public StatsController(IStatsCollector statsCollector, IFileStore fileStore)
{
_statsCollector = statsCollector;
_fileStore = fileStore;
}
[HttpGet]
public async Task<GlobalStats> GetGlobalStats()
{
var bw = await _statsCollector.GetBandwidth();
return new(bw);
var bytes = 0UL;
await foreach (var vf in _fileStore.ListFiles())
{
bytes += vf.Size;
}
return new(bw, bytes);
}
[HttpGet]
@ -30,6 +38,6 @@ namespace VoidCat.Controllers
}
}
public sealed record GlobalStats(Bandwidth Bandwidth);
public sealed record GlobalStats(Bandwidth Bandwidth, ulong TotalBytes);
public sealed record FileStats(Bandwidth Bandwidth);
}

18
VoidCat/Controllers/UploadController.cs
View File

@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Mvc.ModelBinding;
using Newtonsoft.Json;
using VoidCat.Model;
using VoidCat.Services;
using VoidCat.Services.Abstractions;
namespace VoidCat.Controllers
{
@ -31,8 +32,7 @@ namespace VoidCat.Controllers
};
var digest = Request.Headers.GetHeader("X-Digest");
var vf = await (Request.HasFormContentType ?
saveFromForm() : _storage.Ingress(new(Request.Body, meta, digest!), HttpContext.RequestAborted));
var vf = await _storage.Ingress(new(Request.Body, meta, digest!), HttpContext.RequestAborted);
return UploadResult.Success(vf);
}
@ -52,7 +52,7 @@ namespace VoidCat.Controllers
{
var gid = id.FromBase58Guid();
var fileInfo = await _storage.Get(gid);
if (fileInfo == default) return null;
if (fileInfo == default) return UploadResult.Error("File not found");
var editSecret = Request.Headers.GetHeader("X-EditSecret");
var digest = Request.Headers.GetHeader("X-Digest");
@ -72,14 +72,14 @@ namespace VoidCat.Controllers
[HttpGet]
[Route("{id}")]
public Task<VoidFile?> GetInfo([FromRoute] string id)
public ValueTask<VoidFile?> GetInfo([FromRoute] string id)
{
return _storage.Get(id.FromBase58Guid());
}
[HttpPatch]
[Route("{id}")]
public Task UpdateFileInfo([FromRoute] string id, [FromBody] UpdateFileInfoRequest request)
public ValueTask UpdateFileInfo([FromRoute] string id, [FromBody] UpdateFileInfoRequest request)
{
return _storage.UpdateInfo(new VoidFile()
{
@ -88,12 +88,8 @@ namespace VoidCat.Controllers
}, request.EditSecret);
}
private Task<InternalVoidFile> saveFromForm()
{
return Task.FromResult<InternalVoidFile>(null);
}
public record UpdateFileInfoRequest([JsonConverter(typeof(Base58GuidConverter))] Guid EditSecret, VoidFileMeta Metadata);
public record UpdateFileInfoRequest([JsonConverter(typeof(Base58GuidConverter))] Guid EditSecret,
VoidFileMeta Metadata);
}
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]

8
VoidCat/Model/VoidFile.cs
View File

@ -3,25 +3,25 @@ using Newtonsoft.Json;
namespace VoidCat.Model
{
public record class VoidFile
public record VoidFile
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; init; }
public VoidFileMeta Metadata { get; set; }
public VoidFileMeta? Metadata { get; set; }
public ulong Size { get; init; }
public DateTimeOffset Uploaded { get; init; }
}
public record class InternalVoidFile : VoidFile
public record InternalVoidFile : VoidFile
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid EditSecret { get; init; }
}
public record class VoidFileMeta
public record VoidFileMeta
{
public string? Name { get; init; }

17
VoidCat/Model/VoidSettings.cs
View File

@ -5,20 +5,11 @@
public string DataDirectory { get; init; } = "./data";
public TorSettings? TorSettings { get; init; }
public JwtSettings JwtSettings { get; init; } = new("void_cat_internal", "default_key");
}
public class TorSettings
{
public TorSettings(Uri torControl, string privateKey, string controlPassword)
{
TorControl = torControl;
PrivateKey = privateKey;
ControlPassword = controlPassword;
}
public sealed record TorSettings(Uri TorControl, string PrivateKey, string ControlPassword);
public Uri TorControl { get; }
public string PrivateKey { get; }
public string ControlPassword { get; }
}
public sealed record JwtSettings(string Issuer, string Key);
}

21
VoidCat/Program.cs
View File

@ -1,6 +1,10 @@
using System.Text;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.IdentityModel.Tokens;
using Prometheus;
using VoidCat.Model;
using VoidCat.Services;
using VoidCat.Services.Abstractions;
var builder = WebApplication.CreateBuilder(args);
var services = builder.Services;
@ -14,17 +18,32 @@ builder.Logging.AddSeq(seqSettings);
services.AddRouting();
services.AddControllers().AddNewtonsoftJson();
services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddJwtBearer(options =>
{
options.TokenValidationParameters = new()
{
ValidateIssuer = true,
ValidateAudience = false,
ValidateLifetime = true,
ValidateIssuerSigningKey = true,
ValidIssuer = voidSettings.JwtSettings.Issuer,
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(voidSettings.JwtSettings.Key))
};
});
services.AddMemoryCache();
services.AddScoped<IFileMetadataStore, LocalDiskFileMetadataStore>();
services.AddScoped<IFileStore, LocalDiskFileIngressFactory>();
services.AddScoped<IFileStore, LocalDiskFileStore>();
services.AddScoped<IStatsCollector, PrometheusStatsCollector>();
var app = builder.Build();
app.UseStaticFiles();
app.UseAuthentication();
app.UseRouting();
app.UseAuthorization();
app.UseEndpoints(ep =>
{
ep.MapControllers();

12
VoidCat/Services/Abstractions/IFileMetadataStore.cs Normal file
View File

@ -0,0 +1,12 @@
using VoidCat.Model;
namespace VoidCat.Services.Abstractions;
public interface IFileMetadataStore
{
ValueTask<InternalVoidFile?> Get(Guid id);
ValueTask Set(InternalVoidFile meta);
ValueTask Update(VoidFile patch, Guid editSecret);
}

46
VoidCat/Services/Abstractions/IFileStore.cs Normal file
View File

@ -0,0 +1,46 @@
using VoidCat.Model;
namespace VoidCat.Services.Abstractions;
public interface IFileStore
{
ValueTask<VoidFile?> Get(Guid id);
ValueTask<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts);
ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts);
ValueTask UpdateInfo(VoidFile patch, Guid editSecret);
IAsyncEnumerable<VoidFile> ListFiles();
}
public sealed record IngressPayload(Stream InStream, VoidFileMeta Meta, string Hash)
{
public Guid? Id { get; init; }
public Guid? EditSecret { get; init; }
public bool IsAppend => Id.HasValue && EditSecret.HasValue;
}
public sealed record EgressRequest(Guid Id, IEnumerable<RangeRequest> Ranges)
{
}
public sealed record RangeRequest(long? TotalSize, long? Start, long? End)
{
private const long DefaultBufferSize = 1024L * 512L;
public long? Size
=> Start.HasValue ? (End ?? Math.Min(TotalSize!.Value, Start.Value + DefaultBufferSize)) - Start.Value : End;
public bool IsForFullFile
=> Start is 0 && !End.HasValue;
/// <summary>
/// Return Content-Range header content for this range
/// </summary>
/// <returns></returns>
public string ToContentRange()
=> $"bytes {Start}-{End ?? (Start + Size - 1L)}/{TotalSize?.ToString() ?? "*"}";
}

12
VoidCat/Services/Abstractions/IStatsCollector.cs Normal file
View File

@ -0,0 +1,12 @@
namespace VoidCat.Services.Abstractions;
public interface IStatsCollector
{
ValueTask TrackIngress(Guid id, ulong amount);
ValueTask TrackEgress(Guid id, ulong amount);
ValueTask<Bandwidth> GetBandwidth();
ValueTask<Bandwidth> GetBandwidth(Guid id);
}
public sealed record Bandwidth(ulong Ingress, ulong Egress);

10
VoidCat/Services/Abstractions/IUserManager.cs Normal file
View File

@ -0,0 +1,10 @@
namespace VoidCat.Services.Abstractions;
public interface IUserManager
{
ValueTask<VoidUser> Get(string email, string password);
ValueTask<VoidUser> Get(Guid id);
ValueTask Set(VoidUser user);
}
public sealed record VoidUser(Guid Id, string Email, string PasswordHash);

12
VoidCat/Services/IFileMetadataStore.cs
View File

@ -1,12 +0,0 @@
using VoidCat.Model;
namespace VoidCat.Services;
public interface IFileMetadataStore
{
Task<InternalVoidFile?> Get(Guid id);
Task Set(InternalVoidFile meta);
Task Update(VoidFile patch, Guid editSecret);
}

48
VoidCat/Services/IFileStore.cs
View File

@ -1,48 +0,0 @@
using VoidCat.Model;
namespace VoidCat.Services
{
public interface IFileStore
{
Task<VoidFile?> Get(Guid id);
Task<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts);
Task Egress(EgressRequest request, Stream outStream, CancellationToken cts);
Task UpdateInfo(VoidFile patch, Guid editSecret);
IAsyncEnumerable<VoidFile> ListFiles();
}
public record IngressPayload(Stream InStream, VoidFileMeta Meta, string Hash)
{
public Guid? Id { get; init; }
public Guid? EditSecret { get; init; }
public bool IsAppend => Id.HasValue && EditSecret.HasValue;
}
public record EgressRequest(Guid Id, IEnumerable<RangeRequest> Ranges)
{
}
public record RangeRequest(long? TotalSize, long? Start, long? End)
{
private const long DefaultBufferSize = 1024L * 512L;
public long? Size
=> Start.HasValue ?
(End ?? Math.Min(TotalSize!.Value, Start.Value + DefaultBufferSize)) - Start.Value : End;
public bool IsForFullFile
=> Start is 0 && !End.HasValue;
/// <summary>
/// Return Content-Range header content for this range
/// </summary>
/// <returns></returns>
public string ToContentRange()
=> $"bytes {Start}-{End ?? (Start + Size - 1L)}/{TotalSize?.ToString() ?? "*"}";
}
}

13
VoidCat/Services/IStatsCollector.cs
View File

@ -1,13 +0,0 @@
namespace VoidCat.Services
{
public interface IStatsCollector
{
ValueTask TrackIngress(Guid id, ulong amount);
ValueTask TrackEgress(Guid id, ulong amount);
ValueTask<Bandwidth> GetBandwidth();
ValueTask<Bandwidth> GetBandwidth(Guid id);
}
public sealed record Bandwidth(ulong Ingress, ulong Egress);
}

92
VoidCat/Services/InMemoryStatsCollector.cs
View File

@ -1,52 +1,52 @@
using Microsoft.Extensions.Caching.Memory;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services
namespace VoidCat.Services;
public class InMemoryStatsCollector : IStatsCollector
{
public class InMemoryStatsCollector : IStatsCollector
private static Guid _global = new Guid("{A98DFDCC-C4E1-4D42-B818-912086FC6157}");
private readonly IMemoryCache _cache;
public InMemoryStatsCollector(IMemoryCache cache)
{
private static Guid _global = new Guid("{A98DFDCC-C4E1-4D42-B818-912086FC6157}");
private readonly IMemoryCache _cache;
public InMemoryStatsCollector(IMemoryCache cache)
{
_cache = cache;
}
public ValueTask TrackIngress(Guid id, ulong amount)
{
Incr(IngressKey(id), amount);
Incr(IngressKey(_global), amount);
return ValueTask.CompletedTask;
}
public ValueTask TrackEgress(Guid id, ulong amount)
{
Incr(EgressKey(id), amount);
Incr(EgressKey(_global), amount);
return ValueTask.CompletedTask;
}
public ValueTask<Bandwidth> GetBandwidth()
=> ValueTask.FromResult(GetBandwidthInternal(_global));
public ValueTask<Bandwidth> GetBandwidth(Guid id)
=> ValueTask.FromResult(GetBandwidthInternal(id));
private Bandwidth GetBandwidthInternal(Guid id)
{
var i = _cache.Get(IngressKey(id)) as ulong?;
var o = _cache.Get(EgressKey(id)) as ulong?;
return new(i ?? 0UL, o ?? 0UL);
}
private void Incr(string k, ulong amount)
{
ulong v;
_cache.TryGetValue(k, out v);
_cache.Set(k, v + amount);
}
private string IngressKey(Guid id) => $"stats:ingress:{id}";
private string EgressKey(Guid id) => $"stats:egress:{id}";
_cache = cache;
}
public ValueTask TrackIngress(Guid id, ulong amount)
{
Incr(IngressKey(id), amount);
Incr(IngressKey(_global), amount);
return ValueTask.CompletedTask;
}
public ValueTask TrackEgress(Guid id, ulong amount)
{
Incr(EgressKey(id), amount);
Incr(EgressKey(_global), amount);
return ValueTask.CompletedTask;
}
public ValueTask<Bandwidth> GetBandwidth()
=> ValueTask.FromResult(GetBandwidthInternal(_global));
public ValueTask<Bandwidth> GetBandwidth(Guid id)
=> ValueTask.FromResult(GetBandwidthInternal(id));
private Bandwidth GetBandwidthInternal(Guid id)
{
var i = _cache.Get(IngressKey(id)) as ulong?;
var o = _cache.Get(EgressKey(id)) as ulong?;
return new(i ?? 0UL, o ?? 0UL);
}
private void Incr(string k, ulong amount)
{
ulong v;
_cache.TryGetValue(k, out v);
_cache.Set(k, v + amount);
}
private string IngressKey(Guid id) => $"stats:ingress:{id}";
private string EgressKey(Guid id) => $"stats:egress:{id}";
}

11
VoidCat/Services/LocalDiskFileMetadataStore.cs
View File

@ -1,6 +1,7 @@
using Newtonsoft.Json;
using VoidCat.Model;
using VoidCat.Model.Exceptions;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services;
@ -20,23 +21,23 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
}
}
public async Task<InternalVoidFile?> Get(Guid id)
public async ValueTask<InternalVoidFile?> Get(Guid id)
{
var path = MapMeta(id);
if (!File.Exists(path)) throw new VoidFileNotFoundException(id);
if (!File.Exists(path)) return default;
var json = await File.ReadAllTextAsync(path);
return JsonConvert.DeserializeObject<InternalVoidFile>(json);
}
public Task Set(InternalVoidFile meta)
public async ValueTask Set(InternalVoidFile meta)
{
var path = MapMeta(meta.Id);
var json = JsonConvert.SerializeObject(meta);
return File.WriteAllTextAsync(path, json);
await File.WriteAllTextAsync(path, json);
}
public async Task Update(VoidFile patch, Guid editSecret)
public async ValueTask Update(VoidFile patch, Guid editSecret)
{
var oldMeta = await Get(patch.Id);
if (oldMeta?.EditSecret != editSecret)

24
VoidCat/Services/LocalDiskFileStorage.cs
View File

@ -2,16 +2,17 @@ using System.Buffers;
using System.Security.Cryptography;
using VoidCat.Model;
using VoidCat.Model.Exceptions;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services;
public class LocalDiskFileIngressFactory : IFileStore
public class LocalDiskFileStore : IFileStore
{
private readonly VoidSettings _settings;
private readonly IStatsCollector _stats;
private readonly IFileMetadataStore _metadataStore;
public LocalDiskFileIngressFactory(VoidSettings settings, IStatsCollector stats,
public LocalDiskFileStore(VoidSettings settings, IStatsCollector stats,
IFileMetadataStore metadataStore)
{
_settings = settings;
@ -24,12 +25,12 @@ public class LocalDiskFileIngressFactory : IFileStore
}
}
public async Task<VoidFile?> Get(Guid id)
public async ValueTask<VoidFile?> Get(Guid id)
{
return await _metadataStore.Get(id);
}
public async Task Egress(EgressRequest request, Stream outStream, CancellationToken cts)
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{
var path = MapPath(request.Id);
if (!File.Exists(path)) throw new VoidFileNotFoundException(request.Id);
@ -45,7 +46,7 @@ public class LocalDiskFileIngressFactory : IFileStore
}
}
public async Task<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
public async ValueTask<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
var id = payload.Id ?? Guid.NewGuid();
var fPath = MapPath(id);
@ -69,6 +70,7 @@ public class LocalDiskFileIngressFactory : IFileStore
{
throw new CryptographicException("Invalid file hash");
}
if (payload.IsAppend)
{
vf = vf! with
@ -93,7 +95,7 @@ public class LocalDiskFileIngressFactory : IFileStore
return vf;
}
public Task UpdateInfo(VoidFile patch, Guid editSecret)
public ValueTask UpdateInfo(VoidFile patch, Guid editSecret)
{
return _metadataStore.Update(patch, editSecret);
}
@ -123,9 +125,9 @@ public class LocalDiskFileIngressFactory : IFileStore
{
var buf = buffer.Memory[..readLength];
await fs.WriteAsync(buf, cts);
await _stats.TrackIngress(id, (ulong)readLength);
await _stats.TrackIngress(id, (ulong) readLength);
sha.TransformBlock(buf.ToArray(), 0, buf.Length, null, 0);
total += (ulong)readLength;
total += (ulong) readLength;
}
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
@ -140,7 +142,7 @@ public class LocalDiskFileIngressFactory : IFileStore
while ((readLength = await fileStream.ReadAsync(buffer.Memory, cts)) > 0)
{
await outStream.WriteAsync(buffer.Memory[..readLength], cts);
await _stats.TrackEgress(id, (ulong)readLength);
await _stats.TrackEgress(id, (ulong) readLength);
await outStream.FlushAsync(cts);
}
}
@ -160,8 +162,8 @@ public class LocalDiskFileIngressFactory : IFileStore
&& dataRemaining > 0)
{
var toWrite = Math.Min(readLength, dataRemaining);
await outStream.WriteAsync(buffer.Memory[..(int)toWrite], cts);
await _stats.TrackEgress(id, (ulong)toWrite);
await outStream.WriteAsync(buffer.Memory[..(int) toWrite], cts);
await _stats.TrackEgress(id, (ulong) toWrite);
dataRemaining -= toWrite;
await outStream.FlushAsync(cts);
}

5
VoidCat/Services/PrometheusStatsCollector.cs
View File

@ -1,14 +1,15 @@
using Prometheus;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services;
public class PrometheusStatsCollector : IStatsCollector
{
private readonly Counter _egress =
Metrics.CreateCounter("egress", "Outgoing traffic from the site", new[] {"file"});
Metrics.CreateCounter("egress", "Outgoing traffic from the site", "file");
private readonly Counter _ingress =
Metrics.CreateCounter("ingress", "Incoming traffic to the site", new[] {"file"});
Metrics.CreateCounter("ingress", "Incoming traffic to the site", "file");
public ValueTask TrackIngress(Guid id, ulong amount)
{

1
VoidCat/VoidCat.csproj
View File

@ -11,6 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
<PackageReference Include="NBitcoin" Version="6.0.19" />

13
VoidCat/spa/src/FileUpload.js
View File

@ -98,11 +98,11 @@ export function FileUpload(props) {
}
};
req.upload.onprogress = handleProgress;
req.open("POST", typeof(id) === "string" ? `/upload/${id}` : "/upload");
req.open("POST", typeof (id) === "string" ? `/upload/${id}` : "/upload");
req.setRequestHeader("Content-Type", props.file.type);
req.setRequestHeader("X-Filename", props.file.name);
req.setRequestHeader("X-Digest", buf2hex(digest));
if (typeof(editSecret) === "string") {
if (typeof (editSecret) === "string") {
req.setRequestHeader("X-EditSecret", editSecret);
}
req.send(segment);
@ -113,19 +113,20 @@ export function FileUpload(props) {
}
async function doXHRUpload() {
const UploadSize = 100_000_000;
// upload file in segments of 100MB
// upload file in segments of 50MB
const UploadSize = 50_000_000;
let xhr = null;
const segments = props.file.size / UploadSize;
for (let s = 0; s < segments; s++) {
let offset = s * UploadSize;
let slice = props.file.slice(offset, offset + UploadSize, props.file.type);
xhr = await xhrSegment(await slice.arrayBuffer(), xhr?.file?.id, xhr?.file?.editSecret);
if(!xhr.ok) {
if (!xhr.ok) {
break;
}
}
if(xhr.ok) {
if (xhr.ok) {
setUState(UploadState.Done);
setResult(xhr.file);
} else {

2
VoidCat/spa/src/GlobalStats.css
View File

@ -1,5 +1,5 @@
.stats {
display: grid;
grid-auto-flow: column;
margin: 0 100px;
margin: 0 30px;
}

2
VoidCat/spa/src/GlobalStats.js
View File

@ -21,6 +21,8 @@ export function GlobalStats(props) {
<div>{FormatBytes(stats?.bandwidth?.ingress ?? 0)}</div>
<div>Egress:</div>
<div>{FormatBytes(stats?.bandwidth?.egress ?? 0)}</div>
<div>Storage:</div>
<div>{FormatBytes(stats?.totalBytes ?? 0)}</div>
</div>
);
}