godzhigella/void.cat
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

Client side encryption completed

Browse Source
This commit is contained in:
Kieran 2022-09-11 20:07:38 +01:00
parent d0a92fa115
commit 78ced7f4f3
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
26 changed files with 425 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VoidCat/Controllers/Admin/AdminController.cs
View File

@ -36,7 +36,7 @@ public class AdminController : Controller
[Route("file")]
public async Task<RenderedResults<PublicVoidFile>> ListFiles([FromBody] PagedRequest request)
{
var files = await _fileMetadata.ListFiles<VoidFileMeta>(request);
var files = await _fileMetadata.ListFiles<FileMeta>(request);
return new()
{

2
VoidCat/Controllers/IndexController.cs
View File

@ -47,7 +47,7 @@ public class IndexController : Controller
public class IndexModel
{
public VoidFileMeta? Meta { get; init; }
public FileMeta? Meta { get; init; }
public AssetManifest Manifest { get; init; }
}

13
VoidCat/Controllers/UploadController.cs
View File

@ -81,14 +81,15 @@ namespace VoidCat.Controllers
}
}
var meta = new SecretVoidFileMeta
var meta = new SecretFileMeta
{
MimeType = mime,
Name = filename,
Description = Request.Headers.GetHeader("V-Description"),
Digest = Request.Headers.GetHeader("V-Full-Digest"),
Size = (ulong?) Request.ContentLength ?? 0UL,
Storage = store
Storage = store,
EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams")
};
var (segment, totalSegments) = ParseSegmentsHeader();
@ -142,7 +143,7 @@ namespace VoidCat.Controllers
try
{
var gid = id.FromBase58Guid();
var meta = await _metadata.Get<SecretVoidFileMeta>(gid);
var meta = await _metadata.Get<SecretFileMeta>(gid);
if (meta == default) return UploadResult.Error("File not found");
// Parse V-Segment header
@ -250,7 +251,7 @@ namespace VoidCat.Controllers
public async Task<IActionResult> SetPaymentConfig([FromRoute] string id, [FromBody] SetPaymentConfigRequest req)
{
var gid = id.FromBase58Guid();
var meta = await _metadata.Get<SecretVoidFileMeta>(gid);
var meta = await _metadata.Get<SecretFileMeta>(gid);
if (meta == default) return NotFound();
if (!meta.CanEdit(req.EditSecret)) return Unauthorized();
@ -283,10 +284,10 @@ namespace VoidCat.Controllers
/// </remarks>
[HttpPost]
[Route("{id}/meta")]
public async Task<IActionResult> UpdateFileMeta([FromRoute] string id, [FromBody] SecretVoidFileMeta fileMeta)
public async Task<IActionResult> UpdateFileMeta([FromRoute] string id, [FromBody] SecretFileMeta fileMeta)
{
var gid = id.FromBase58Guid();
var meta = await _metadata.Get<SecretVoidFileMeta>(gid);
var meta = await _metadata.Get<SecretFileMeta>(gid);
if (meta == default) return NotFound();
if (!meta.CanEdit(fileMeta.EditSecret)) return Unauthorized();

17
VoidCat/Model/Extensions.cs
View File

@ -78,7 +78,7 @@ public static class Extensions
return !string.IsNullOrEmpty(h.Value.ToString()) ? h.Value.ToString() : default;
}
public static bool CanEdit(this SecretVoidFileMeta file, Guid? editSecret)
public static bool CanEdit(this SecretFileMeta file, Guid? editSecret)
{
return file.EditSecret == editSecret;
}
@ -236,6 +236,21 @@ public static class Extensions
return vu.Password == password.Hash(hashParts[0], hashParts.Length == 3 ? hashParts[1] : null);
}
/// <summary>
/// Patch metadata
/// </summary>
/// <param name="oldMeta"></param>
/// <param name="meta"></param>
public static void Patch(this FileMeta oldMeta, FileMeta meta)
{
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.EncryptionParams = meta.EncryptionParams ?? oldMeta.EncryptionParams;
}
public static bool HasPostgres(this VoidSettings settings)
=> !string.IsNullOrEmpty(settings.Postgres);

2
VoidCat/Model/IngressPayload.cs
View File

@ -1,6 +1,6 @@
namespace VoidCat.Model;
public sealed record IngressPayload(Stream InStream, SecretVoidFileMeta Meta, int Segment, int TotalSegments)
public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments)
{
public Guid Id { get; init; } = Guid.NewGuid();
public Guid? EditSecret { get; init; }

6
VoidCat/Model/VoidFile.cs
View File

@ -4,7 +4,7 @@ using VoidCat.Model.User;
namespace VoidCat.Model
{
public abstract record VoidFile<TMeta> where TMeta : VoidFileMeta
public abstract record VoidFile<TMeta> where TMeta : FileMeta
{
/// <summary>
/// Id of the file
@ -38,11 +38,11 @@ namespace VoidCat.Model
public VirusScanResult? VirusScan { get; init; }
}
public sealed record PublicVoidFile : VoidFile<VoidFileMeta>
public sealed record PublicVoidFile : VoidFile<FileMeta>
{
}
public sealed record PrivateVoidFile : VoidFile<SecretVoidFileMeta>
public sealed record PrivateVoidFile : VoidFile<SecretFileMeta>
{
}
}

13
VoidCat/Model/VoidFileMeta.cs
View File

@ -8,7 +8,7 @@ namespace VoidCat.Model;
/// <summary>
/// Base metadata must contain version number
/// </summary>
public interface IVoidFileMeta
public interface IFileMeta
{
const int CurrentVersion = 3;
@ -18,12 +18,12 @@ public interface IVoidFileMeta
/// <summary>
/// File metadata which is managed by <see cref="IFileMetadataStore"/>
/// </summary>
public record VoidFileMeta : IVoidFileMeta
public record FileMeta : IFileMeta
{
/// <summary>
/// Metadata version
/// </summary>
public int Version { get; init; } = IVoidFileMeta.CurrentVersion;
public int Version { get; init; } = IFileMeta.CurrentVersion;
/// <summary>
/// Internal Id of the file
@ -75,12 +75,17 @@ public record VoidFileMeta : IVoidFileMeta
/// What storage system the file is on
/// </summary>
public string? Storage { get; set; }
/// <summary>
/// Encryption params as JSON string
/// </summary>
public string? EncryptionParams { get; set; }
}
/// <summary>
/// <see cref="VoidFile"/> with attached <see cref="EditSecret"/>
/// </summary>
public record SecretVoidFileMeta : VoidFileMeta
public record SecretFileMeta : FileMeta
{
/// <summary>
/// A secret key used to make edits to the file after its uploaded

10
VoidCat/Services/Abstractions/IFileMetadataStore.cs
View File

@ -5,7 +5,7 @@ namespace VoidCat.Services.Abstractions;
/// <summary>
/// File metadata contains all data about a file except for the file data itself
/// </summary>
public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta>
public interface IFileMetadataStore : IPublicPrivateStore<FileMeta, SecretFileMeta>
{
/// <summary>
/// Get metadata for a single file
@ -13,7 +13,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="id"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta;
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta;
/// <summary>
/// Get metadata for multiple files
@ -21,7 +21,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="ids"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta;
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta;
/// <summary>
/// Update file metadata
@ -30,7 +30,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="meta"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta;
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta;
/// <summary>
/// List all files in the store
@ -38,7 +38,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="request"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta;
ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta;
/// <summary>
/// Returns basic stats about the file store

2
VoidCat/Services/Background/DeleteExpiredFiles.cs
View File

@ -27,7 +27,7 @@ public sealed class DeleteExpiredFiles : BackgroundService
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
var files = await metadata.ListFiles<SecretFileMeta>(new(0, int.MaxValue));
await foreach (var f in files.Results.WithCancellation(stoppingToken))
{
try

2
VoidCat/Services/Background/VirusScannerService.cs
View File

@ -29,7 +29,7 @@ public class VirusScannerService : BackgroundService
var page = 0;
while (true)
{
var files = await _fileStore.ListFiles<VoidFileMeta>(new(page, 1_000));
var files = await _fileStore.ListFiles<FileMeta>(new(page, 1_000));
if (files.Pages < page) break;
page++;

6
VoidCat/Services/Files/FileInfoManager.cs
View File

@ -35,7 +35,7 @@ public sealed class FileInfoManager
/// <returns></returns>
public ValueTask<PublicVoidFile?> Get(Guid id)
{
return Get<PublicVoidFile, VoidFileMeta>(id);
return Get<PublicVoidFile, FileMeta>(id);
}
/// <summary>
@ -45,7 +45,7 @@ public sealed class FileInfoManager
/// <returns></returns>
public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
{
return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
return Get<PrivateVoidFile, SecretFileMeta>(id);
}
/// <summary>
@ -82,7 +82,7 @@ public sealed class FileInfoManager
}
private async ValueTask<TFile?> Get<TFile, TMeta>(Guid id)
where TMeta : VoidFileMeta where TFile : VoidFile<TMeta>, new()
where TMeta : FileMeta where TFile : VoidFile<TMeta>, new()
{
var meta = _metadataStore.Get<TMeta>(id);
var payment = _paymentStore.Get(id);

29
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@ -22,13 +22,13 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
return GetMeta<TMeta>(id);
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
@ -44,22 +44,17 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await Get<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
async IAsyncEnumerable<TMeta> EnumerateFiles()
{
@ -102,26 +97,26 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
return GetMeta<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
return GetMeta<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
public async ValueTask Set(Guid id, SecretFileMeta meta)
{
var path = MapMeta(id);
var json = JsonConvert.SerializeObject(meta);

37
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@ -18,32 +18,33 @@ public class PostgresFileMetadataStore : IFileMetadataStore
public string? Key => "postgres";
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return Get<VoidFileMeta>(id);
return Get<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return Get<SecretVoidFileMeta>(id);
return Get<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta obj)
public async ValueTask Set(Guid id, SecretFileMeta obj)
{
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"insert into
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store)
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"", ""EncryptionParams"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store, :encryptionParams)
on conflict (""Id"") do update set
""Name"" = :name,
""Size"" = :size,
""Description"" = :description,
""MimeType"" = :mimeType,
""Expires"" = :expires,
""Storage"" = :store",
""Storage"" = :store,
""EncryptionParams"" = :encryptionParams",
new
{
id,
@ -55,7 +56,8 @@ on conflict (""Id"") do update set
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires?.ToUniversalTime(),
store = obj.Storage
store = obj.Storage,
encryptionParams = obj.EncryptionParams
});
}
@ -67,7 +69,7 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
return await conn.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
@ -75,7 +77,7 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
var ret = await conn.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
@ -83,22 +85,17 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await Get<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
var count = await conn.ExecuteScalarAsync<int>(@"select count(*) from ""Files""");

31
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@ -23,13 +23,13 @@ public class S3FileMetadataStore : IFileMetadataStore
public string? Key => _config.Name;
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
return GetMeta<TMeta>(id);
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
@ -45,22 +45,17 @@ public class S3FileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
async IAsyncEnumerable<TMeta> Enumerate()
{
@ -95,26 +90,26 @@ public class S3FileMetadataStore : IFileMetadataStore
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
return GetMeta<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
return GetMeta<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
public async ValueTask Set(Guid id, SecretFileMeta meta)
{
await _client.PutObjectAsync(new()
{
@ -131,7 +126,7 @@ public class S3FileMetadataStore : IFileMetadataStore
await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
}
private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : VoidFileMeta
private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : FileMeta
{
try
{

20
VoidCat/Services/Migrations/Database/06-EncryptionParams.cs Normal file
View File

@ -0,0 +1,20 @@
using FluentMigrator;
namespace VoidCat.Services.Migrations.Database;
[Migration(20220911_1635)]
public class EncryptionParams : Migration{
public override void Up()
{
Create.Column("EncryptionParams")
.OnTable("Files")
.AsString()
.Nullable();
}
public override void Down()
{
Delete.Column("EncryptionParams")
.FromTable("Files");
}
}

2
VoidCat/Services/Migrations/FixSize.cs
View File

@ -24,7 +24,7 @@ public class FixSize : IMigration
/// <inheritdoc />
public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{
var files = await _fileMetadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
var files = await _fileMetadata.ListFiles<SecretFileMeta>(new(0, int.MaxValue));
await foreach (var file in files.Results)
{
try

4
VoidCat/Services/Migrations/MigrateToPostgres.cs
View File

@ -102,7 +102,7 @@ public class MigrateToPostgres : IMigration
{
var cachePaywallStore = new CachePaymentStore(_cache);
var files = await _fileMetadata.ListFiles<VoidFileMeta>(new(0, int.MaxValue));
var files = await _fileMetadata.ListFiles<FileMeta>(new(0, int.MaxValue));
await foreach (var file in files.Results)
{
try
@ -161,7 +161,7 @@ public class MigrateToPostgres : IMigration
public string? Password { get; set; }
}
private record UploaderSecretVoidFileMeta : SecretVoidFileMeta
private record UploaderSecretVoidFileMeta : SecretFileMeta
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid? Uploader { get; set; }

2
VoidCat/Services/Migrations/PopulateMetadataId.cs
View File

@ -24,7 +24,7 @@ public class PopulateMetadataId : IMigration
return IMigration.MigrationResult.Skipped;
}
var files = await _metadataStore.ListFiles<SecretVoidFileMeta>(new(0, Int32.MaxValue));
var files = await _metadataStore.ListFiles<SecretFileMeta>(new(0, Int32.MaxValue));
await foreach (var file in files.Results)
{
// read-write file metadata

2
VoidCat/Services/VirusScanner/CacheVirusScanStore.cs
View File

@ -23,7 +23,7 @@ public class CacheVirusScanStore : BasicCacheStore<VirusScanResult>, IVirusScanS
var scans = await _cache.GetList(MapFilesKey(id));
if (scans.Length > 0)
{
return await Get(Guid.Parse(scans.First()));
return await Get(Guid.Parse(scans.Last()));
}
return default;

2
VoidCat/VoidCat.csproj
View File

@ -10,7 +10,7 @@
<HostSPA>True</HostSPA>
<DefineConstants Condition="'$(HostSPA)' == 'True'">$(DefineConstants);HostSPA</DefineConstants>
<DocumentationFile>$(AssemblyName).xml</DocumentationFile>
<Version>4.1.0</Version>
<Version>4.2.0</Version>
</PropertyGroup>
<ItemGroup>

143
VoidCat/spa/src/Components/FileUpload/FileUpload.js
View File

@ -1,13 +1,12 @@
import "./FileUpload.css";
import {useEffect, useState} from "react";
import * as CryptoJS from 'crypto-js';
import {useSelector} from "react-redux";
import sjcl from "sjcl";
import {sjclcodec} from "../../codecBytes";
import {ConstName, FormatBytes} from "../Shared/Util";
import {RateCalculator} from "../Shared/RateCalculator";
import {buf2hex, ConstName, FormatBytes} from "../Shared/Util";
import {ApiHost} from "../Shared/Const";
import {StreamEncryption} from "../Shared/StreamEncryption";
import {VoidButton} from "../Shared/VoidButton";
import {useFileTransfer} from "../Shared/FileTransferHook";
const UploadState = {
NotStarted: 0,
@ -20,45 +19,28 @@ const UploadState = {
};
export const DigestAlgo = "SHA-256";
const BlockSize = 16;
export function FileUpload(props) {
const auth = useSelector(state => state.login.jwt);
const info = useSelector(state => state.info.info);
const [speed, setSpeed] = useState(0);
const [progress, setProgress] = useState(0);
const {speed, progress, loaded, setFileSize, reset, update} = useFileTransfer();
const [result, setResult] = useState();
const [uState, setUState] = useState(UploadState.NotStarted);
const [challenge, setChallenge] = useState();
const [encryptionKey, setEncryptionKey] = useState();
const calc = new RateCalculator();
const [encrypt, setEncrypt] = useState(true);
function handleProgress(e) {
if (e instanceof ProgressEvent) {
let newProgress = e.loaded / e.total;
calc.ReportLoaded(e.loaded);
setSpeed(calc.RateWindow(5));
setProgress(newProgress);
loaded(e.loaded);
}
}
function generateEncryptionKey() {
let key = {
key: sjclcodec.toBits(window.crypto.getRandomValues(new Uint8Array(16))),
iv: sjclcodec.toBits(window.crypto.getRandomValues(new Uint8Array(12)))
};
setEncryptionKey(key);
return key;
}
async function doStreamUpload() {
let key = generateEncryptionKey();
let aes = new sjcl.cipher.aes(key.key);
setFileSize(props.file.size);
setUState(UploadState.Hashing);
let hash = await digest(props.file);
calc.Reset();
reset();
let offset = 0;
async function readChunk(size) {
@ -72,36 +54,23 @@ export function FileUpload(props) {
return new Uint8Array(data);
}
async function readEncryptedChunk(size) {
if (offset >= props.file.size) {
return new Uint8Array(0);
}
size -= size % BlockSize;
let end = Math.min(offset + size, props.file.size);
let blob = props.file.slice(offset, end, props.file.type);
let data = new Uint8Array(await blob.arrayBuffer());
offset += data.byteLength;
let encryptedData = sjcl.mode.gcm.encrypt(aes, sjclcodec.toBits(data), key.iv);
return new Uint8Array(sjclcodec.fromBits(encryptedData));
}
let rs = new ReadableStream({
start: () => {
start: async () => {
setUState(UploadState.Uploading);
},
pull: async (controller) => {
let chunkSize = controller.desiredSize;
let chunk = key ? await readEncryptedChunk(chunkSize) : await readChunk(chunkSize);
if (chunk.byteLength === 0) {
controller.close();
return;
try {
let chunk = await readChunk(controller.desiredSize);
if (chunk.byteLength === 0) {
controller.close();
return;
}
update(chunk.length);
controller.enqueue(chunk);
} catch (e) {
console.error(e);
throw e;
}
calc.ReportProgress(chunk.byteLength);
setSpeed(calc.RateWindow(5));
setProgress(offset / props.file.size);
controller.enqueue(chunk);
},
cancel: (reason) => {
console.log(reason);
@ -111,16 +80,28 @@ export function FileUpload(props) {
highWaterMark: 1024 * 1024
});
let req = await fetch("/upload", {
let enc = encrypt ? (() => {
let ret = new StreamEncryption();
setEncryptionKey(ret.getKey());
return ret;
})() : null;
rs = encrypt ? rs.pipeThrough(enc.getEncryptionTransform()) : rs;
let headers = {
"Content-Type": "application/octet-stream",
"V-Content-Type": props.file.type,
"V-Filename": props.file.name,
"V-Full-Digest": hash
};
if (encrypt) {
headers["V-EncryptionParams"] = JSON.stringify(enc.getParams());
}
let req = await fetch("https://localhost:7195/upload", {
method: "POST",
mode: "cors",
body: rs,
headers: {
"Content-Type": "application/octet-stream",
"V-Content-Type": props.file.type,
"V-Filename": props.file.name,
"V-Full-Digest": hash
},
headers,
duplex: 'half'
});
@ -183,11 +164,12 @@ export function FileUpload(props) {
}
async function doXHRUpload() {
setFileSize(props.file.size);
let uploadSize = info.uploadSegmentSize ?? Number.MAX_VALUE;
setUState(UploadState.Hashing);
let hash = await digest(props.file);
calc.Reset();
reset();
if (props.file.size >= uploadSize) {
await doSplitXHRUpload(hash, uploadSize);
} else {
@ -198,10 +180,9 @@ export function FileUpload(props) {
async function doSplitXHRUpload(hash, splitSize) {
let xhr = null;
setProgress(0);
const segments = Math.ceil(props.file.size / splitSize);
for (let s = 0; s < segments; s++) {
calc.Reset();
reset();
let offset = s * splitSize;
let slice = props.file.slice(offset, offset + splitSize, props.file.type);
xhr = await xhrSegment(slice, hash, xhr?.file?.id, xhr?.file?.metadata?.editSecret, s + 1, segments);
@ -229,30 +210,36 @@ export function FileUpload(props) {
}
async function digest(file) {
const chunkSize = 100_000_000;
let sha = CryptoJS.algo.SHA256.create();
for (let x = 0; x < Math.ceil(file.size / chunkSize); x++) {
let offset = x * chunkSize;
let slice = file.slice(offset, offset + chunkSize, file.type);
let data = Uint32Array.from(await slice.arrayBuffer());
sha.update(new CryptoJS.lib.WordArray.init(data, slice.length));
calc.ReportLoaded(offset);
setSpeed(calc.RateWindow(5));
setProgress(offset / parseFloat(file.size));
}
return sha.finalize().toString();
let h = await window.crypto.subtle.digest(DigestAlgo, await file.arrayBuffer());
return buf2hex(new Uint8Array(h));
}
function renderStatus() {
if (result) {
let link = encryptionKey ? `/${result.id}#${sjcl.codec.hex.fromBits(encryptionKey.key)}:${sjcl.codec.hex.fromBits(encryptionKey.iv)}` : `/${result.id}`;
let link = `/${result.id}`;
return uState === UploadState.Done ?
<dl>
<dt>Link:</dt>
<dd><a target="_blank" href={link}>{result.id}</a></dd>
{encryptionKey ? <>
<dt>Encryption Key:</dt>
<dd>
<VoidButton onClick={() => navigator.clipboard.writeText(encryptionKey)}>Copy</VoidButton>
</dd>
</> : null}
</dl>
: <b>{result}</b>;
} else if (uState === UploadState.NotStarted) {
return (
<>
<dl>
<dt>Encrypt file:</dt>
<dd><input type="checkbox" checked={encrypt} onChange={(e) => setEncrypt(e.target.checked)}/>
</dd>
</dl>
<VoidButton onClick={() => doStreamUpload()}>Upload</VoidButton>
</>
)
} else {
return (
<dl>
@ -274,11 +261,9 @@ export function FileUpload(props) {
}
useEffect(() => {
console.log(props.file);
let chromeVersion = getChromeVersion();
if (chromeVersion >= 105) {
doStreamUpload().catch(console.error);
//doStreamUpload().catch(console.error);
} else {
doXHRUpload().catch(console.error);
}

26
VoidCat/spa/src/Components/Shared/FileTransferHook.js Normal file
View File

@ -0,0 +1,26 @@
import {useState} from "react";
import {RateCalculator} from "./RateCalculator";
export function useFileTransfer() {
const [speed, setSpeed] = useState(0);
const [progress, setProgress] = useState(0);
const calc = new RateCalculator();
return {
speed, progress,
setFileSize: (size) => {
calc.SetFileSize(size);
},
update: (bytes) => {
calc.ReportProgress(bytes);
setSpeed(calc.GetSpeed());
setProgress(calc.GetProgress());
},
loaded: (loaded) => {
calc.ReportLoaded(loaded);
setSpeed(calc.GetSpeed());
setProgress(calc.GetProgress());
},
reset: () => calc.Reset()
}
}

23
VoidCat/spa/src/Components/Shared/RateCalculator.js
View File

@ -1,12 +1,26 @@
export class RateCalculator {
constructor() {
this.reports = [];
this.lastLoaded = 0;
this.Reset();
this.fileSize = 0;
}
SetFileSize(size) {
this.fileSize = size;
}
GetProgress() {
return this.progress;
}
GetSpeed() {
return this.speed;
}
Reset() {
this.reports = [];
this.lastLoaded = 0;
this.progress = 0;
this.speed = 0;
}
ReportProgress(amount) {
@ -14,6 +28,9 @@ export class RateCalculator {
time: new Date().getTime(),
amount
});
this.lastLoaded += amount;
this.progress = this.lastLoaded / parseFloat(this.fileSize);
this.speed = this.RateWindow(5);
}
ReportLoaded(loaded) {
@ -22,6 +39,8 @@ export class RateCalculator {
amount: loaded - this.lastLoaded
});
this.lastLoaded = loaded;
this.progress = this.lastLoaded / parseFloat(this.fileSize);
this.speed = this.RateWindow(5);
}
RateWindow(s) {

110
VoidCat/spa/src/Components/Shared/StreamEncryption.js Normal file
View File

@ -0,0 +1,110 @@
import {sjclcodec} from "../../codecBytes";
import sjcl from "sjcl";
import {buf2hex} from "./Util";
/**
* AES-GCM TransformStream
*/
export class StreamEncryption {
constructor(key, iv, params) {
if (key === undefined && iv === undefined) {
key = buf2hex(window.crypto.getRandomValues(new Uint8Array(16)));
iv = buf2hex(window.crypto.getRandomValues(new Uint8Array(12)));
}
if (typeof key === "string" && typeof iv === "string") {
key = sjcl.codec.hex.toBits(key);
iv = sjcl.codec.hex.toBits(iv);
} else if (!Array.isArray(key) || !Array.isArray(iv)) {
throw "Key and IV must be hex string or bitArray";
}
if (typeof params === "string") {
params = JSON.parse(params);
}
this.TagSize = params?.ts ?? 128;
this.ChunkSize = params?.cs ?? (1024 * 1024 * 10);
this.aes = new sjcl.cipher.aes(key);
this.key = key;
this.iv = iv;
console.log(`ts=${this.TagSize}, cs=${this.ChunkSize}, key=${key}, iv=${this.iv}`);
}
/**
* Return formatted encryption key
* @returns {string}
*/
getKey() {
return `${sjcl.codec.hex.fromBits(this.key)}:${sjcl.codec.hex.fromBits(this.iv)}`;
}
/**
* Get encryption params
* @returns {{cs: (*|number), ts: number}}
*/
getParams() {
return {
ts: this.TagSize,
cs: this.ChunkSize
}
}
/**
* Get encryption TransformStream
* @returns {TransformStream<any, any>}
*/
getEncryptionTransform() {
return this._getCryptoStream(0);
}
/**
* Get decryption TransformStream
* @returns {TransformStream<any, any>}
*/
getDecryptionTransform() {
return this._getCryptoStream(1);
}
_getCryptoStream(mode) {
let offset = 0;
let buffer = new Uint8Array(this.ChunkSize + (mode === 1 ? this.TagSize / 8 : 0));
return new window.TransformStream({
transform: async (chunk, controller) => {
chunk = await chunk;
try {
let toBuffer = Math.min(chunk.byteLength, buffer.byteLength - offset);
buffer.set(chunk.slice(0, toBuffer), offset);
offset += toBuffer;
if (offset === buffer.byteLength) {
let buff = sjclcodec.toBits(buffer);
let encryptedBuf = sjclcodec.fromBits(
mode === 0 ?
sjcl.mode.gcm.encrypt(this.aes, buff, this.iv, [], this.TagSize) :
sjcl.mode.gcm.decrypt(this.aes, buff, this.iv, [], this.TagSize)
);
controller.enqueue(new Uint8Array(encryptedBuf));
offset = chunk.byteLength - toBuffer;
buffer.set(chunk.slice(toBuffer));
}
} catch (e) {
console.error(e);
throw e;
}
},
flush: (controller) => {
let lastBuffer = buffer.slice(0, offset);
let buff = sjclcodec.toBits(lastBuffer);
let encryptedBuf = sjclcodec.fromBits(
mode === 0 ?
sjcl.mode.gcm.encrypt(this.aes, buff, this.iv, [], this.TagSize) :
sjcl.mode.gcm.decrypt(this.aes, buff, this.iv, [], this.TagSize)
);
controller.enqueue(new Uint8Array(encryptedBuf));
}
}, {
highWaterMark: this.ChunkSize
});
}
}

11
VoidCat/spa/src/Pages/FilePreview.css
View File

@ -26,4 +26,15 @@
border-radius: 10px;
border: 1px solid red;
margin-bottom: 5px;
}
.preview .encrypted {
padding: 10px;
border-radius: 10px;
border: 2px solid #bbbbbb;
text-align: center;
}
.error {
color: red;
}

118
VoidCat/spa/src/Pages/FilePreview.js
View File

@ -1,17 +1,19 @@
import "./FilePreview.css";
import {Fragment, useEffect, useState} from "react";
import {useParams} from "react-router-dom";
import {TextPreview} from "../Components/FilePreview/TextPreview";
import FeatherIcon from "feather-icons-react";
import {Helmet} from "react-helmet";
import {TextPreview} from "../Components/FilePreview/TextPreview";
import {FileEdit} from "../Components/FileEdit/FileEdit";
import {FilePayment} from "../Components/FilePreview/FilePayment";
import {useApi} from "../Components/Shared/Api";
import {Helmet} from "react-helmet";
import {FormatBytes} from "../Components/Shared/Util";
import {ApiHost} from "../Components/Shared/Const";
import {InlineProfile} from "../Components/Shared/InlineProfile";
import sjcl from "sjcl";
import {sjclcodec} from "../codecBytes";
import {StreamEncryption} from "../Components/Shared/StreamEncryption";
import {VoidButton} from "../Components/Shared/VoidButton";
import {useFileTransfer} from "../Components/Shared/FileTransferHook";
export function FilePreview() {
const {Api} = useApi();
@ -19,6 +21,9 @@ export function FilePreview() {
const [info, setInfo] = useState();
const [order, setOrder] = useState();
const [link, setLink] = useState("#");
const [key, setKey] = useState("");
const [error, setError] = useState("");
const {speed, progress, update, setFileSize} = useFileTransfer();
async function loadInfo() {
let req = await Api.fileInfo(params.id);
@ -28,13 +33,74 @@ export function FilePreview() {
}
}
function isFileEncrypted() {
return "string" === typeof info?.metadata?.encryptionParams
}
function isDecrypted() {
return link.startsWith("blob:");
}
function isPaymentRequired() {
return info?.payment?.required === true && !order;
}
function canAccessFile() {
if (info?.payment?.required === true && !order) {
if (isPaymentRequired()) {
return false;
}
if (isFileEncrypted() && !isDecrypted()) {
return false;
}
return true;
}
async function decryptFile() {
try {
let hashKey = key.match(/([0-9a-z]{32}):([0-9a-z]{24})/);
if (hashKey?.length === 3) {
let [key, iv] = [hashKey[1], hashKey[2]];
let enc = new StreamEncryption(key, iv, info.metadata?.encryptionParams);
let rsp = await fetch(link);
if (rsp.ok) {
let reader = rsp.body
.pipeThrough(enc.getDecryptionTransform())
.pipeThrough(decryptionProgressTransform());
let newResponse = new Response(reader);
setLink(window.URL.createObjectURL(await newResponse.blob(), {type: info.metadata.mimeType}));
}
} else {
setError("Invalid encryption key format");
}
} catch (e) {
setError(e.message);
}
}
function decryptionProgressTransform() {
return new window.TransformStream({
transform: (chunk, controller) => {
update(chunk.length);
controller.enqueue(chunk);
}
});
}
function renderEncryptedDownload() {
if (!isFileEncrypted() || isDecrypted() || isPaymentRequired()) return;
return (
<div className="encrypted">
<h3>This file is encrypted, please enter the encryption key:</h3>
<input type="password" placeholder="Encryption key" value={key}
onChange={(e) => setKey(e.target.value)}/>
<VoidButton onClick={() => decryptFile()}>Decrypt</VoidButton>
{progress > 0 ? `${(100 * progress).toFixed(0)}% (${FormatBytes(speed)}/s)` : null}
{error ? <h4 className="error">{error}</h4> : null}
</div>
);
}
function renderPayment() {
if (info.payment && info.payment.service !== 0) {
if (!order) {
@ -46,6 +112,8 @@ export function FilePreview() {
}
function renderPreview() {
if (!canAccessFile()) return;
if (info.metadata) {
switch (info.metadata.mimeType) {
case "image/avif":
@ -145,41 +213,8 @@ export function FilePreview() {
useEffect(() => {
if (info) {
let fileLink = info.metadata?.url ?? `${ApiHost}/d/${info.id}`;
// detect encrypted file link
let hashKey = window.location.hash.match(/#([0-9a-z]{32}):([0-9a-z]{24})/);
if (hashKey.length === 3) {
let [key, iv] = [sjcl.codec.hex.toBits(hashKey[1]), sjcl.codec.hex.toBits(hashKey[2])];
console.log(key, iv);
let aes = new sjcl.cipher.aes(key);
async function load() {
let decryptStream = new window.TransformStream({
transform: async (chunk, controller) => {
chunk = await chunk;
console.log("Transforming chunk:", chunk);
let buff = sjclcodec.toBits(chunk);
let decryptedBuff = sjclcodec.fromBits(sjcl.mode.gcm.decrypt(aes, buff, iv));
console.log("Decrypted data:", decryptedBuff);
controller.enqueue(new Uint8Array(decryptedBuff));
}
});
let rsp = await fetch(fileLink);
if (rsp.ok) {
let reader = rsp.body
.pipeThrough(decryptStream);
console.log("Pipe reader", reader);
let newResponse = new Response(reader);
setLink(window.URL.createObjectURL(await newResponse.blob(), {type: info.metadata.mimeType}));
}
}
load();
return;
}
setFileSize(info.metadata.size);
let order = window.localStorage.getItem(`payment-${info.id}`);
if (order) {
let orderObj = JSON.parse(order);
@ -213,7 +248,8 @@ export function FilePreview() {
</div>
</div>
{renderPayment()}
{canAccessFile() ? renderPreview() : null}
{renderPreview()}
{renderEncryptedDownload()}
<div className="file-stats">
<div>
<FeatherIcon icon="download-cloud"/>