diff --git a/VoidCat/Controllers/Admin/AdminController.cs b/VoidCat/Controllers/Admin/AdminController.cs index 4d674dc..ca53d58 100644 --- a/VoidCat/Controllers/Admin/AdminController.cs +++ b/VoidCat/Controllers/Admin/AdminController.cs @@ -36,7 +36,7 @@ public class AdminController : Controller [Route("file")] public async Task> ListFiles([FromBody] PagedRequest request) { - var files = await _fileMetadata.ListFiles(request); + var files = await _fileMetadata.ListFiles(request); return new() { diff --git a/VoidCat/Controllers/IndexController.cs b/VoidCat/Controllers/IndexController.cs index 917daa5..e642e84 100644 --- a/VoidCat/Controllers/IndexController.cs +++ b/VoidCat/Controllers/IndexController.cs @@ -47,7 +47,7 @@ public class IndexController : Controller public class IndexModel { - public VoidFileMeta? Meta { get; init; } + public FileMeta? Meta { get; init; } public AssetManifest Manifest { get; init; } } diff --git a/VoidCat/Controllers/UploadController.cs b/VoidCat/Controllers/UploadController.cs index d6be560..fcb26ce 100644 --- a/VoidCat/Controllers/UploadController.cs +++ b/VoidCat/Controllers/UploadController.cs @@ -81,14 +81,15 @@ namespace VoidCat.Controllers } } - var meta = new SecretVoidFileMeta + var meta = new SecretFileMeta { MimeType = mime, Name = filename, Description = Request.Headers.GetHeader("V-Description"), Digest = Request.Headers.GetHeader("V-Full-Digest"), Size = (ulong?) Request.ContentLength ?? 0UL, - Storage = store + Storage = store, + EncryptionParams = Request.Headers.GetHeader("V-EncryptionParams") }; var (segment, totalSegments) = ParseSegmentsHeader(); @@ -142,7 +143,7 @@ namespace VoidCat.Controllers try { var gid = id.FromBase58Guid(); - var meta = await _metadata.Get(gid); + var meta = await _metadata.Get(gid); if (meta == default) return UploadResult.Error("File not found"); // Parse V-Segment header @@ -250,7 +251,7 @@ namespace VoidCat.Controllers public async Task SetPaymentConfig([FromRoute] string id, [FromBody] SetPaymentConfigRequest req) { var gid = id.FromBase58Guid(); - var meta = await _metadata.Get(gid); + var meta = await _metadata.Get(gid); if (meta == default) return NotFound(); if (!meta.CanEdit(req.EditSecret)) return Unauthorized(); @@ -283,10 +284,10 @@ namespace VoidCat.Controllers /// [HttpPost] [Route("{id}/meta")] - public async Task UpdateFileMeta([FromRoute] string id, [FromBody] SecretVoidFileMeta fileMeta) + public async Task UpdateFileMeta([FromRoute] string id, [FromBody] SecretFileMeta fileMeta) { var gid = id.FromBase58Guid(); - var meta = await _metadata.Get(gid); + var meta = await _metadata.Get(gid); if (meta == default) return NotFound(); if (!meta.CanEdit(fileMeta.EditSecret)) return Unauthorized(); diff --git a/VoidCat/Model/Extensions.cs b/VoidCat/Model/Extensions.cs index 7bafb86..7ff252d 100644 --- a/VoidCat/Model/Extensions.cs +++ b/VoidCat/Model/Extensions.cs @@ -78,7 +78,7 @@ public static class Extensions return !string.IsNullOrEmpty(h.Value.ToString()) ? h.Value.ToString() : default; } - public static bool CanEdit(this SecretVoidFileMeta file, Guid? editSecret) + public static bool CanEdit(this SecretFileMeta file, Guid? editSecret) { return file.EditSecret == editSecret; } @@ -236,6 +236,21 @@ public static class Extensions return vu.Password == password.Hash(hashParts[0], hashParts.Length == 3 ? hashParts[1] : null); } + /// + /// Patch metadata + /// + /// + /// + public static void Patch(this FileMeta oldMeta, FileMeta meta) + { + oldMeta.Description = meta.Description ?? oldMeta.Description; + oldMeta.Name = meta.Name ?? oldMeta.Name; + oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; + oldMeta.Storage = meta.Storage ?? oldMeta.Storage; + oldMeta.Expires = meta.Expires; + oldMeta.EncryptionParams = meta.EncryptionParams ?? oldMeta.EncryptionParams; + } + public static bool HasPostgres(this VoidSettings settings) => !string.IsNullOrEmpty(settings.Postgres); diff --git a/VoidCat/Model/IngressPayload.cs b/VoidCat/Model/IngressPayload.cs index 14671a7..8d92153 100644 --- a/VoidCat/Model/IngressPayload.cs +++ b/VoidCat/Model/IngressPayload.cs @@ -1,6 +1,6 @@ namespace VoidCat.Model; -public sealed record IngressPayload(Stream InStream, SecretVoidFileMeta Meta, int Segment, int TotalSegments) +public sealed record IngressPayload(Stream InStream, SecretFileMeta Meta, int Segment, int TotalSegments) { public Guid Id { get; init; } = Guid.NewGuid(); public Guid? EditSecret { get; init; } diff --git a/VoidCat/Model/VoidFile.cs b/VoidCat/Model/VoidFile.cs index d712ce3..c832145 100644 --- a/VoidCat/Model/VoidFile.cs +++ b/VoidCat/Model/VoidFile.cs @@ -4,7 +4,7 @@ using VoidCat.Model.User; namespace VoidCat.Model { - public abstract record VoidFile where TMeta : VoidFileMeta + public abstract record VoidFile where TMeta : FileMeta { /// /// Id of the file @@ -38,11 +38,11 @@ namespace VoidCat.Model public VirusScanResult? VirusScan { get; init; } } - public sealed record PublicVoidFile : VoidFile + public sealed record PublicVoidFile : VoidFile { } - public sealed record PrivateVoidFile : VoidFile + public sealed record PrivateVoidFile : VoidFile { } } \ No newline at end of file diff --git a/VoidCat/Model/VoidFileMeta.cs b/VoidCat/Model/VoidFileMeta.cs index 9baf25e..753cdfd 100644 --- a/VoidCat/Model/VoidFileMeta.cs +++ b/VoidCat/Model/VoidFileMeta.cs @@ -8,7 +8,7 @@ namespace VoidCat.Model; /// /// Base metadata must contain version number /// -public interface IVoidFileMeta +public interface IFileMeta { const int CurrentVersion = 3; @@ -18,12 +18,12 @@ public interface IVoidFileMeta /// /// File metadata which is managed by /// -public record VoidFileMeta : IVoidFileMeta +public record FileMeta : IFileMeta { /// /// Metadata version /// - public int Version { get; init; } = IVoidFileMeta.CurrentVersion; + public int Version { get; init; } = IFileMeta.CurrentVersion; /// /// Internal Id of the file @@ -75,12 +75,17 @@ public record VoidFileMeta : IVoidFileMeta /// What storage system the file is on /// public string? Storage { get; set; } + + /// + /// Encryption params as JSON string + /// + public string? EncryptionParams { get; set; } } /// /// with attached /// -public record SecretVoidFileMeta : VoidFileMeta +public record SecretFileMeta : FileMeta { /// /// A secret key used to make edits to the file after its uploaded diff --git a/VoidCat/Services/Abstractions/IFileMetadataStore.cs b/VoidCat/Services/Abstractions/IFileMetadataStore.cs index a1ed3c7..b1a4ea6 100644 --- a/VoidCat/Services/Abstractions/IFileMetadataStore.cs +++ b/VoidCat/Services/Abstractions/IFileMetadataStore.cs @@ -5,7 +5,7 @@ namespace VoidCat.Services.Abstractions; /// /// File metadata contains all data about a file except for the file data itself /// -public interface IFileMetadataStore : IPublicPrivateStore +public interface IFileMetadataStore : IPublicPrivateStore { /// /// Get metadata for a single file @@ -13,7 +13,7 @@ public interface IFileMetadataStore : IPublicPrivateStore /// /// - ValueTask Get(Guid id) where TMeta : VoidFileMeta; + ValueTask Get(Guid id) where TMeta : FileMeta; /// /// Get metadata for multiple files @@ -21,7 +21,7 @@ public interface IFileMetadataStore : IPublicPrivateStore /// /// - ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta; + ValueTask> Get(Guid[] ids) where TMeta : FileMeta; /// /// Update file metadata @@ -30,7 +30,7 @@ public interface IFileMetadataStore : IPublicPrivateStore /// /// - ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta; + ValueTask Update(Guid id, TMeta meta) where TMeta : FileMeta; /// /// List all files in the store @@ -38,7 +38,7 @@ public interface IFileMetadataStore : IPublicPrivateStore /// /// - ValueTask> ListFiles(PagedRequest request) where TMeta : VoidFileMeta; + ValueTask> ListFiles(PagedRequest request) where TMeta : FileMeta; /// /// Returns basic stats about the file store diff --git a/VoidCat/Services/Background/DeleteExpiredFiles.cs b/VoidCat/Services/Background/DeleteExpiredFiles.cs index 2a5ec4e..d4a757c 100644 --- a/VoidCat/Services/Background/DeleteExpiredFiles.cs +++ b/VoidCat/Services/Background/DeleteExpiredFiles.cs @@ -27,7 +27,7 @@ public sealed class DeleteExpiredFiles : BackgroundService var fileInfoManager = scope.ServiceProvider.GetRequiredService(); var fileStoreFactory = scope.ServiceProvider.GetRequiredService(); - var files = await metadata.ListFiles(new(0, int.MaxValue)); + var files = await metadata.ListFiles(new(0, int.MaxValue)); await foreach (var f in files.Results.WithCancellation(stoppingToken)) { try diff --git a/VoidCat/Services/Background/VirusScannerService.cs b/VoidCat/Services/Background/VirusScannerService.cs index 76fc170..ada6570 100644 --- a/VoidCat/Services/Background/VirusScannerService.cs +++ b/VoidCat/Services/Background/VirusScannerService.cs @@ -29,7 +29,7 @@ public class VirusScannerService : BackgroundService var page = 0; while (true) { - var files = await _fileStore.ListFiles(new(page, 1_000)); + var files = await _fileStore.ListFiles(new(page, 1_000)); if (files.Pages < page) break; page++; diff --git a/VoidCat/Services/Files/FileInfoManager.cs b/VoidCat/Services/Files/FileInfoManager.cs index 96c2ea8..75322e8 100644 --- a/VoidCat/Services/Files/FileInfoManager.cs +++ b/VoidCat/Services/Files/FileInfoManager.cs @@ -35,7 +35,7 @@ public sealed class FileInfoManager /// public ValueTask Get(Guid id) { - return Get(id); + return Get(id); } /// @@ -45,7 +45,7 @@ public sealed class FileInfoManager /// public ValueTask GetPrivate(Guid id) { - return Get(id); + return Get(id); } /// @@ -82,7 +82,7 @@ public sealed class FileInfoManager } private async ValueTask Get(Guid id) - where TMeta : VoidFileMeta where TFile : VoidFile, new() + where TMeta : FileMeta where TFile : VoidFile, new() { var meta = _metadataStore.Get(id); var payment = _paymentStore.Get(id); diff --git a/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs b/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs index 3d236d3..0395bed 100644 --- a/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs +++ b/VoidCat/Services/Files/LocalDiskFileMetadataStore.cs @@ -22,13 +22,13 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore } /// - public ValueTask Get(Guid id) where TMeta : VoidFileMeta + public ValueTask Get(Guid id) where TMeta : FileMeta { return GetMeta(id); } /// - public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + public async ValueTask> Get(Guid[] ids) where TMeta : FileMeta { var ret = new List(); foreach (var id in ids) @@ -44,22 +44,17 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore } /// - public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta + public async ValueTask Update(Guid id, TMeta meta) where TMeta : FileMeta { - var oldMeta = await Get(id); + var oldMeta = await Get(id); if (oldMeta == default) return; - oldMeta.Description = meta.Description ?? oldMeta.Description; - oldMeta.Name = meta.Name ?? oldMeta.Name; - oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; - oldMeta.Storage = meta.Storage ?? oldMeta.Storage; - oldMeta.Expires = meta.Expires; - + oldMeta.Patch(meta); await Set(id, oldMeta); } /// - public ValueTask> ListFiles(PagedRequest request) where TMeta : VoidFileMeta + public ValueTask> ListFiles(PagedRequest request) where TMeta : FileMeta { async IAsyncEnumerable EnumerateFiles() { @@ -102,26 +97,26 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore /// public async ValueTask Stats() { - var files = await ListFiles(new(0, Int32.MaxValue)); + var files = await ListFiles(new(0, Int32.MaxValue)); var count = await files.Results.CountAsync(); var size = await files.Results.SumAsync(a => (long) a.Size); return new(count, (ulong) size); } /// - public ValueTask Get(Guid id) + public ValueTask Get(Guid id) { - return GetMeta(id); + return GetMeta(id); } /// - public ValueTask GetPrivate(Guid id) + public ValueTask GetPrivate(Guid id) { - return GetMeta(id); + return GetMeta(id); } /// - public async ValueTask Set(Guid id, SecretVoidFileMeta meta) + public async ValueTask Set(Guid id, SecretFileMeta meta) { var path = MapMeta(id); var json = JsonConvert.SerializeObject(meta); diff --git a/VoidCat/Services/Files/PostgresFileMetadataStore.cs b/VoidCat/Services/Files/PostgresFileMetadataStore.cs index 65cb82e..8c4d93b 100644 --- a/VoidCat/Services/Files/PostgresFileMetadataStore.cs +++ b/VoidCat/Services/Files/PostgresFileMetadataStore.cs @@ -18,32 +18,33 @@ public class PostgresFileMetadataStore : IFileMetadataStore public string? Key => "postgres"; /// - public ValueTask Get(Guid id) + public ValueTask Get(Guid id) { - return Get(id); + return Get(id); } /// - public ValueTask GetPrivate(Guid id) + public ValueTask GetPrivate(Guid id) { - return Get(id); + return Get(id); } /// - public async ValueTask Set(Guid id, SecretVoidFileMeta obj) + public async ValueTask Set(Guid id, SecretFileMeta obj) { await using var conn = await _connection.Get(); await conn.ExecuteAsync( @"insert into -""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"") -values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store) +""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"", ""EncryptionParams"") +values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store, :encryptionParams) on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Description"" = :description, ""MimeType"" = :mimeType, ""Expires"" = :expires, -""Storage"" = :store", +""Storage"" = :store, +""EncryptionParams"" = :encryptionParams", new { id, @@ -55,7 +56,8 @@ on conflict (""Id"") do update set digest = obj.Digest, editSecret = obj.EditSecret, expires = obj.Expires?.ToUniversalTime(), - store = obj.Storage + store = obj.Storage, + encryptionParams = obj.EncryptionParams }); } @@ -67,7 +69,7 @@ on conflict (""Id"") do update set } /// - public async ValueTask Get(Guid id) where TMeta : VoidFileMeta + public async ValueTask Get(Guid id) where TMeta : FileMeta { await using var conn = await _connection.Get(); return await conn.QuerySingleOrDefaultAsync(@"select * from ""Files"" where ""Id"" = :id", @@ -75,7 +77,7 @@ on conflict (""Id"") do update set } /// - public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + public async ValueTask> Get(Guid[] ids) where TMeta : FileMeta { await using var conn = await _connection.Get(); var ret = await conn.QueryAsync("select * from \"Files\" where \"Id\" in :ids", new {ids}); @@ -83,22 +85,17 @@ on conflict (""Id"") do update set } /// - public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta + public async ValueTask Update(Guid id, TMeta meta) where TMeta : FileMeta { - var oldMeta = await Get(id); + var oldMeta = await Get(id); if (oldMeta == default) return; - oldMeta.Description = meta.Description ?? oldMeta.Description; - oldMeta.Name = meta.Name ?? oldMeta.Name; - oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; - oldMeta.Storage = meta.Storage ?? oldMeta.Storage; - oldMeta.Expires = meta.Expires; - + oldMeta.Patch(meta); await Set(id, oldMeta); } /// - public async ValueTask> ListFiles(PagedRequest request) where TMeta : VoidFileMeta + public async ValueTask> ListFiles(PagedRequest request) where TMeta : FileMeta { await using var conn = await _connection.Get(); var count = await conn.ExecuteScalarAsync(@"select count(*) from ""Files"""); diff --git a/VoidCat/Services/Files/S3FileMetadataStore.cs b/VoidCat/Services/Files/S3FileMetadataStore.cs index 48a804d..18381b2 100644 --- a/VoidCat/Services/Files/S3FileMetadataStore.cs +++ b/VoidCat/Services/Files/S3FileMetadataStore.cs @@ -23,13 +23,13 @@ public class S3FileMetadataStore : IFileMetadataStore public string? Key => _config.Name; /// - public ValueTask Get(Guid id) where TMeta : VoidFileMeta + public ValueTask Get(Guid id) where TMeta : FileMeta { return GetMeta(id); } /// - public async ValueTask> Get(Guid[] ids) where TMeta : VoidFileMeta + public async ValueTask> Get(Guid[] ids) where TMeta : FileMeta { var ret = new List(); foreach (var id in ids) @@ -45,22 +45,17 @@ public class S3FileMetadataStore : IFileMetadataStore } /// - public async ValueTask Update(Guid id, TMeta meta) where TMeta : VoidFileMeta + public async ValueTask Update(Guid id, TMeta meta) where TMeta : FileMeta { - var oldMeta = await GetMeta(id); + var oldMeta = await Get(id); if (oldMeta == default) return; - oldMeta.Description = meta.Description ?? oldMeta.Description; - oldMeta.Name = meta.Name ?? oldMeta.Name; - oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType; - oldMeta.Storage = meta.Storage ?? oldMeta.Storage; - oldMeta.Expires = meta.Expires; - + oldMeta.Patch(meta); await Set(id, oldMeta); } /// - public ValueTask> ListFiles(PagedRequest request) where TMeta : VoidFileMeta + public ValueTask> ListFiles(PagedRequest request) where TMeta : FileMeta { async IAsyncEnumerable Enumerate() { @@ -95,26 +90,26 @@ public class S3FileMetadataStore : IFileMetadataStore /// public async ValueTask Stats() { - var files = await ListFiles(new(0, Int32.MaxValue)); + var files = await ListFiles(new(0, Int32.MaxValue)); var count = await files.Results.CountAsync(); var size = await files.Results.SumAsync(a => (long) a.Size); return new(count, (ulong) size); } /// - public ValueTask Get(Guid id) + public ValueTask Get(Guid id) { - return GetMeta(id); + return GetMeta(id); } /// - public ValueTask GetPrivate(Guid id) + public ValueTask GetPrivate(Guid id) { - return GetMeta(id); + return GetMeta(id); } /// - public async ValueTask Set(Guid id, SecretVoidFileMeta meta) + public async ValueTask Set(Guid id, SecretFileMeta meta) { await _client.PutObjectAsync(new() { @@ -131,7 +126,7 @@ public class S3FileMetadataStore : IFileMetadataStore await _client.DeleteObjectAsync(_config.BucketName, ToKey(id)); } - private async ValueTask GetMeta(Guid id) where TMeta : VoidFileMeta + private async ValueTask GetMeta(Guid id) where TMeta : FileMeta { try { diff --git a/VoidCat/Services/Migrations/Database/06-EncryptionParams.cs b/VoidCat/Services/Migrations/Database/06-EncryptionParams.cs new file mode 100644 index 0000000..b25e0e3 --- /dev/null +++ b/VoidCat/Services/Migrations/Database/06-EncryptionParams.cs @@ -0,0 +1,20 @@ +using FluentMigrator; + +namespace VoidCat.Services.Migrations.Database; + +[Migration(20220911_1635)] +public class EncryptionParams : Migration{ + public override void Up() + { + Create.Column("EncryptionParams") + .OnTable("Files") + .AsString() + .Nullable(); + } + + public override void Down() + { + Delete.Column("EncryptionParams") + .FromTable("Files"); + } +} \ No newline at end of file diff --git a/VoidCat/Services/Migrations/FixSize.cs b/VoidCat/Services/Migrations/FixSize.cs index 95c6ee5..b7df523 100644 --- a/VoidCat/Services/Migrations/FixSize.cs +++ b/VoidCat/Services/Migrations/FixSize.cs @@ -24,7 +24,7 @@ public class FixSize : IMigration /// public async ValueTask Migrate(string[] args) { - var files = await _fileMetadata.ListFiles(new(0, int.MaxValue)); + var files = await _fileMetadata.ListFiles(new(0, int.MaxValue)); await foreach (var file in files.Results) { try diff --git a/VoidCat/Services/Migrations/MigrateToPostgres.cs b/VoidCat/Services/Migrations/MigrateToPostgres.cs index 15c3e10..8566175 100644 --- a/VoidCat/Services/Migrations/MigrateToPostgres.cs +++ b/VoidCat/Services/Migrations/MigrateToPostgres.cs @@ -102,7 +102,7 @@ public class MigrateToPostgres : IMigration { var cachePaywallStore = new CachePaymentStore(_cache); - var files = await _fileMetadata.ListFiles(new(0, int.MaxValue)); + var files = await _fileMetadata.ListFiles(new(0, int.MaxValue)); await foreach (var file in files.Results) { try @@ -161,7 +161,7 @@ public class MigrateToPostgres : IMigration public string? Password { get; set; } } - private record UploaderSecretVoidFileMeta : SecretVoidFileMeta + private record UploaderSecretVoidFileMeta : SecretFileMeta { [JsonConverter(typeof(Base58GuidConverter))] public Guid? Uploader { get; set; } diff --git a/VoidCat/Services/Migrations/PopulateMetadataId.cs b/VoidCat/Services/Migrations/PopulateMetadataId.cs index 9f887fe..7e38548 100644 --- a/VoidCat/Services/Migrations/PopulateMetadataId.cs +++ b/VoidCat/Services/Migrations/PopulateMetadataId.cs @@ -24,7 +24,7 @@ public class PopulateMetadataId : IMigration return IMigration.MigrationResult.Skipped; } - var files = await _metadataStore.ListFiles(new(0, Int32.MaxValue)); + var files = await _metadataStore.ListFiles(new(0, Int32.MaxValue)); await foreach (var file in files.Results) { // read-write file metadata diff --git a/VoidCat/Services/VirusScanner/CacheVirusScanStore.cs b/VoidCat/Services/VirusScanner/CacheVirusScanStore.cs index ea47c9f..6c9e143 100644 --- a/VoidCat/Services/VirusScanner/CacheVirusScanStore.cs +++ b/VoidCat/Services/VirusScanner/CacheVirusScanStore.cs @@ -23,7 +23,7 @@ public class CacheVirusScanStore : BasicCacheStore, IVirusScanS var scans = await _cache.GetList(MapFilesKey(id)); if (scans.Length > 0) { - return await Get(Guid.Parse(scans.First())); + return await Get(Guid.Parse(scans.Last())); } return default; diff --git a/VoidCat/VoidCat.csproj b/VoidCat/VoidCat.csproj index 8928aaa..674dd9f 100644 --- a/VoidCat/VoidCat.csproj +++ b/VoidCat/VoidCat.csproj @@ -10,7 +10,7 @@ True $(DefineConstants);HostSPA $(AssemblyName).xml - 4.1.0 + 4.2.0 diff --git a/VoidCat/spa/src/Components/FileUpload/FileUpload.js b/VoidCat/spa/src/Components/FileUpload/FileUpload.js index 8bfb8da..e4c5027 100644 --- a/VoidCat/spa/src/Components/FileUpload/FileUpload.js +++ b/VoidCat/spa/src/Components/FileUpload/FileUpload.js @@ -1,13 +1,12 @@ import "./FileUpload.css"; import {useEffect, useState} from "react"; -import * as CryptoJS from 'crypto-js'; import {useSelector} from "react-redux"; -import sjcl from "sjcl"; -import {sjclcodec} from "../../codecBytes"; -import {ConstName, FormatBytes} from "../Shared/Util"; -import {RateCalculator} from "../Shared/RateCalculator"; +import {buf2hex, ConstName, FormatBytes} from "../Shared/Util"; import {ApiHost} from "../Shared/Const"; +import {StreamEncryption} from "../Shared/StreamEncryption"; +import {VoidButton} from "../Shared/VoidButton"; +import {useFileTransfer} from "../Shared/FileTransferHook"; const UploadState = { NotStarted: 0, @@ -20,45 +19,28 @@ const UploadState = { }; export const DigestAlgo = "SHA-256"; -const BlockSize = 16; export function FileUpload(props) { const auth = useSelector(state => state.login.jwt); const info = useSelector(state => state.info.info); - const [speed, setSpeed] = useState(0); - const [progress, setProgress] = useState(0); + const {speed, progress, loaded, setFileSize, reset, update} = useFileTransfer(); const [result, setResult] = useState(); const [uState, setUState] = useState(UploadState.NotStarted); const [challenge, setChallenge] = useState(); const [encryptionKey, setEncryptionKey] = useState(); - const calc = new RateCalculator(); + const [encrypt, setEncrypt] = useState(true); function handleProgress(e) { if (e instanceof ProgressEvent) { - let newProgress = e.loaded / e.total; - - calc.ReportLoaded(e.loaded); - setSpeed(calc.RateWindow(5)); - setProgress(newProgress); + loaded(e.loaded); } } - function generateEncryptionKey() { - let key = { - key: sjclcodec.toBits(window.crypto.getRandomValues(new Uint8Array(16))), - iv: sjclcodec.toBits(window.crypto.getRandomValues(new Uint8Array(12))) - }; - setEncryptionKey(key); - return key; - } - async function doStreamUpload() { - let key = generateEncryptionKey(); - let aes = new sjcl.cipher.aes(key.key); - + setFileSize(props.file.size); setUState(UploadState.Hashing); let hash = await digest(props.file); - calc.Reset(); + reset(); let offset = 0; async function readChunk(size) { @@ -72,36 +54,23 @@ export function FileUpload(props) { return new Uint8Array(data); } - async function readEncryptedChunk(size) { - if (offset >= props.file.size) { - return new Uint8Array(0); - } - size -= size % BlockSize; - - let end = Math.min(offset + size, props.file.size); - let blob = props.file.slice(offset, end, props.file.type); - let data = new Uint8Array(await blob.arrayBuffer()); - offset += data.byteLength; - let encryptedData = sjcl.mode.gcm.encrypt(aes, sjclcodec.toBits(data), key.iv); - return new Uint8Array(sjclcodec.fromBits(encryptedData)); - } - let rs = new ReadableStream({ - start: () => { + start: async () => { setUState(UploadState.Uploading); }, pull: async (controller) => { - let chunkSize = controller.desiredSize; - let chunk = key ? await readEncryptedChunk(chunkSize) : await readChunk(chunkSize); - if (chunk.byteLength === 0) { - controller.close(); - return; + try { + let chunk = await readChunk(controller.desiredSize); + if (chunk.byteLength === 0) { + controller.close(); + return; + } + update(chunk.length); + controller.enqueue(chunk); + } catch (e) { + console.error(e); + throw e; } - - calc.ReportProgress(chunk.byteLength); - setSpeed(calc.RateWindow(5)); - setProgress(offset / props.file.size); - controller.enqueue(chunk); }, cancel: (reason) => { console.log(reason); @@ -111,16 +80,28 @@ export function FileUpload(props) { highWaterMark: 1024 * 1024 }); - let req = await fetch("/upload", { + let enc = encrypt ? (() => { + let ret = new StreamEncryption(); + setEncryptionKey(ret.getKey()); + return ret; + })() : null; + rs = encrypt ? rs.pipeThrough(enc.getEncryptionTransform()) : rs; + + let headers = { + "Content-Type": "application/octet-stream", + "V-Content-Type": props.file.type, + "V-Filename": props.file.name, + "V-Full-Digest": hash + }; + if (encrypt) { + headers["V-EncryptionParams"] = JSON.stringify(enc.getParams()); + } + + let req = await fetch("https://localhost:7195/upload", { method: "POST", mode: "cors", body: rs, - headers: { - "Content-Type": "application/octet-stream", - "V-Content-Type": props.file.type, - "V-Filename": props.file.name, - "V-Full-Digest": hash - }, + headers, duplex: 'half' }); @@ -183,11 +164,12 @@ export function FileUpload(props) { } async function doXHRUpload() { + setFileSize(props.file.size); let uploadSize = info.uploadSegmentSize ?? Number.MAX_VALUE; setUState(UploadState.Hashing); let hash = await digest(props.file); - calc.Reset(); + reset(); if (props.file.size >= uploadSize) { await doSplitXHRUpload(hash, uploadSize); } else { @@ -198,10 +180,9 @@ export function FileUpload(props) { async function doSplitXHRUpload(hash, splitSize) { let xhr = null; - setProgress(0); const segments = Math.ceil(props.file.size / splitSize); for (let s = 0; s < segments; s++) { - calc.Reset(); + reset(); let offset = s * splitSize; let slice = props.file.slice(offset, offset + splitSize, props.file.type); xhr = await xhrSegment(slice, hash, xhr?.file?.id, xhr?.file?.metadata?.editSecret, s + 1, segments); @@ -229,30 +210,36 @@ export function FileUpload(props) { } async function digest(file) { - const chunkSize = 100_000_000; - let sha = CryptoJS.algo.SHA256.create(); - for (let x = 0; x < Math.ceil(file.size / chunkSize); x++) { - let offset = x * chunkSize; - let slice = file.slice(offset, offset + chunkSize, file.type); - let data = Uint32Array.from(await slice.arrayBuffer()); - sha.update(new CryptoJS.lib.WordArray.init(data, slice.length)); - - calc.ReportLoaded(offset); - setSpeed(calc.RateWindow(5)); - setProgress(offset / parseFloat(file.size)); - } - return sha.finalize().toString(); + let h = await window.crypto.subtle.digest(DigestAlgo, await file.arrayBuffer()); + return buf2hex(new Uint8Array(h)); } function renderStatus() { if (result) { - let link = encryptionKey ? `/${result.id}#${sjcl.codec.hex.fromBits(encryptionKey.key)}:${sjcl.codec.hex.fromBits(encryptionKey.iv)}` : `/${result.id}`; + let link = `/${result.id}`; return uState === UploadState.Done ?
Link:
{result.id}
+ {encryptionKey ? <> +
Encryption Key:
+
+ navigator.clipboard.writeText(encryptionKey)}>Copy +
+ : null}
: {result}; + } else if (uState === UploadState.NotStarted) { + return ( + <> +
+
Encrypt file:
+
setEncrypt(e.target.checked)}/> +
+
+ doStreamUpload()}>Upload + + ) } else { return (
@@ -274,11 +261,9 @@ export function FileUpload(props) { } useEffect(() => { - console.log(props.file); - let chromeVersion = getChromeVersion(); if (chromeVersion >= 105) { - doStreamUpload().catch(console.error); + //doStreamUpload().catch(console.error); } else { doXHRUpload().catch(console.error); } diff --git a/VoidCat/spa/src/Components/Shared/FileTransferHook.js b/VoidCat/spa/src/Components/Shared/FileTransferHook.js new file mode 100644 index 0000000..30a68ff --- /dev/null +++ b/VoidCat/spa/src/Components/Shared/FileTransferHook.js @@ -0,0 +1,26 @@ +import {useState} from "react"; +import {RateCalculator} from "./RateCalculator"; + +export function useFileTransfer() { + const [speed, setSpeed] = useState(0); + const [progress, setProgress] = useState(0); + const calc = new RateCalculator(); + + return { + speed, progress, + setFileSize: (size) => { + calc.SetFileSize(size); + }, + update: (bytes) => { + calc.ReportProgress(bytes); + setSpeed(calc.GetSpeed()); + setProgress(calc.GetProgress()); + }, + loaded: (loaded) => { + calc.ReportLoaded(loaded); + setSpeed(calc.GetSpeed()); + setProgress(calc.GetProgress()); + }, + reset: () => calc.Reset() + } +} \ No newline at end of file diff --git a/VoidCat/spa/src/Components/Shared/RateCalculator.js b/VoidCat/spa/src/Components/Shared/RateCalculator.js index 03add97..e8be691 100644 --- a/VoidCat/spa/src/Components/Shared/RateCalculator.js +++ b/VoidCat/spa/src/Components/Shared/RateCalculator.js @@ -1,12 +1,26 @@ export class RateCalculator { constructor() { - this.reports = []; - this.lastLoaded = 0; + this.Reset(); + this.fileSize = 0; } + SetFileSize(size) { + this.fileSize = size; + } + + GetProgress() { + return this.progress; + } + + GetSpeed() { + return this.speed; + } + Reset() { this.reports = []; this.lastLoaded = 0; + this.progress = 0; + this.speed = 0; } ReportProgress(amount) { @@ -14,6 +28,9 @@ export class RateCalculator { time: new Date().getTime(), amount }); + this.lastLoaded += amount; + this.progress = this.lastLoaded / parseFloat(this.fileSize); + this.speed = this.RateWindow(5); } ReportLoaded(loaded) { @@ -22,6 +39,8 @@ export class RateCalculator { amount: loaded - this.lastLoaded }); this.lastLoaded = loaded; + this.progress = this.lastLoaded / parseFloat(this.fileSize); + this.speed = this.RateWindow(5); } RateWindow(s) { diff --git a/VoidCat/spa/src/Components/Shared/StreamEncryption.js b/VoidCat/spa/src/Components/Shared/StreamEncryption.js new file mode 100644 index 0000000..721a6a2 --- /dev/null +++ b/VoidCat/spa/src/Components/Shared/StreamEncryption.js @@ -0,0 +1,110 @@ +import {sjclcodec} from "../../codecBytes"; +import sjcl from "sjcl"; +import {buf2hex} from "./Util"; + +/** + * AES-GCM TransformStream + */ +export class StreamEncryption { + constructor(key, iv, params) { + if (key === undefined && iv === undefined) { + key = buf2hex(window.crypto.getRandomValues(new Uint8Array(16))); + iv = buf2hex(window.crypto.getRandomValues(new Uint8Array(12))); + } + if (typeof key === "string" && typeof iv === "string") { + key = sjcl.codec.hex.toBits(key); + iv = sjcl.codec.hex.toBits(iv); + } else if (!Array.isArray(key) || !Array.isArray(iv)) { + throw "Key and IV must be hex string or bitArray"; + } + if (typeof params === "string") { + params = JSON.parse(params); + } + + this.TagSize = params?.ts ?? 128; + this.ChunkSize = params?.cs ?? (1024 * 1024 * 10); + this.aes = new sjcl.cipher.aes(key); + this.key = key; + this.iv = iv; + + console.log(`ts=${this.TagSize}, cs=${this.ChunkSize}, key=${key}, iv=${this.iv}`); + } + + /** + * Return formatted encryption key + * @returns {string} + */ + getKey() { + return `${sjcl.codec.hex.fromBits(this.key)}:${sjcl.codec.hex.fromBits(this.iv)}`; + } + + /** + * Get encryption params + * @returns {{cs: (*|number), ts: number}} + */ + getParams() { + return { + ts: this.TagSize, + cs: this.ChunkSize + } + } + + /** + * Get encryption TransformStream + * @returns {TransformStream} + */ + getEncryptionTransform() { + return this._getCryptoStream(0); + } + + /** + * Get decryption TransformStream + * @returns {TransformStream} + */ + getDecryptionTransform() { + return this._getCryptoStream(1); + } + + _getCryptoStream(mode) { + let offset = 0; + let buffer = new Uint8Array(this.ChunkSize + (mode === 1 ? this.TagSize / 8 : 0)); + return new window.TransformStream({ + transform: async (chunk, controller) => { + chunk = await chunk; + try { + let toBuffer = Math.min(chunk.byteLength, buffer.byteLength - offset); + buffer.set(chunk.slice(0, toBuffer), offset); + offset += toBuffer; + + if (offset === buffer.byteLength) { + let buff = sjclcodec.toBits(buffer); + let encryptedBuf = sjclcodec.fromBits( + mode === 0 ? + sjcl.mode.gcm.encrypt(this.aes, buff, this.iv, [], this.TagSize) : + sjcl.mode.gcm.decrypt(this.aes, buff, this.iv, [], this.TagSize) + ); + controller.enqueue(new Uint8Array(encryptedBuf)); + + offset = chunk.byteLength - toBuffer; + buffer.set(chunk.slice(toBuffer)); + } + } catch (e) { + console.error(e); + throw e; + } + }, + flush: (controller) => { + let lastBuffer = buffer.slice(0, offset); + let buff = sjclcodec.toBits(lastBuffer); + let encryptedBuf = sjclcodec.fromBits( + mode === 0 ? + sjcl.mode.gcm.encrypt(this.aes, buff, this.iv, [], this.TagSize) : + sjcl.mode.gcm.decrypt(this.aes, buff, this.iv, [], this.TagSize) + ); + controller.enqueue(new Uint8Array(encryptedBuf)); + } + }, { + highWaterMark: this.ChunkSize + }); + } +} \ No newline at end of file diff --git a/VoidCat/spa/src/Pages/FilePreview.css b/VoidCat/spa/src/Pages/FilePreview.css index ab61828..d0cf970 100644 --- a/VoidCat/spa/src/Pages/FilePreview.css +++ b/VoidCat/spa/src/Pages/FilePreview.css @@ -26,4 +26,15 @@ border-radius: 10px; border: 1px solid red; margin-bottom: 5px; +} + +.preview .encrypted { + padding: 10px; + border-radius: 10px; + border: 2px solid #bbbbbb; + text-align: center; +} + +.error { + color: red; } \ No newline at end of file diff --git a/VoidCat/spa/src/Pages/FilePreview.js b/VoidCat/spa/src/Pages/FilePreview.js index 324ce0f..c9bf295 100644 --- a/VoidCat/spa/src/Pages/FilePreview.js +++ b/VoidCat/spa/src/Pages/FilePreview.js @@ -1,17 +1,19 @@ import "./FilePreview.css"; import {Fragment, useEffect, useState} from "react"; import {useParams} from "react-router-dom"; -import {TextPreview} from "../Components/FilePreview/TextPreview"; import FeatherIcon from "feather-icons-react"; +import {Helmet} from "react-helmet"; + +import {TextPreview} from "../Components/FilePreview/TextPreview"; import {FileEdit} from "../Components/FileEdit/FileEdit"; import {FilePayment} from "../Components/FilePreview/FilePayment"; import {useApi} from "../Components/Shared/Api"; -import {Helmet} from "react-helmet"; import {FormatBytes} from "../Components/Shared/Util"; import {ApiHost} from "../Components/Shared/Const"; import {InlineProfile} from "../Components/Shared/InlineProfile"; -import sjcl from "sjcl"; -import {sjclcodec} from "../codecBytes"; +import {StreamEncryption} from "../Components/Shared/StreamEncryption"; +import {VoidButton} from "../Components/Shared/VoidButton"; +import {useFileTransfer} from "../Components/Shared/FileTransferHook"; export function FilePreview() { const {Api} = useApi(); @@ -19,6 +21,9 @@ export function FilePreview() { const [info, setInfo] = useState(); const [order, setOrder] = useState(); const [link, setLink] = useState("#"); + const [key, setKey] = useState(""); + const [error, setError] = useState(""); + const {speed, progress, update, setFileSize} = useFileTransfer(); async function loadInfo() { let req = await Api.fileInfo(params.id); @@ -28,13 +33,74 @@ export function FilePreview() { } } + function isFileEncrypted() { + return "string" === typeof info?.metadata?.encryptionParams + } + + function isDecrypted() { + return link.startsWith("blob:"); + } + + function isPaymentRequired() { + return info?.payment?.required === true && !order; + } + function canAccessFile() { - if (info?.payment?.required === true && !order) { + if (isPaymentRequired()) { + return false; + } + if (isFileEncrypted() && !isDecrypted()) { return false; } return true; } + async function decryptFile() { + try { + let hashKey = key.match(/([0-9a-z]{32}):([0-9a-z]{24})/); + if (hashKey?.length === 3) { + let [key, iv] = [hashKey[1], hashKey[2]]; + let enc = new StreamEncryption(key, iv, info.metadata?.encryptionParams); + + let rsp = await fetch(link); + if (rsp.ok) { + let reader = rsp.body + .pipeThrough(enc.getDecryptionTransform()) + .pipeThrough(decryptionProgressTransform()); + let newResponse = new Response(reader); + setLink(window.URL.createObjectURL(await newResponse.blob(), {type: info.metadata.mimeType})); + } + } else { + setError("Invalid encryption key format"); + } + } catch (e) { + setError(e.message); + } + } + + function decryptionProgressTransform() { + return new window.TransformStream({ + transform: (chunk, controller) => { + update(chunk.length); + controller.enqueue(chunk); + } + }); + } + + function renderEncryptedDownload() { + if (!isFileEncrypted() || isDecrypted() || isPaymentRequired()) return; + return ( +
+

This file is encrypted, please enter the encryption key:

+ setKey(e.target.value)}/> + decryptFile()}>Decrypt + {progress > 0 ? `${(100 * progress).toFixed(0)}% (${FormatBytes(speed)}/s)` : null} + {error ?

{error}

: null} +
+ ); + } + function renderPayment() { if (info.payment && info.payment.service !== 0) { if (!order) { @@ -46,6 +112,8 @@ export function FilePreview() { } function renderPreview() { + if (!canAccessFile()) return; + if (info.metadata) { switch (info.metadata.mimeType) { case "image/avif": @@ -145,41 +213,8 @@ export function FilePreview() { useEffect(() => { if (info) { let fileLink = info.metadata?.url ?? `${ApiHost}/d/${info.id}`; - - // detect encrypted file link - let hashKey = window.location.hash.match(/#([0-9a-z]{32}):([0-9a-z]{24})/); - if (hashKey.length === 3) { - let [key, iv] = [sjcl.codec.hex.toBits(hashKey[1]), sjcl.codec.hex.toBits(hashKey[2])]; - console.log(key, iv); - let aes = new sjcl.cipher.aes(key); - - async function load() { - let decryptStream = new window.TransformStream({ - transform: async (chunk, controller) => { - chunk = await chunk; - console.log("Transforming chunk:", chunk); - - let buff = sjclcodec.toBits(chunk); - let decryptedBuff = sjclcodec.fromBits(sjcl.mode.gcm.decrypt(aes, buff, iv)); - console.log("Decrypted data:", decryptedBuff); - controller.enqueue(new Uint8Array(decryptedBuff)); - } - }); - let rsp = await fetch(fileLink); - if (rsp.ok) { - let reader = rsp.body - .pipeThrough(decryptStream); - - console.log("Pipe reader", reader); - let newResponse = new Response(reader); - setLink(window.URL.createObjectURL(await newResponse.blob(), {type: info.metadata.mimeType})); - } - } - - load(); - return; - } - + setFileSize(info.metadata.size); + let order = window.localStorage.getItem(`payment-${info.id}`); if (order) { let orderObj = JSON.parse(order); @@ -213,7 +248,8 @@ export function FilePreview() { {renderPayment()} - {canAccessFile() ? renderPreview() : null} + {renderPreview()} + {renderEncryptedDownload()}