diff --git a/VoidCat/Controllers/InfoController.cs b/VoidCat/Controllers/InfoController.cs index a79beaf..4c12b4d 100644 --- a/VoidCat/Controllers/InfoController.cs +++ b/VoidCat/Controllers/InfoController.cs @@ -34,12 +34,28 @@ public class InfoController : Controller var bw = await _statsReporter.GetBandwidth(); var storeStats = await _fileMetadata.Stats(); - return new(bw, storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(), - _settings.CaptchaSettings?.SiteKey, - await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow), - _fileStores); + return new() + { + Bandwidth = bw, + TotalBytes = storeStats.Size, + Count = storeStats.Files, + BuildInfo = BuildInfo.GetBuildInfo(), + CaptchaSiteKey = _settings.CaptchaSettings?.SiteKey, + TimeSeriesMetrics = await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow), + FileStores = _fileStores, + UploadSegmentSize = _settings.UploadSegmentSize + }; } - public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo, - string? CaptchaSiteKey, IEnumerable TimeSeriesMetrics, IEnumerable FileStores); -} + public sealed class GlobalInfo + { + public Bandwidth Bandwidth { get; init; } + public ulong TotalBytes { get; init; } + public long Count { get; init; } + public BuildInfo BuildInfo { get; init; } + public string? CaptchaSiteKey { get; init; } + public IEnumerable TimeSeriesMetrics { get; init; } + public IEnumerable FileStores { get; init; } + public ulong? UploadSegmentSize { get; init; } + } +} \ No newline at end of file diff --git a/VoidCat/Model/VoidSettings.cs b/VoidCat/Model/VoidSettings.cs index a6e21a9..6c382b8 100644 --- a/VoidCat/Model/VoidSettings.cs +++ b/VoidCat/Model/VoidSettings.cs @@ -11,7 +11,12 @@ namespace VoidCat.Model /// Data directory to store files in /// public string DataDirectory { get; init; } = "./data"; - + + /// + /// Size in bytes to split uploads into chunks + /// + public ulong? UploadSegmentSize { get; init; } = null; + /// /// Tor configuration /// diff --git a/VoidCat/Services/Files/FileStorageStartup.cs b/VoidCat/Services/Files/FileStorageStartup.cs index 347e2b1..b024c3a 100644 --- a/VoidCat/Services/Files/FileStorageStartup.cs +++ b/VoidCat/Services/Files/FileStorageStartup.cs @@ -30,11 +30,11 @@ public static class FileStorageStartup } } - if (!string.IsNullOrEmpty(settings.Postgres)) + if (settings.HasPostgres()) { services.AddTransient(); services.AddTransient(); - if (settings.MetadataStore == "postgres") + if (settings.MetadataStore is "postgres" or "local-disk") { services.AddSingleton(); } diff --git a/VoidCat/spa/package.json b/VoidCat/spa/package.json index 3cadd29..61bf663 100644 --- a/VoidCat/spa/package.json +++ b/VoidCat/spa/package.json @@ -6,6 +6,7 @@ "dependencies": { "@hcaptcha/react-hcaptcha": "^1.1.1", "@reduxjs/toolkit": "^1.7.2", + "crypto-js": "^4.1.1", "feather-icons-react": "^0.5.0", "moment": "^2.29.4", "preval.macro": "^5.0.0", diff --git a/VoidCat/spa/src/FileUpload.js b/VoidCat/spa/src/FileUpload.js index 88004a3..54310b6 100644 --- a/VoidCat/spa/src/FileUpload.js +++ b/VoidCat/spa/src/FileUpload.js @@ -1,6 +1,7 @@ import {useEffect, useState} from "react"; -import {buf2hex, ConstName, FormatBytes} from "./Util"; +import {ConstName, FormatBytes} from "./Util"; import {RateCalculator} from "./RateCalculator"; +import * as CryptoJS from 'crypto-js'; import "./FileUpload.css"; import {useSelector} from "react-redux"; @@ -20,6 +21,7 @@ export const DigestAlgo = "SHA-256"; export function FileUpload(props) { const auth = useSelector(state => state.login.jwt); + const info = useSelector(state => state.info.info); const [speed, setSpeed] = useState(0); const [progress, setProgress] = useState(0); const [result, setResult] = useState(); @@ -84,14 +86,14 @@ export function FileUpload(props) { /** * Upload a segment of the file * @param segment {ArrayBuffer} - * @param id {string} + * @param fullDigest {string} Full file hash + * @param id {string?} * @param editSecret {string?} - * @param fullDigest {string?} Full file hash * @param part {int?} Segment number * @param partOf {int?} Total number of segments * @returns {Promise} */ - async function xhrSegment(segment, id, editSecret, fullDigest, part, partOf) { + async function xhrSegment(segment, fullDigest, id, editSecret, part, partOf) { setUState(UploadState.Uploading); return await new Promise((resolve, reject) => { @@ -133,23 +135,36 @@ export function FileUpload(props) { } async function doXHRUpload() { - // upload file in segments of 50MB - const UploadSize = 50_000_000; + let uploadSize = info.uploadSegmentSize ?? Number.MAX_VALUE; setUState(UploadState.Hashing); - let digest = await crypto.subtle.digest(DigestAlgo, await props.file.arrayBuffer()); + let hash = await digest(props.file); + if(props.file.size >= uploadSize) { + await doSplitXHRUpload(hash, uploadSize); + } else { + let xhr = await xhrSegment(props.file, hash); + handleXHRResult(xhr); + } + } + + async function doSplitXHRUpload(hash, splitSize) { let xhr = null; - const segments = Math.ceil(props.file.size / UploadSize); + setProgress(0); + const segments = Math.ceil(props.file.size / splitSize); for (let s = 0; s < segments; s++) { calc.ResetLastLoaded(); - let offset = s * UploadSize; - let slice = props.file.slice(offset, offset + UploadSize, props.file.type); + let offset = s * splitSize; + let slice = props.file.slice(offset, offset + splitSize, props.file.type); let segment = await slice.arrayBuffer(); - xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, buf2hex(digest), s + 1, segments); + xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, hash, s + 1, segments); if (!xhr.ok) { break; } } + handleXHRResult(xhr); + } + + function handleXHRResult(xhr) { if (xhr.ok) { setUState(UploadState.Done); setResult(xhr.file); @@ -159,6 +174,19 @@ export function FileUpload(props) { setResult(xhr.errorMessage); } } + + async function digest(file) { + const chunkSize = 10_000_000; + let sha = CryptoJS.algo.SHA256.create(); + for (let x = 0; x < Math.ceil(file.size / chunkSize); x++) { + let offset = x * chunkSize; + let slice = file.slice(offset, offset + chunkSize, file.type); + let data = Uint32Array.from(await slice.arrayBuffer()); + sha.update(new CryptoJS.lib.WordArray.init(data, slice.length)); + setProgress(offset / parseFloat(file.size)); + } + return sha.finalize().toString(); + } function renderStatus() { if (result) { diff --git a/VoidCat/spa/src/GlobalStats.js b/VoidCat/spa/src/GlobalStats.js index 4f0aa8e..985663a 100644 --- a/VoidCat/spa/src/GlobalStats.js +++ b/VoidCat/spa/src/GlobalStats.js @@ -7,7 +7,7 @@ import moment from "moment"; import {useSelector} from "react-redux"; export function GlobalStats() { - let stats = useSelector(state => state.info.stats); + let stats = useSelector(state => state.info.info); return ( diff --git a/VoidCat/spa/src/Header.js b/VoidCat/spa/src/Header.js index f5c970e..bbca1a3 100644 --- a/VoidCat/spa/src/Header.js +++ b/VoidCat/spa/src/Header.js @@ -5,7 +5,7 @@ import {InlineProfile} from "./InlineProfile"; import {useApi} from "./Api"; import {logout, setProfile} from "./LoginState"; import {useEffect} from "react"; -import {setStats} from "./SiteInfoStore"; +import {setInfo} from "./SiteInfoStore"; export function Header() { const dispatch = useDispatch(); @@ -26,7 +26,7 @@ export function Header() { async function loadStats() { let req = await Api.info(); if (req.ok) { - dispatch(setStats(await req.json())); + dispatch(setInfo(await req.json())); } } diff --git a/VoidCat/spa/src/SiteInfoStore.js b/VoidCat/spa/src/SiteInfoStore.js index d24badc..cc8fe9e 100644 --- a/VoidCat/spa/src/SiteInfoStore.js +++ b/VoidCat/spa/src/SiteInfoStore.js @@ -3,14 +3,14 @@ export const SiteInfoState = createSlice({ name: "SiteInfo", initialState: { - stats: null + info: null }, reducers: { - setStats: (state, action) => { - state.stats = action.payload; + setInfo: (state, action) => { + state.info = action.payload; }, } }); -export const {setStats} = SiteInfoState.actions; +export const {setInfo} = SiteInfoState.actions; export default SiteInfoState.reducer; \ No newline at end of file diff --git a/VoidCat/spa/yarn.lock b/VoidCat/spa/yarn.lock index 73479ee..0972bee 100644 --- a/VoidCat/spa/yarn.lock +++ b/VoidCat/spa/yarn.lock @@ -3022,6 +3022,11 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +crypto-js@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.1.1.tgz#9e485bcf03521041bd85844786b83fb7619736cf" + integrity sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw== + crypto-random-string@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" diff --git a/docker-compose.yml b/docker-compose.yml index d72ea3a..d1a6ea9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,8 +11,12 @@ services: - redis redis: image: "redis:alpine" + ports: + - "6379:6379" postgres: image: "postgres:14.1" + ports: + - "5432:5432" environment: - "POSTGRES_DB=void" - "POSTGRES_HOST_AUTH_METHOD=trust"