laylarr00/voidkitty
1
0
Fork 0
forked from Kieran/void.cat
Code Pull Requests Activity

Cleanup VirusTotal scanner

Browse Source
This commit is contained in:
Kieran 2022-03-07 21:13:42 +00:00
parent 8340916c15
commit 801edafbcf
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
7 changed files with 188 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
VoidCat/Controllers/StatsController.cs
View File

@ -29,7 +29,8 @@ namespace VoidCat.Controllers
bytes += vf.Metadata?.Size ?? 0;
count++;
}
return new(bw, bytes, count);
return new(bw, bytes, count, BuildInfo.GetBuildInfo());
}
[HttpGet]
@ -41,6 +42,7 @@ namespace VoidCat.Controllers
}
}
public sealed record GlobalStats(Bandwidth Bandwidth, ulong TotalBytes, int Count);
public sealed record GlobalStats(Bandwidth Bandwidth, ulong TotalBytes, int Count, BuildInfo BuildInfo);
public sealed record FileStats(Bandwidth Bandwidth);
}
}

31
VoidCat/Model/BuildInfo.cs Normal file
View File

@ -0,0 +1,31 @@
using System.Reflection;
namespace VoidCat.Model;
public class BuildInfo
{
public string? Version { get; init; }
public string? GitHash { get; init; }
public DateTime BuildTime { get; init; }
public static BuildInfo GetBuildInfo()
{
var asm = Assembly.GetEntryAssembly();
var version = asm.GetName().Version;
var gitHash = asm
.GetCustomAttributes<AssemblyMetadataAttribute>()
.FirstOrDefault(attr => attr.Key == "GitHash")?.Value;
var buildTime = asm
.GetCustomAttributes<AssemblyMetadataAttribute>()
.FirstOrDefault(attr => attr.Key == "BuildTime");
return new()
{
Version = $"{version.Major}.{version.Minor}.{version.Build}",
GitHash = gitHash,
BuildTime = DateTime.FromBinary(long.Parse(buildTime?.Value ?? "0"))
};
}
}

7
VoidCat/Services/Background/VirusScannerService.cs
View File

@ -1,4 +1,5 @@
using VoidCat.Services.Abstractions;
using VoidCat.Services.VirusScanner.Exceptions;
namespace VoidCat.Services.Background;
@ -42,6 +43,12 @@ public class VirusScannerService : BackgroundService
var result = await _scanner.ScanFile(file.Id, stoppingToken);
await _scanStore.Set(file.Id, result);
}
catch (RateLimitedException rx)
{
var sleep = rx.RetryAfter ?? DateTimeOffset.UtcNow.AddMinutes(10);
_logger.LogWarning("VirusScanner was rate limited, sleeping until {Time}", sleep);
await Task.Delay(sleep - DateTimeOffset.UtcNow, stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to scan file {Id} error={Message}", file.Id, ex.Message);

6
VoidCat/Services/VirusScanner/Exceptions/RateLimitedException.cs Normal file
View File

@ -0,0 +1,6 @@
namespace VoidCat.Services.VirusScanner.Exceptions;
public class RateLimitedException : Exception
{
public DateTimeOffset? RetryAfter { get; init; }
}

339
VoidCat/Services/VirusScanner/VirusTotal/VirusTotalClasses.cs
View File

@ -1,285 +1,133 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json;
// ReSharper disable InconsistentNaming
#pragma warning disable CS8618
namespace VoidCat.Services.VirusScanner.VirusTotal;
// Root myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
public class AlertContext
{
[JsonProperty("proto")] public string Proto { get; set; }
[JsonProperty("src_ip")] public string SrcIp { get; set; }
[JsonProperty("src_port")] public int SrcPort { get; set; }
}
public class CrowdsourcedIdsResult
{
[JsonProperty("alert_context")] public List<AlertContext> AlertContext { get; set; }
[JsonProperty("alert_severity")] public string AlertSeverity { get; set; }
[JsonProperty("rule_category")] public string RuleCategory { get; set; }
[JsonProperty("rule_id")] public string RuleId { get; set; }
[JsonProperty("rule_msg")] public string RuleMsg { get; set; }
[JsonProperty("rule_source")] public string RuleSource { get; set; }
}
public class CrowdsourcedIdsStats
{
[JsonProperty("high")] public int High { get; set; }
[JsonProperty("info")] public int Info { get; set; }
[JsonProperty("low")] public int Low { get; set; }
[JsonProperty("medium")] public int Medium { get; set; }
}
public class CrowdsourcedYaraResult
{
[JsonProperty("description")] public string Description { get; set; }
[JsonProperty("match_in_subfile")] public bool MatchInSubfile { get; set; }
[JsonProperty("rule_name")] public string RuleName { get; set; }
[JsonProperty("ruleset_id")] public string RulesetId { get; set; }
[JsonProperty("ruleset_name")] public string RulesetName { get; set; }
[JsonProperty("source")] public string Source { get; set; }
}
public class ALYac
{
[JsonProperty("category")] public string Category { get; set; }
[JsonProperty("engine_name")] public string EngineName { get; set; }
[JsonProperty("engine_update")] public string EngineUpdate { get; set; }
[JsonProperty("engine_version")] public string EngineVersion { get; set; }
[JsonProperty("method")] public string Method { get; set; }
[JsonProperty("result")] public string Result { get; set; }
}
public class APEX
{
[JsonProperty("category")] public string Category { get; set; }
[JsonProperty("engine_name")] public string EngineName { get; set; }
[JsonProperty("engine_update")] public string EngineUpdate { get; set; }
[JsonProperty("engine_version")] public string EngineVersion { get; set; }
[JsonProperty("method")] public string Method { get; set; }
[JsonProperty("result")] public string Result { get; set; }
}
public class AVG
{
[JsonProperty("category")] public string Category { get; set; }
[JsonProperty("engine_name")] public string EngineName { get; set; }
[JsonProperty("engine_update")] public string EngineUpdate { get; set; }
[JsonProperty("engine_version")] public string EngineVersion { get; set; }
[JsonProperty("method")] public string Method { get; set; }
[JsonProperty("result")] public string Result { get; set; }
}
public class Acronis
{
[JsonProperty("category")] public string Category { get; set; }
[JsonProperty("engine_name")] public string EngineName { get; set; }
[JsonProperty("engine_update")] public string EngineUpdate { get; set; }
[JsonProperty("engine_version")] public string EngineVersion { get; set; }
[JsonProperty("method")] public string Method { get; set; }
[JsonProperty("result")] public object Result { get; set; }
}
public class LastAnalysisResults
{
[JsonProperty("ALYac")] public ALYac ALYac { get; set; }
[JsonProperty("APEX")] public APEX APEX { get; set; }
[JsonProperty("AVG")] public AVG AVG { get; set; }
[JsonProperty("Acronis")] public Acronis Acronis { get; set; }
}
public class LastAnalysisStats
{
[JsonProperty("confirmed-timeout")] public int ConfirmedTimeout { get; set; }
[JsonProperty("confirmed-timeout")]
public int ConfirmedTimeout { get; set; }
[JsonProperty("failure")] public int Failure { get; set; }
[JsonProperty("failure")]
public int Failure { get; set; }
[JsonProperty("harmless")] public int Harmless { get; set; }
[JsonProperty("harmless")]
public int Harmless { get; set; }
[JsonProperty("malicious")] public int Malicious { get; set; }
[JsonProperty("malicious")]
public int Malicious { get; set; }
[JsonProperty("suspicious")] public int Suspicious { get; set; }
[JsonProperty("suspicious")]
public int Suspicious { get; set; }
[JsonProperty("timeout")] public int Timeout { get; set; }
[JsonProperty("timeout")]
public int Timeout { get; set; }
[JsonProperty("type-unsupported")] public int TypeUnsupported { get; set; }
[JsonProperty("type-unsupported")]
public int TypeUnsupported { get; set; }
[JsonProperty("undetected")] public int Undetected { get; set; }
}
public class VirusTotalJujubox
{
[JsonProperty("category")] public string Category { get; set; }
[JsonProperty("confidence")] public int Confidence { get; set; }
[JsonProperty("malware_classification")]
public List<string> MalwareClassification { get; set; }
[JsonProperty("malware_names")] public List<string> MalwareNames { get; set; }
[JsonProperty("sandbox_name")] public string SandboxName { get; set; }
}
public class SandboxVerdicts
{
[JsonProperty("VirusTotal Jujubox")] public VirusTotalJujubox VirusTotalJujubox { get; set; }
}
public class SigmaAnalysisStats
{
[JsonProperty("critical")] public int Critical { get; set; }
[JsonProperty("high")] public int High { get; set; }
[JsonProperty("low")] public int Low { get; set; }
[JsonProperty("medium")] public int Medium { get; set; }
}
public class SigmaIntegratedRuleSetGitHub
{
[JsonProperty("critical")] public int Critical { get; set; }
[JsonProperty("high")] public int High { get; set; }
[JsonProperty("low")] public int Low { get; set; }
[JsonProperty("medium")] public int Medium { get; set; }
}
public class SigmaAnalysisSummary
{
[JsonProperty("Sigma Integrated Rule Set (GitHub)")]
public SigmaIntegratedRuleSetGitHub SigmaIntegratedRuleSetGitHub { get; set; }
[JsonProperty("undetected")]
public int Undetected { get; set; }
}
public class TotalVotes
{
[JsonProperty("harmless")] public int Harmless { get; set; }
[JsonProperty("harmless")]
public int Harmless { get; set; }
[JsonProperty("malicious")] public int Malicious { get; set; }
[JsonProperty("malicious")]
public int Malicious { get; set; }
}
public class Attributes
{
[JsonProperty("capabilities_tags")] public List<string> CapabilitiesTags { get; set; }
[JsonProperty("capabilities_tags")]
public List<string> CapabilitiesTags { get; set; }
[JsonProperty("creation_date")] public int CreationDate { get; set; }
[JsonProperty("creation_date")]
public int CreationDate { get; set; }
[JsonProperty("crowdsourced_ids_results")]
public List<CrowdsourcedIdsResult> CrowdsourcedIdsResults { get; set; }
[JsonProperty("crowdsourced_ids_stats")]
public CrowdsourcedIdsStats CrowdsourcedIdsStats { get; set; }
[JsonProperty("crowdsourced_yara_results")]
public List<CrowdsourcedYaraResult> CrowdsourcedYaraResults { get; set; }
[JsonProperty("downloadable")] public bool Downloadable { get; set; }
[JsonProperty("downloadable")]
public bool Downloadable { get; set; }
[JsonProperty("first_submission_date")]
public int FirstSubmissionDate { get; set; }
[JsonProperty("last_analysis_date")] public int LastAnalysisDate { get; set; }
[JsonProperty("last_analysis_date")]
public int LastAnalysisDate { get; set; }
[JsonProperty("last_analysis_results")]
public LastAnalysisResults LastAnalysisResults { get; set; }
[JsonProperty("last_analysis_stats")] public LastAnalysisStats LastAnalysisStats { get; set; }
[JsonProperty("last_analysis_stats")]
public LastAnalysisStats LastAnalysisStats { get; set; }
[JsonProperty("last_modification_date")]
public int LastModificationDate { get; set; }
[JsonProperty("last_submission_date")] public int LastSubmissionDate { get; set; }
[JsonProperty("last_submission_date")]
public int LastSubmissionDate { get; set; }
[JsonProperty("md5")] public string Md5 { get; set; }
[JsonProperty("md5")]
public string Md5 { get; set; }
[JsonProperty("meaningful_name")] public string MeaningfulName { get; set; }
[JsonProperty("meaningful_name")]
public string MeaningfulName { get; set; }
[JsonProperty("names")] public List<string> Names { get; set; }
[JsonProperty("names")]
public List<string> Names { get; set; }
[JsonProperty("reputation")] public int Reputation { get; set; }
[JsonProperty("reputation")]
public int Reputation { get; set; }
[JsonProperty("sandbox_verdicts")] public SandboxVerdicts SandboxVerdicts { get; set; }
[JsonProperty("sha1")]
public string Sha1 { get; set; }
[JsonProperty("sha1")] public string Sha1 { get; set; }
[JsonProperty("sha256")]
public string Sha256 { get; set; }
[JsonProperty("sha256")] public string Sha256 { get; set; }
[JsonProperty("size")]
public int Size { get; set; }
[JsonProperty("sigma_analysis_stats")] public SigmaAnalysisStats SigmaAnalysisStats { get; set; }
[JsonProperty("tags")]
public List<string> Tags { get; set; }
[JsonProperty("sigma_analysis_summary")]
public SigmaAnalysisSummary SigmaAnalysisSummary { get; set; }
[JsonProperty("times_submitted")]
public int TimesSubmitted { get; set; }
[JsonProperty("size")] public int Size { get; set; }
[JsonProperty("total_votes")]
public TotalVotes TotalVotes { get; set; }
[JsonProperty("tags")] public List<string> Tags { get; set; }
[JsonProperty("type_description")]
public string TypeDescription { get; set; }
[JsonProperty("times_submitted")] public int TimesSubmitted { get; set; }
[JsonProperty("type_tag")]
public string TypeTag { get; set; }
[JsonProperty("total_votes")] public TotalVotes TotalVotes { get; set; }
[JsonProperty("unique_sources")]
public int UniqueSources { get; set; }
[JsonProperty("type_description")] public string TypeDescription { get; set; }
[JsonProperty("type_tag")] public string TypeTag { get; set; }
[JsonProperty("unique_sources")] public int UniqueSources { get; set; }
[JsonProperty("vhash")] public string Vhash { get; set; }
[JsonProperty("vhash")]
public string Vhash { get; set; }
}
public class Links
{
[JsonProperty("self")] public string Self { get; set; }
[JsonProperty("self")]
public string Self { get; set; }
}
public class File
{
[JsonProperty("attributes")] public Attributes Attributes { get; set; }
[JsonProperty("attributes")]
public Attributes Attributes { get; set; }
[JsonProperty("id")] public string Id { get; set; }
[JsonProperty("id")]
public string Id { get; set; }
[JsonProperty("links")] public Links Links { get; set; }
[JsonProperty("links")]
public Links Links { get; set; }
[JsonProperty("type")] public string Type { get; set; }
[JsonProperty("type")]
public string Type { get; set; }
}
public class Error
@ -305,23 +153,32 @@ public class VTException : Exception
{
public VTException(Error error)
{
Error = error;
ErrorCode = Enum.TryParse<VTErrorCodes>(error.Code, out var c) ? c : VTErrorCodes.UnknownError;
Message = error.Message;
}
protected VTException(SerializationInfo info, StreamingContext context, Error error) : base(info, context)
{
Error = error;
}
public VTErrorCodes ErrorCode { get; }
public string Message { get; }
}
public VTException(string? message, Error error) : base(message)
{
Error = error;
}
public VTException(string? message, Exception? innerException, Error error) : base(message, innerException)
{
Error = error;
}
public Error Error { get; }
public enum VTErrorCodes
{
UnknownError,
BadRequestError,
InvalidArgumentError,
NotAvailableYet,
UnselectiveContentQueryError,
UnsupportedContentQueryError,
AuthenticationRequiredError,
UserNotActiveError,
WrongCredentialsError,
ForbiddenError,
NotFoundError,
AlreadyExistsError,
FailedDependencyError,
QuotaExceededError,
TooManyRequestsError,
TransientError,
DeadlineExceededError
}

28
VoidCat/Services/VirusScanner/VirusTotal/VirusTotalScanner.cs
View File

@ -1,6 +1,7 @@
using System.Security.Cryptography;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.VirusScanner.Exceptions;
namespace VoidCat.Services.VirusScanner.VirusTotal;
@ -24,13 +25,30 @@ public class VirusTotalScanner : IVirusScanner
// hash file and check on VT
var hash = await SHA256.Create().ComputeHashAsync(fs, cts);
var report = await _client.GetReport(hash.ToHex());
if (report != default)
try
{
return new()
var report = await _client.GetReport(hash.ToHex());
if (report != default)
{
IsVirus = report.Attributes.Reputation == 0
};
return new()
{
IsVirus = report.Attributes.Reputation == 0
};
}
}
catch (VTException vx)
{
if (vx.ErrorCode == VTErrorCodes.QuotaExceededError)
{
throw new RateLimitedException()
{
// retry tomorrow :(
// this makes it pretty much unusable unless you have a paid subscription
RetryAfter = DateTimeOffset.Now.Date.AddDays(1)
};
}
throw;
}
throw new InvalidOperationException();

18
VoidCat/VoidCat.csproj
View File

@ -16,11 +16,16 @@
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
<PackageReference Include="MSBuildGitHash" Version="2.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NBitcoin" Version="6.0.19" />
<PackageReference Include="nClam" Version="7.0.0" />
<PackageReference Include="prometheus-net.AspNetCore" Version="5.0.2" />
<PackageReference Include="Seq.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="StackExchange.Redis" Version="2.5.27-prerelease" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
@ -64,4 +69,17 @@
</ItemGroup>
<Copy SourceFiles="@(_CopyItems)" DestinationFolder="$(PublishDir)wwwroot\%(RecursiveDir)" />
</Target>
<ItemGroup>
<AssemblyMetadata Include="%(AssemblyMetadataCommand.Identity)">
<Value>$(Value)</Value>
</AssemblyMetadata>
<AssemblyAttributes Include="AssemblyMetadata">
<_Parameter1>BuildTime</_Parameter1>
<_Parameter2>$([System.DateTime]::UtcNow.ToBinary())</_Parameter2>
</AssemblyAttributes>
<Compile Include="$(IntermediateOutputPath)\build_date.cs" />
</ItemGroup>
<Target Name="GetBuildDate" BeforeTargets="CoreCompile">
<WriteCodeFragment Language="C#" OutputFile="$(IntermediateOutputPath)\build_date.cs" AssemblyAttributes="@(AssemblyAttributes)" />
</Target>
</Project>