2022-03-01 16:48:42 +00:00
|
|
|
|
using Amazon.S3;
|
|
|
|
|
using Newtonsoft.Json;
|
|
|
|
|
using VoidCat.Model;
|
|
|
|
|
using VoidCat.Services.Abstractions;
|
|
|
|
|
|
|
|
|
|
namespace VoidCat.Services.Files;
|
|
|
|
|
|
|
|
|
|
public class S3FileMetadataStore : IFileMetadataStore
|
|
|
|
|
{
|
|
|
|
|
private readonly ILogger<S3FileMetadataStore> _logger;
|
|
|
|
|
private readonly AmazonS3Client _client;
|
|
|
|
|
private readonly S3BlobConfig _config;
|
2022-03-02 19:19:02 +00:00
|
|
|
|
private readonly bool _includeUrl;
|
|
|
|
|
|
2022-03-01 16:48:42 +00:00
|
|
|
|
public S3FileMetadataStore(VoidSettings settings, ILogger<S3FileMetadataStore> logger)
|
|
|
|
|
{
|
|
|
|
|
_logger = logger;
|
2022-03-02 19:19:02 +00:00
|
|
|
|
_includeUrl = settings.CloudStorage?.ServeFromCloud ?? false;
|
2022-03-01 16:48:42 +00:00
|
|
|
|
_config = settings.CloudStorage!.S3!;
|
|
|
|
|
_client = _config.CreateClient();
|
|
|
|
|
}
|
2022-03-02 19:19:02 +00:00
|
|
|
|
|
2022-03-07 13:38:53 +00:00
|
|
|
|
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
|
|
|
|
|
{
|
|
|
|
|
return GetMeta<TMeta>(id);
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-06 21:51:25 +00:00
|
|
|
|
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
|
|
|
|
|
{
|
|
|
|
|
var ret = new List<TMeta>();
|
|
|
|
|
foreach (var id in ids)
|
|
|
|
|
{
|
|
|
|
|
var r = await GetMeta<TMeta>(id);
|
|
|
|
|
if (r != null)
|
|
|
|
|
{
|
|
|
|
|
ret.Add(r);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-15 10:39:36 +00:00
|
|
|
|
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
|
|
|
|
|
{
|
|
|
|
|
var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
|
|
|
|
|
if (oldMeta == default) return;
|
2022-06-06 21:51:25 +00:00
|
|
|
|
|
2022-03-15 10:39:36 +00:00
|
|
|
|
oldMeta.Description = meta.Description ?? oldMeta.Description;
|
|
|
|
|
oldMeta.Name = meta.Name ?? oldMeta.Name;
|
|
|
|
|
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
|
2022-06-06 21:51:25 +00:00
|
|
|
|
|
2022-03-15 10:39:36 +00:00
|
|
|
|
await Set(id, oldMeta);
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-06 21:51:25 +00:00
|
|
|
|
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
|
|
|
|
|
{
|
|
|
|
|
var count = 0;
|
|
|
|
|
var size = 0UL;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var obj = await _client.ListObjectsV2Async(new()
|
|
|
|
|
{
|
|
|
|
|
BucketName = _config.BucketName,
|
2022-06-06 22:06:02 +00:00
|
|
|
|
Prefix = "metadata_",
|
|
|
|
|
MaxKeys = 5_000
|
2022-06-06 21:51:25 +00:00
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
foreach (var file in obj.S3Objects)
|
|
|
|
|
{
|
2022-06-06 22:06:02 +00:00
|
|
|
|
if (Guid.TryParse(file.Key.Split("metadata_")[1], out var id))
|
2022-06-06 21:51:25 +00:00
|
|
|
|
{
|
|
|
|
|
var meta = await GetMeta<VoidFileMeta>(id);
|
|
|
|
|
if (meta != default)
|
|
|
|
|
{
|
|
|
|
|
count++;
|
|
|
|
|
size += meta.Size;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
catch (AmazonS3Exception aex)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogError(aex, "Failed to list files: {Error}", aex.Message);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return new(count, size);
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-07 13:38:53 +00:00
|
|
|
|
public ValueTask<VoidFileMeta?> Get(Guid id)
|
|
|
|
|
{
|
|
|
|
|
return GetMeta<VoidFileMeta>(id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
|
|
|
|
|
{
|
|
|
|
|
await _client.PutObjectAsync(new()
|
|
|
|
|
{
|
|
|
|
|
BucketName = _config.BucketName,
|
|
|
|
|
Key = ToKey(id),
|
|
|
|
|
ContentBody = JsonConvert.SerializeObject(meta),
|
|
|
|
|
ContentType = "application/json"
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async ValueTask Delete(Guid id)
|
|
|
|
|
{
|
|
|
|
|
await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : VoidFileMeta
|
2022-03-01 16:48:42 +00:00
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var obj = await _client.GetObjectAsync(_config.BucketName, ToKey(id));
|
|
|
|
|
|
|
|
|
|
using var sr = new StreamReader(obj.ResponseStream);
|
|
|
|
|
var json = await sr.ReadToEndAsync();
|
2022-03-02 19:19:02 +00:00
|
|
|
|
var ret = JsonConvert.DeserializeObject<TMeta>(json);
|
|
|
|
|
if (ret != default && _includeUrl)
|
|
|
|
|
{
|
|
|
|
|
var ub = new UriBuilder(_config.ServiceUrl!)
|
|
|
|
|
{
|
|
|
|
|
Path = $"/{_config.BucketName}/{id}"
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
ret.Url = ub.Uri;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ret;
|
2022-03-01 16:48:42 +00:00
|
|
|
|
}
|
|
|
|
|
catch (AmazonS3Exception aex)
|
|
|
|
|
{
|
|
|
|
|
_logger.LogError(aex, "Failed to get metadata for {Id}, {Error}", id, aex.Message);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return default;
|
|
|
|
|
}
|
2022-06-06 21:51:25 +00:00
|
|
|
|
|
2022-06-06 22:06:02 +00:00
|
|
|
|
private static string ToKey(Guid id) => $"metadata_{id}";
|
2022-06-06 21:51:25 +00:00
|
|
|
|
}
|