Kieran/zap-stream-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload thumbnails to S3

Browse Source
This commit is contained in:
Kieran
2023-08-01 23:08:57 +01:00
parent 99ad1dc439
commit d2b56c14f9
20 changed files with 482 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
NostrStreamer/Config.cs
View File

@ -37,7 +37,7 @@ public class Config
public LndConfig Lnd { get; init; } = null!;
public S3BlobConfig DvrStore { get; init; } = null!;
public S3BlobConfig S3Store { get; init; } = null!;
public DateTime TosDate { get; init; }
}

33
NostrStreamer/Controllers/PlaylistController.cs
View File

@ -16,11 +16,9 @@ public class PlaylistController : Controller
private readonly SrsApi _srsApi;
private readonly ViewCounter _viewCounter;
private readonly StreamManagerFactory _streamManagerFactory;
private readonly ThumbnailService _thumbnailService;
public PlaylistController(Config config, ILogger<PlaylistController> logger,
HttpClient client, SrsApi srsApi, ViewCounter viewCounter, StreamManagerFactory streamManagerFactory,
ThumbnailService thumbnailService)
HttpClient client, SrsApi srsApi, ViewCounter viewCounter, StreamManagerFactory streamManagerFactory)
{
_config = config;
_logger = logger;
@ -28,7 +26,6 @@ public class PlaylistController : Controller
_srsApi = srsApi;
_viewCounter = viewCounter;
_streamManagerFactory = streamManagerFactory;
_thumbnailService = thumbnailService;
}
[ResponseCache(Duration = 1, Location = ResponseCacheLocation.Any)]
@ -85,34 +82,6 @@ public class PlaylistController : Controller
}
}
[ResponseCache(Duration = 30, Location = ResponseCacheLocation.Any)]
[HttpGet("{id}.jpg")]
public async Task GetPreview([FromRoute] Guid id)
{
try
{
var stream = _thumbnailService.GetThumbnail(id);
if (stream != default)
{
Response.ContentLength = stream.Length;
Response.ContentType = "image/jpg";
Response.Headers.CacheControl = "public, max-age=60";
await Response.StartAsync();
await stream.CopyToAsync(Response.Body);
await Response.CompleteAsync();
}
else
{
Response.StatusCode = 404;
}
}
catch (Exception ex)
{
_logger.LogWarning("Failed to get preview image for {id} {message}", id, ex.Message);
Response.StatusCode = 404;
}
}
[ResponseCache(Duration = 1, Location = ResponseCacheLocation.Any)]
[HttpGet("{pubkey}.m3u8")]
public async Task<IActionResult> GetCurrentStreamRedirect([FromRoute] string pubkey)

2
NostrStreamer/Database/Configuration/UserStreamConfiguration.cs
View File

@ -22,7 +22,7 @@ public class UserStreamConfiguration : IEntityTypeConfiguration<UserStream>
builder.Property(a => a.Event)
.IsRequired();
builder.Property(a => a.Recording);
builder.Property(a => a.Thumbnail);
builder.Property(a => a.EdgeIp)
.IsRequired();

4
NostrStreamer/Database/UserStream.cs
View File

@ -21,9 +21,9 @@ public class UserStream
public string Event { get; set; } = null!;
/// <summary>
/// Recording URL of ended stream
/// URL of auto-generated thumbnail
/// </summary>
public string? Recording { get; set; }
public string? Thumbnail { get; set; }
public Guid EndpointId { get; init; }
public IngestEndpoint Endpoint { get; init; } = null!;

314
NostrStreamer/Migrations/20230801210018_Thumbnails.Designer.cs generated Normal file
View File

@ -0,0 +1,314 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NostrStreamer.Database;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace NostrStreamer.Migrations
{
[DbContext(typeof(StreamerContext))]
[Migration("20230801210018_Thumbnails")]
partial class Thumbnails
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("NostrStreamer.Database.IngestEndpoint", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("App")
.IsRequired()
.HasColumnType("text");
b.Property<List<string>>("Capabilities")
.IsRequired()
.HasColumnType("text[]");
b.Property<int>("Cost")
.HasColumnType("integer");
b.Property<string>("Forward")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("App")
.IsUnique();
b.ToTable("Endpoints");
});
modelBuilder.Entity("NostrStreamer.Database.Payment", b =>
{
b.Property<string>("PaymentHash")
.HasColumnType("text");
b.Property<decimal>("Amount")
.HasColumnType("numeric(20,0)");
b.Property<DateTime>("Created")
.HasColumnType("timestamp with time zone");
b.Property<string>("Invoice")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("IsPaid")
.HasColumnType("boolean");
b.Property<string>("Nostr")
.HasColumnType("text");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<int>("Type")
.HasColumnType("integer");
b.HasKey("PaymentHash");
b.HasIndex("PubKey");
b.ToTable("Payments");
});
modelBuilder.Entity("NostrStreamer.Database.User", b =>
{
b.Property<string>("PubKey")
.HasColumnType("text");
b.Property<long>("Balance")
.HasColumnType("bigint");
b.Property<string>("ContentWarning")
.HasColumnType("text");
b.Property<string>("Image")
.HasColumnType("text");
b.Property<string>("StreamKey")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Summary")
.HasColumnType("text");
b.Property<string>("Tags")
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<DateTime?>("TosAccepted")
.IsRequired()
.HasColumnType("timestamp with time zone");
b.Property<uint>("Version")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("xid")
.HasColumnName("xmin");
b.HasKey("PubKey");
b.ToTable("Users");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("EdgeIp")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("EndpointId")
.HasColumnType("uuid");
b.Property<DateTime?>("Ends")
.HasColumnType("timestamp with time zone");
b.Property<string>("Event")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ForwardClientId")
.IsRequired()
.HasColumnType("text");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Starts")
.HasColumnType("timestamp with time zone");
b.Property<int>("State")
.HasColumnType("integer");
b.Property<string>("StreamId")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("EndpointId");
b.HasIndex("PubKey");
b.ToTable("Streams");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamGuest", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Relay")
.HasColumnType("text");
b.Property<string>("Role")
.HasColumnType("text");
b.Property<string>("Sig")
.HasColumnType("text");
b.Property<Guid>("StreamId")
.HasColumnType("uuid");
b.Property<decimal>("ZapSplit")
.HasColumnType("numeric");
b.HasKey("Id");
b.HasIndex("StreamId", "PubKey")
.IsUnique();
b.ToTable("Guests");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamRecording", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<double>("Duration")
.HasColumnType("double precision");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("UserStreamId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("UserStreamId");
b.ToTable("Recordings");
});
modelBuilder.Entity("NostrStreamer.Database.Payment", b =>
{
b.HasOne("NostrStreamer.Database.User", "User")
.WithMany("Payments")
.HasForeignKey("PubKey")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.HasOne("NostrStreamer.Database.IngestEndpoint", "Endpoint")
.WithMany()
.HasForeignKey("EndpointId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("NostrStreamer.Database.User", "User")
.WithMany("Streams")
.HasForeignKey("PubKey")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Endpoint");
b.Navigation("User");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamGuest", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "Stream")
.WithMany("Guests")
.HasForeignKey("StreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Stream");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamRecording", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "Stream")
.WithMany()
.HasForeignKey("UserStreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Stream");
});
modelBuilder.Entity("NostrStreamer.Database.User", b =>
{
b.Navigation("Payments");
b.Navigation("Streams");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.Navigation("Guests");
});
#pragma warning restore 612, 618
}
}
}

28
NostrStreamer/Migrations/20230801210018_Thumbnails.cs Normal file
View File

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace NostrStreamer.Migrations
{
/// <inheritdoc />
public partial class Thumbnails : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.RenameColumn(
name: "Recording",
table: "Streams",
newName: "Thumbnail");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.RenameColumn(
name: "Thumbnail",
table: "Streams",
newName: "Recording");
}
}
}

6
NostrStreamer/Migrations/StreamerContextModelSnapshot.cs
View File

@ -161,9 +161,6 @@ namespace NostrStreamer.Migrations
.IsRequired()
.HasColumnType("text");
b.Property<string>("Recording")
.HasColumnType("text");
b.Property<DateTime>("Starts")
.HasColumnType("timestamp with time zone");
@ -174,6 +171,9 @@ namespace NostrStreamer.Migrations
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("EndpointId");

5
NostrStreamer/Program.cs
View File

@ -5,8 +5,10 @@ using Microsoft.EntityFrameworkCore;
using Nostr.Client.Client;
using NostrStreamer.Database;
using NostrStreamer.Services;
using NostrStreamer.Services.Background;
using NostrStreamer.Services.Dvr;
using NostrStreamer.Services.StreamManager;
using NostrStreamer.Services.Thumbnail;
namespace NostrStreamer;
@ -58,7 +60,8 @@ internal static class Program
services.AddTransient<StreamEventBuilder>();
services.AddTransient<StreamManagerFactory>();
services.AddTransient<UserService>();
services.AddTransient<ThumbnailService>();
services.AddTransient<IThumbnailService, S3ThumbnailService>();
services.AddHostedService<ThumbnailGenerator>();
services.AddTransient<IDvrStore, S3DvrStore>();

2
NostrStreamer/Services/BackgroundStreamManager.cs → NostrStreamer/Services/Background/BackgroundStreamManager.cs
View File

@ -2,7 +2,7 @@ using Microsoft.EntityFrameworkCore;
using NostrStreamer.Database;
using NostrStreamer.Services.StreamManager;
namespace NostrStreamer.Services;
namespace NostrStreamer.Services.Background;
public class BackgroundStreamManager : BackgroundService
{

2
NostrStreamer/Services/LndInvoiceStream.cs → NostrStreamer/Services/Background/LndInvoiceStream.cs
View File

@ -5,7 +5,7 @@ using Microsoft.EntityFrameworkCore;
using Nostr.Client.Utils;
using NostrStreamer.Database;
namespace NostrStreamer.Services;
namespace NostrStreamer.Services.Background;
public class LndInvoicesStream : BackgroundService
{

2
NostrStreamer/Services/NostrListener.cs → NostrStreamer/Services/Background/NostrListener.cs
View File

@ -5,7 +5,7 @@ using Nostr.Client.Communicator;
using Nostr.Client.Requests;
using Websocket.Client.Models;
namespace NostrStreamer.Services;
namespace NostrStreamer.Services.Background;
public class NostrListener : IDisposable
{

5
NostrStreamer/Services/ThumbnailGenerator.cs → NostrStreamer/Services/Background/ThumbnailGenerator.cs
View File

@ -1,7 +1,8 @@
using Microsoft.EntityFrameworkCore;
using NostrStreamer.Database;
using NostrStreamer.Services.Thumbnail;
namespace NostrStreamer.Services;
namespace NostrStreamer.Services.Background;
public class ThumbnailGenerator : BackgroundService
{
@ -22,7 +23,7 @@ public class ThumbnailGenerator : BackgroundService
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<StreamerContext>();
var gen = scope.ServiceProvider.GetRequiredService<ThumbnailService>();
var gen = scope.ServiceProvider.GetRequiredService<IThumbnailService>();
var streams = await db.Streams
.AsNoTracking()

4
NostrStreamer/Services/Dvr/S3DvrStore.cs
View File

@ -17,8 +17,8 @@ public class S3DvrStore : IDvrStore
{
_httpClient = httpClient;
_logger = logger;
_config = config.DvrStore;
_client = config.DvrStore.CreateClient();
_config = config.S3Store;
_client = config.S3Store.CreateClient();
}
public async Task<UploadResult> UploadRecording(UserStream stream, Uri source)

2
NostrStreamer/Services/StreamEventBuilder.cs
View File

@ -37,7 +37,7 @@ public class StreamEventBuilder
new("d", stream.Id.ToString()),
new("title", user.Title ?? ""),
new("summary", user.Summary ?? ""),
new("image", new Uri(_config.DataHost, $"{stream.Id}.jpg").ToString()),
new("image", stream.Thumbnail ?? user.Image ?? ""),
new("status", status),
new("p", user.PubKey, "", "host"),
new("relays", _config.Relays),

7
NostrStreamer/Services/StreamManager/NostrStreamManager.cs
View File

@ -12,7 +12,6 @@ public class NostrStreamManager : IStreamManager
private readonly StreamManagerContext _context;
private readonly StreamEventBuilder _eventBuilder;
private readonly IDvrStore _dvrStore;
private readonly ThumbnailService _thumbnailService;
private readonly Config _config;
public NostrStreamManager(ILogger<NostrStreamManager> logger, StreamManagerContext context, IServiceProvider serviceProvider)
@ -21,7 +20,6 @@ public class NostrStreamManager : IStreamManager
_context = context;
_eventBuilder = serviceProvider.GetRequiredService<StreamEventBuilder>();
_dvrStore = serviceProvider.GetRequiredService<IDvrStore>();
_thumbnailService = serviceProvider.GetRequiredService<ThumbnailService>();
_config = serviceProvider.GetRequiredService<Config>();
}
@ -56,11 +54,8 @@ public class NostrStreamManager : IStreamManager
{
_logger.LogInformation("Stream started for: {pubkey}", _context.User.PubKey);
TestCanStream();
await UpdateStreamState(UserStreamState.Live);
#pragma warning disable CS4014
Task.Run(async () => await _thumbnailService.GenerateThumb(_context.UserStream));
#pragma warning restore CS4014
await UpdateStreamState(UserStreamState.Live);
}
public async Task StreamStopped()

29
NostrStreamer/Services/Thumbnail/BaseThumbnailService.cs Normal file
View File

@ -0,0 +1,29 @@
using FFMpegCore;
using NostrStreamer.Database;
namespace NostrStreamer.Services.Thumbnail;
public abstract class BaseThumbnailService
{
protected readonly ILogger Logger;
protected readonly Config Config;
protected BaseThumbnailService(Config config, ILogger logger)
{
Config = config;
Logger = logger;
}
protected async Task<string> GenerateThumbnail(UserStream stream)
{
var path = Path.ChangeExtension(Path.GetTempFileName(), ".jpg");
var cmd = FFMpegArguments
.FromUrlInput(new Uri(Config.RtmpHost, $"{stream.Endpoint.App}/source/{stream.User.StreamKey}?vhost=hls.zap.stream"))
.OutputToFile(path, true, o => { o.ForceFormat("image2").WithCustomArgument("-vframes 1"); })
.CancellableThrough(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Logger.LogInformation("Running command {cmd}", cmd.Arguments);
await cmd.ProcessAsynchronously();
return path;
}
}

8
NostrStreamer/Services/Thumbnail/IThumbnailService.cs Normal file
View File

@ -0,0 +1,8 @@
using NostrStreamer.Database;
namespace NostrStreamer.Services.Thumbnail;
public interface IThumbnailService
{
Task GenerateThumb(UserStream stream);
}

80
NostrStreamer/Services/Thumbnail/S3ThumbnailService.cs Normal file
View File

@ -0,0 +1,80 @@
using System.Diagnostics;
using Amazon.S3;
using Microsoft.EntityFrameworkCore;
using NostrStreamer.Database;
namespace NostrStreamer.Services.Thumbnail;
public class S3ThumbnailService : BaseThumbnailService, IThumbnailService
{
private readonly AmazonS3Client _client;
private readonly StreamerContext _context;
public S3ThumbnailService(Config config, ILogger<S3ThumbnailService> logger, StreamerContext context) : base(config, logger)
{
_client = config.S3Store.CreateClient();
_context = context;
}
public async Task GenerateThumb(UserStream stream)
{
try
{
var sw = Stopwatch.StartNew();
var path = await GenerateThumbnail(stream);
var tGen = sw.Elapsed;
var s3Path = MapPath(stream.Id);
sw.Restart();
await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
await _client.PutObjectAsync(new()
{
BucketName = Config.S3Store.BucketName,
Key = s3Path,
InputStream = fs,
AutoCloseStream = false,
AutoResetStreamPosition = false,
ContentType = "image/jpeg",
DisablePayloadSigning = Config.S3Store.DisablePayloadSigning
});
var uri = _client.GetPreSignedURL(new()
{
BucketName = Config.S3Store.BucketName,
Key = s3Path,
Expires = DateTime.UtcNow.AddYears(1000)
});
var ub = new UriBuilder(uri)
{
Scheme = Config.S3Store.PublicHost.Scheme,
Host = Config.S3Store.PublicHost.Host,
Port = Config.S3Store.PublicHost.Port
};
var tUpload = sw.Elapsed;
sw.Restart();
await _context.Streams.Where(a => a.Id == stream.Id)
.ExecuteUpdateAsync(o => o.SetProperty(v => v.Thumbnail, ub.Uri.ToString()));
var tDbUpdate = sw.Elapsed;
stream.Thumbnail = ub.Uri.ToString();
fs.Close();
File.Delete(path);
Logger.LogInformation("{id} generated={tg:#,##0}ms, uploaded={tu:#,##0}ms, db={td:#,##0}ms", stream.Id, tGen.TotalMilliseconds,
tUpload.TotalMilliseconds, tDbUpdate.TotalMilliseconds);
}
catch (Exception ex)
{
Logger.LogWarning("Failed to generate {id} thumbnail {msg}", stream.Id, ex.Message);
}
}
private string MapPath(Guid id)
{
return $"{id}/thumb.jpg";
}
}

55
NostrStreamer/Services/ThumbnailService.cs
View File

@ -1,55 +0,0 @@
using System.Diagnostics;
using FFMpegCore;
using NostrStreamer.Database;
namespace NostrStreamer.Services;
public class ThumbnailService
{
private const string Dir = "thumbs";
private readonly Config _config;
private readonly ILogger<ThumbnailService> _logger;
public ThumbnailService(Config config, ILogger<ThumbnailService> logger)
{
_config = config;
_logger = logger;
if (!Directory.Exists(Dir))
{
Directory.CreateDirectory(Dir);
}
}
public async Task GenerateThumb(UserStream stream)
{
var path = MapPath(stream.Id);
try
{
var sw = Stopwatch.StartNew();
var cmd = FFMpegArguments
.FromUrlInput(new Uri(_config.RtmpHost, $"{stream.Endpoint.App}/source/{stream.User.StreamKey}?vhost=hls.zap.stream"))
.OutputToFile(path, true, o => { o.ForceFormat("image2").WithCustomArgument("-vframes 1"); })
.CancellableThrough(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
_logger.LogInformation("Running command {cmd}", cmd.Arguments);
await cmd.ProcessAsynchronously();
sw.Stop();
_logger.LogInformation("Generated {id} thumb in {n:#,##0}ms", stream.Id, sw.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
_logger.LogWarning("Failed to generate {id} thumbnail {msg}", stream.Id, ex.Message);
}
}
public System.IO.Stream? GetThumbnail(Guid id)
{
var path = MapPath(id);
return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null;
}
private string MapPath(Guid id)
{
return Path.Combine(Dir, $"{id}.jpg");
}
}

2
NostrStreamer/appsettings.json
View File

@ -27,7 +27,7 @@
"CertPath": "/Users/kieran/.polar/networks/1/volumes/lnd/bob/tls.cert",
"MacaroonPath": "/Users/kieran/.polar/networks/1/volumes/lnd/bob/data/chain/bitcoin/regtest/admin.macaroon"
},
"DvrStore": {
"S3Store": {
"ServiceUrl": "http://localhost:9010",
"AccessKey": "TQcxug1ZAXfnZ5bvc9n5",
"SecretKey": "p7EK4qew6DBkBPqrpRPuJgTOc6ChUlfIcEdAwE7K",