Kieran/zap-stream-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup recordings

Browse Source
This commit is contained in:
Kieran
2024-02-27 21:43:37 +00:00
parent 103963f81f
commit 6003534272
16 changed files with 646 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
NostrStreamer/Config.cs
View File

@ -56,6 +56,8 @@ public class Config
public Uri SnortApi { get; init; } = null!;
public Uri? DiscordLiveWebhook { get; init; }
public int RetainRecordingsDays { get; init; } = 90;
}
public class VapidKeyDetails

2
NostrStreamer/Database/Configuration/UserStreamRecordingConfiguration.cs
View File

@ -18,7 +18,7 @@ public class UserStreamRecordingConfiguration : IEntityTypeConfiguration<UserStr
.IsRequired();
builder.HasOne(a => a.Stream)
.WithMany()
.WithMany(a => a.Recordings)
.HasForeignKey(a => a.UserStreamId);
}
}

2
NostrStreamer/Database/UserStream.cs
View File

@ -51,6 +51,8 @@ public class UserStream
public decimal Length { get; set; }
public List<UserStreamGuest> Guests { get; init; } = new();
public List<UserStreamRecording> Recordings { get; init; } = new();
}
public enum UserStreamState

8
NostrStreamer/Extensions.cs
View File

@ -112,7 +112,13 @@ public static class Extensions
}
return new NostrEventIdentifier(ev.Id!, ev.Pubkey, null, ev.Kind);
}
}
public static NostrIdentifier ToIdentifier(this UserStream stream)
{
var ev = NostrJson.Deserialize<NostrEvent>(stream.Event);
return ev!.ToIdentifier();
}
}
public class Variant

3
NostrStreamer/Migrations/20230725123250_Endpoints.cs
View File

@ -53,6 +53,9 @@ namespace NostrStreamer.Migrations
principalTable: "Endpoints",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
migrationBuilder.Sql(
"INSERT INTO public.\"Endpoints\"(\"Id\", \"Name\", \"App\", \"Forward\", \"Cost\", \"Capabilities\") VALUES(gen_random_uuid(), 'basic', 'basic', 'base.in.zap.stream', 1000, '{variant:source,dvr:source}');");
}
/// <inheritdoc />

466
NostrStreamer/Migrations/20240226120459_StreamRecordings.Designer.cs generated Normal file
View File

@ -0,0 +1,466 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NostrStreamer.Database;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace NostrStreamer.Migrations
{
[DbContext(typeof(StreamerContext))]
[Migration("20240226120459_StreamRecordings")]
partial class StreamRecordings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("NostrStreamer.Database.IngestEndpoint", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("App")
.IsRequired()
.HasColumnType("text");
b.Property<List<string>>("Capabilities")
.IsRequired()
.HasColumnType("text[]");
b.Property<int>("Cost")
.HasColumnType("integer");
b.Property<string>("Forward")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("App")
.IsUnique();
b.ToTable("Endpoints");
});
modelBuilder.Entity("NostrStreamer.Database.Payment", b =>
{
b.Property<string>("PaymentHash")
.HasColumnType("text");
b.Property<decimal>("Amount")
.HasColumnType("numeric(20,0)");
b.Property<DateTime>("Created")
.HasColumnType("timestamp with time zone");
b.Property<string>("Invoice")
.HasColumnType("text");
b.Property<bool>("IsPaid")
.HasColumnType("boolean");
b.Property<string>("Nostr")
.HasColumnType("text");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<int>("Type")
.HasColumnType("integer");
b.HasKey("PaymentHash");
b.HasIndex("PubKey");
b.ToTable("Payments");
});
modelBuilder.Entity("NostrStreamer.Database.PushSubscription", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Auth")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Created")
.HasColumnType("timestamp with time zone");
b.Property<string>("Endpoint")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Key")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("LastUsed")
.HasColumnType("timestamp with time zone");
b.Property<string>("Pubkey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("Scope")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("PushSubscriptions");
});
modelBuilder.Entity("NostrStreamer.Database.PushSubscriptionTarget", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("SubscriberPubkey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("TargetPubkey")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasKey("Id");
b.HasIndex("TargetPubkey");
b.HasIndex("SubscriberPubkey", "TargetPubkey")
.IsUnique();
b.ToTable("PushSubscriptionTargets");
});
modelBuilder.Entity("NostrStreamer.Database.User", b =>
{
b.Property<string>("PubKey")
.HasColumnType("text");
b.Property<long>("Balance")
.HasColumnType("bigint");
b.Property<string>("ContentWarning")
.HasColumnType("text");
b.Property<string>("Goal")
.HasColumnType("text");
b.Property<string>("Image")
.HasColumnType("text");
b.Property<string>("StreamKey")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Summary")
.HasColumnType("text");
b.Property<string>("Tags")
.HasColumnType("text");
b.Property<string>("Title")
.HasColumnType("text");
b.Property<DateTime?>("TosAccepted")
.HasColumnType("timestamp with time zone");
b.Property<uint>("Version")
.IsConcurrencyToken()
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("xid")
.HasColumnName("xmin");
b.HasKey("PubKey");
b.ToTable("Users");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("EdgeIp")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("EndpointId")
.HasColumnType("uuid");
b.Property<DateTime?>("Ends")
.HasColumnType("timestamp with time zone");
b.Property<string>("Event")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ForwardClientId")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("LastSegment")
.HasColumnType("timestamp with time zone");
b.Property<decimal>("Length")
.HasColumnType("numeric");
b.Property<decimal>("MilliSatsCollected")
.HasColumnType("numeric");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Starts")
.HasColumnType("timestamp with time zone");
b.Property<int>("State")
.HasColumnType("integer");
b.Property<string>("StreamId")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Thumbnail")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("EndpointId");
b.HasIndex("PubKey");
b.ToTable("Streams");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamClip", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<DateTime>("Created")
.HasColumnType("timestamp with time zone");
b.Property<string>("TakenByPubkey")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("UserStreamId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("UserStreamId");
b.ToTable("Clips");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamForwards", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Target")
.IsRequired()
.HasColumnType("text");
b.Property<string>("UserPubkey")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("UserPubkey");
b.ToTable("Forwards");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamGuest", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("PubKey")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Relay")
.HasColumnType("text");
b.Property<string>("Role")
.HasColumnType("text");
b.Property<string>("Sig")
.HasColumnType("text");
b.Property<Guid>("StreamId")
.HasColumnType("uuid");
b.Property<decimal>("ZapSplit")
.HasColumnType("numeric");
b.HasKey("Id");
b.HasIndex("StreamId", "PubKey")
.IsUnique();
b.ToTable("Guests");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamRecording", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<double>("Duration")
.HasColumnType("double precision");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<string>("Url")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("UserStreamId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("UserStreamId");
b.ToTable("Recordings");
});
modelBuilder.Entity("NostrStreamer.Database.Payment", b =>
{
b.HasOne("NostrStreamer.Database.User", "User")
.WithMany("Payments")
.HasForeignKey("PubKey")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.HasOne("NostrStreamer.Database.IngestEndpoint", "Endpoint")
.WithMany()
.HasForeignKey("EndpointId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("NostrStreamer.Database.User", "User")
.WithMany("Streams")
.HasForeignKey("PubKey")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Endpoint");
b.Navigation("User");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamClip", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "UserStream")
.WithMany()
.HasForeignKey("UserStreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("UserStream");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamForwards", b =>
{
b.HasOne("NostrStreamer.Database.User", "User")
.WithMany("Forwards")
.HasForeignKey("UserPubkey")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("User");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamGuest", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "Stream")
.WithMany("Guests")
.HasForeignKey("StreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Stream");
});
modelBuilder.Entity("NostrStreamer.Database.UserStreamRecording", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "Stream")
.WithMany("Recordings")
.HasForeignKey("UserStreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Stream");
});
modelBuilder.Entity("NostrStreamer.Database.User", b =>
{
b.Navigation("Forwards");
b.Navigation("Payments");
b.Navigation("Streams");
});
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.Navigation("Guests");
b.Navigation("Recordings");
});
#pragma warning restore 612, 618
}
}
}

22
NostrStreamer/Migrations/20240226120459_StreamRecordings.cs Normal file
View File

@ -0,0 +1,22 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace NostrStreamer.Migrations
{
/// <inheritdoc />
public partial class StreamRecordings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}

4
NostrStreamer/Migrations/StreamerContextModelSnapshot.cs
View File

@ -434,7 +434,7 @@ namespace NostrStreamer.Migrations
modelBuilder.Entity("NostrStreamer.Database.UserStreamRecording", b =>
{
b.HasOne("NostrStreamer.Database.UserStream", "Stream")
.WithMany()
.WithMany("Recordings")
.HasForeignKey("UserStreamId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
@ -454,6 +454,8 @@ namespace NostrStreamer.Migrations
modelBuilder.Entity("NostrStreamer.Database.UserStream", b =>
{
b.Navigation("Guests");
b.Navigation("Recordings");
});
#pragma warning restore 612, 618
}

1
NostrStreamer/Program.cs
View File

@ -105,6 +105,7 @@ internal static class Program
// dvr services
services.AddTransient<IDvrStore, S3DvrStore>();
services.AddHostedService<RecordingDeleter>();
// thumbnail services
services.AddTransient<IThumbnailService, S3ThumbnailService>();

70
NostrStreamer/Services/Background/RecordingDeleter.cs Normal file
View File

@ -0,0 +1,70 @@
using Microsoft.EntityFrameworkCore;
using NostrStreamer.Database;
using NostrStreamer.Services.Dvr;
namespace NostrStreamer.Services.Background;
public class RecordingDeleter : BackgroundService
{
private readonly ILogger<RecordingDeleter> _logger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly Config _config;
public RecordingDeleter(ILogger<RecordingDeleter> logger, IServiceScopeFactory scopeFactory, Config config)
{
_logger = logger;
_scopeFactory = scopeFactory;
_config = config;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
await using var db = scope.ServiceProvider.GetRequiredService<StreamerContext>();
var dvrStore = scope.ServiceProvider.GetRequiredService<IDvrStore>();
var olderThan = DateTime.UtcNow.Subtract(TimeSpan.FromDays(_config.RetainRecordingsDays));
var toDelete = await db.Streams
.AsNoTracking()
.Where(a => a.Starts < olderThan)
.Where(a => a.Recordings.Count > 0)
.ToListAsync(cancellationToken: stoppingToken);
_logger.LogInformation("Starting delete of {n:###0} stream recordings", toDelete.Count);
foreach (var stream in toDelete)
{
try
{
var streamRecordings = await db.Streams
.AsNoTracking()
.Include(a => a.Recordings)
.SingleAsync(a => a.Id == stream.Id, cancellationToken: stoppingToken);
var deleted = await dvrStore.DeleteRecordings(streamRecordings);
await db.Recordings
.Where(a => deleted.Contains(a.Id))
.ExecuteDeleteAsync(cancellationToken: stoppingToken);
_logger.LogInformation("Deleted {n}/{m} recordings from stream {id}", deleted.Count,
streamRecordings.Recordings.Count, stream.Id);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete stream recordings {id} {msg}", stream.Id, ex.Message);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to run background deleter");
}
await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken);
}
}
}

7
NostrStreamer/Services/Dvr/IDvrStore.cs
View File

@ -11,6 +11,13 @@ public interface IDvrStore
/// <param name="source"></param>
/// <returns></returns>
Task<UploadResult> UploadRecording(UserStream stream, Uri source);
/// <summary>
/// Delete all recordings from the storage by stream
/// </summary>
/// <param name="stream"></param>
/// <returns>List of deleted recordings</returns>
Task<List<Guid>> DeleteRecordings(UserStream stream);
}
public record UploadResult(Uri Result, double Duration);

26
NostrStreamer/Services/Dvr/S3DvrStore.cs
View File

@ -86,10 +86,30 @@ public class S3DvrStore : IDvrStore
// cleanup temp file
fs.Close();
File.Delete(tmpFile);
_logger.LogInformation("download={tc:#,##0}ms, probe={pc:#,##0}ms, upload={uc:#,##0}ms", tsDownload.TotalMilliseconds,
_logger.LogInformation("download={tc:#,##0}ms, probe={pc:#,##0}ms, upload={uc:#,##0}ms",
tsDownload.TotalMilliseconds,
tsProbe.TotalMilliseconds, tsUpload.TotalMilliseconds);
return new(ub.Uri, probe.Duration.TotalSeconds);
}
}
public async Task<List<Guid>> DeleteRecordings(UserStream stream)
{
var deleted = new List<Guid>();
foreach (var batch in stream.Recordings.Select((a, i) => (Batch: i / 1000, Item: a)).GroupBy(a => a.Batch))
{
var res = await _client.DeleteObjectsAsync(new()
{
BucketName = _config.BucketName,
Objects = batch.Select(a => new KeyVersion()
{
Key = $"{stream.Id}/{a.Item.Id}.ts"
}).ToList()
});
deleted.AddRange(res.DeletedObjects.Select(a => Guid.Parse(Path.GetFileNameWithoutExtension(a.Key))));
}
return deleted;
}
}

18
NostrStreamer/Services/StreamEventBuilder.cs
View File

@ -102,8 +102,24 @@ public class StreamEventBuilder
return ev.Sign(pk);
}
public NostrEvent CreateDm(UserStream stream, string message)
{
var pk = NostrPrivateKey.FromBech32(_config.PrivateKey);
var ev = new NostrEvent
{
Kind = NostrKind.EncryptedDm,
Content = message,
CreatedAt = DateTime.Now,
Tags = new NostrEventTags(
new NostrEventTag("p", stream.PubKey)
)
};
return ev.EncryptDirect(pk, NostrPublicKey.FromHex(stream.PubKey)).Sign(pk);
}
public void BroadcastEvent(NostrEvent ev)
{
_nostrClient.Send(new NostrEventRequest(ev));
}
}
}

10
NostrStreamer/Services/StreamManager/NostrStreamManager.cs
View File

@ -107,6 +107,16 @@ public class NostrStreamManager : IStreamManager
_logger.LogInformation("Stream stopped for: {pubkey}", _context.User.PubKey);
await UpdateStreamState(UserStreamState.Ended);
// send DM with link to summary
var msg =
(_context.UserStream.Thumbnail != default ? $"{_context.UserStream.Thumbnail}\n" : "") +
$"Your stream summary is available here: https://zap.stream/summary/{_context.UserStream.ToIdentifier().ToBech32()}\n\n" +
$"You paid {_context.UserStream.MilliSatsCollected / 1000:#,##0.###} sats for this stream!\n\n" +
$"You streamed for {_context.UserStream.Length / 60:#,##0} mins!";
var chat = _eventBuilder.CreateDm(_context.UserStream, msg);
_eventBuilder.BroadcastEvent(chat);
}
public async Task ConsumeQuota(double duration)

10
NostrStreamer/appsettings.json
View File

@ -29,13 +29,13 @@
"MacaroonPath": "/Users/kieran/.polar/networks/1/volumes/lnd/bob/data/chain/bitcoin/regtest/admin.macaroon"
},
"S3Store": {
"ServiceUrl": "http://localhost:9010",
"AccessKey": "TQcxug1ZAXfnZ5bvc9n5",
"SecretKey": "p7EK4qew6DBkBPqrpRPuJgTOc6ChUlfIcEdAwE7K",
"PublicHost": "http://localhost:9010"
"ServiceUrl": "http://localhost:9000",
"AccessKey": "GfvDg9FoarDnNZj5axOq",
"SecretKey": "1XtAmUcJDPHWEarFLmOx75z6Ok3FyMEdP7cvCkQP",
"PublicHost": "http://localhost:9000"
},
"SnortApi": "https://api.snort.social",
"GeoIpDatabase": "/Users/kieran/Downloads/GeoLite2-City_20230801/GeoLite2-City.mmdb",
"GeoIpDatabase": "C:\\Users\\Kieran\\Downloads\\GeoLite2-City.mmdb",
"Edges": [
{
"Name": "US0",

11
docker-compose.yaml
View File

@ -1,5 +1,6 @@
volumes:
minio-dvr:
postgres:
services:
srs-origin:
image: ossrs/srs:5
@ -29,18 +30,20 @@ services:
- "POSTGRES_HOST_AUTH_METHOD=trust"
ports:
- "5431:5432"
volumes:
- "postgres:/var/lib/postgresql/data"
minio:
image: quay.io/minio/minio
command:
- "server"
- "/data"
- "--console-address"
- ":9001"
- ":9011"
environment:
- "MINIO_SERVER_URL=http://localhost:9010"
- "MINIO_SERVER_URL=http://localhost:9000"
ports:
- "9010:9000"
- "9011:9001"
- "9000:9000"
- "9011:9011"
volumes:
- "minio-dvr:/data"
redis: