From 53b187d250acd8e6e710f1b9468d5451f5211fd3 Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 10 Jan 2024 09:56:41 +0000 Subject: [PATCH] Init --- .dockerignore | 25 ++ .gitignore | 5 + NostrServices.sln | 21 ++ NostrServices/Config.cs | 6 + NostrServices/Controllers/ImportController.cs | 28 +++ .../Controllers/LinkPreviewController.cs | 123 +++++++++ .../Controllers/OpenGraphController.cs | 235 ++++++++++++++++++ NostrServices/Database/Configuration/Relay.cs | 23 ++ .../Database/NostrServicesContext.cs | 21 ++ NostrServices/Database/Relay.cs | 17 ++ NostrServices/Dockerfile | 23 ++ NostrServices/Extensions.cs | 54 ++++ .../20240110094022_Relay.Designer.cs | 54 ++++ .../Migrations/20240110094022_Relay.cs | 37 +++ NostrServices/NostrServices.csproj | 32 +++ NostrServices/Program.cs | 99 ++++++++ NostrServices/Properties/launchSettings.json | 15 ++ .../Services/EventHandlers/IEventHandler.cs | 23 ++ .../Services/EventHandlers/RedisEventCache.cs | 29 +++ .../EventHandlers/RedisStreamPublisher.cs | 22 ++ NostrServices/Services/NostrListener.cs | 136 ++++++++++ NostrServices/Services/RedisStore.cs | 155 ++++++++++++ NostrServices/Services/RelayListener.cs | 105 ++++++++ NostrServices/appsettings.Development.json | 8 + NostrServices/appsettings.json | 15 ++ docker-compose.yml | 12 + 26 files changed, 1323 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 NostrServices.sln create mode 100644 NostrServices/Config.cs create mode 100644 NostrServices/Controllers/ImportController.cs create mode 100644 NostrServices/Controllers/LinkPreviewController.cs create mode 100644 NostrServices/Controllers/OpenGraphController.cs create mode 100644 NostrServices/Database/Configuration/Relay.cs create mode 100644 NostrServices/Database/NostrServicesContext.cs create mode 100644 NostrServices/Database/Relay.cs create mode 100644 NostrServices/Dockerfile create mode 100644 NostrServices/Extensions.cs create mode 100644 NostrServices/Migrations/20240110094022_Relay.Designer.cs create mode 100644 NostrServices/Migrations/20240110094022_Relay.cs create mode 100644 NostrServices/NostrServices.csproj create mode 100644 NostrServices/Program.cs create mode 100644 NostrServices/Properties/launchSettings.json create mode 100644 NostrServices/Services/EventHandlers/IEventHandler.cs create mode 100644 NostrServices/Services/EventHandlers/RedisEventCache.cs create mode 100644 NostrServices/Services/EventHandlers/RedisStreamPublisher.cs create mode 100644 NostrServices/Services/NostrListener.cs create mode 100644 NostrServices/Services/RedisStore.cs create mode 100644 NostrServices/Services/RelayListener.cs create mode 100644 NostrServices/appsettings.Development.json create mode 100644 NostrServices/appsettings.json create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cd967fc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.idea +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..add57be --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +bin/ +obj/ +/packages/ +riderModule.iml +/_ReSharper.Caches/ \ No newline at end of file diff --git a/NostrServices.sln b/NostrServices.sln new file mode 100644 index 0000000..7cea7d8 --- /dev/null +++ b/NostrServices.sln @@ -0,0 +1,21 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NostrServices", "NostrServices\NostrServices.csproj", "{51CD83E4-2CEC-4852-B285-2327EF86C6E7}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{52604625-FC24-4D1E-A152-637973A967F9}" + ProjectSection(SolutionItems) = preProject + docker-compose.yml = docker-compose.yml + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {51CD83E4-2CEC-4852-B285-2327EF86C6E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {51CD83E4-2CEC-4852-B285-2327EF86C6E7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {51CD83E4-2CEC-4852-B285-2327EF86C6E7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {51CD83E4-2CEC-4852-B285-2327EF86C6E7}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/NostrServices/Config.cs b/NostrServices/Config.cs new file mode 100644 index 0000000..133929d --- /dev/null +++ b/NostrServices/Config.cs @@ -0,0 +1,6 @@ +namespace NostrServices; + +public class Config +{ + public string Redis { get; init; } = null!; +} diff --git a/NostrServices/Controllers/ImportController.cs b/NostrServices/Controllers/ImportController.cs new file mode 100644 index 0000000..35017a1 --- /dev/null +++ b/NostrServices/Controllers/ImportController.cs @@ -0,0 +1,28 @@ +using Microsoft.AspNetCore.Mvc; +using Nostr.Client.Messages; +using NostrServices.Services; + +namespace NostrServices.Controllers; + +[Route("/api/v1/import")] +public class ImportController : Controller +{ + private readonly RedisStore _redisStore; + + public ImportController(RedisStore redisStore) + { + _redisStore = redisStore; + } + + [HttpPost] + [Consumes("text/json", "application/json")] + public async Task ImportEvent([FromBody] NostrEvent ev) + { + if (await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev))) + { + return Accepted(); + } + + return Ok(); + } +} diff --git a/NostrServices/Controllers/LinkPreviewController.cs b/NostrServices/Controllers/LinkPreviewController.cs new file mode 100644 index 0000000..78c6c24 --- /dev/null +++ b/NostrServices/Controllers/LinkPreviewController.cs @@ -0,0 +1,123 @@ +using System.Security.Cryptography; +using System.Text; +using AngleSharp; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Memory; +using Newtonsoft.Json; +using ProtoBuf; +using StackExchange.Redis; + +namespace NostrServices.Controllers; + +[Route("/api/v1/preview")] +public class LinkPreviewController : Controller +{ + private readonly IDatabase _database; + private readonly IMemoryCache _memoryCache; + private readonly HttpClient _client; + private readonly ILogger _logger; + + public LinkPreviewController(HttpClient client, ILogger logger, IDatabase database, IMemoryCache memoryCache) + { + _client = client; + _logger = logger; + _database = database; + _memoryCache = memoryCache; + _client.DefaultRequestHeaders.Add("user-agent", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Snort/1.0 (LinkPreview; https://snort.social)"); + + _client.Timeout = TimeSpan.FromSeconds(30); + } + + [HttpGet] + [ResponseCache(Duration = 21600, VaryByQueryKeys = new[] {"url"}, Location = ResponseCacheLocation.Any)] + public async Task GetPreview([FromQuery] string url) + { + var urlHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(url.ToLower()))); + var key = $"link-preview:{urlHash}"; + var keyEmpty = $"{key}:empty"; + if (_memoryCache.Get(keyEmpty)) + { + return default; + } + + var cached = await _database.GetAsync(key); + if (cached != default) + { + return cached; + } + + var cts = HttpContext.RequestAborted; + try + { + var rsp = await _client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cts); + if (rsp.IsSuccessStatusCode && rsp.Content.Headers.ContentType?.MediaType == "text/html") + { + var body = await rsp.Content.ReadAsStringAsync(cts); + var config = Configuration.Default; + var context = BrowsingContext.New(config); + var doc = await context.OpenAsync(c => c.Content(body), cts); + var ogTags = doc.Head?.QuerySelectorAll("meta[property*='og']").Select(a => + { + var k = a.Attributes["property"]?.Value; + var v = a.Attributes["content"]?.Value; + if (!string.IsNullOrEmpty(k) && !string.IsNullOrEmpty(v)) + { + return new KeyValuePair(k, v); + } + + return default; + }).Where(a => !string.IsNullOrEmpty(a.Key)).ToArray(); + + var obj = new LinkPreviewData + { + OgTags = ogTags?.Select(a => new KeyValuePair(a.Key, a.Value)).ToList() ?? new(), + Title = ogTags?.FirstOrDefault(a => a.Key.Equals("og:title")).Value + ?? doc.Head?.QuerySelector("title")?.TextContent, + Description = ogTags?.FirstOrDefault(a => a.Key.Equals("og:description")).Value + ?? doc.Head?.QuerySelector("meta[name='description']")?.Attributes["content"]?.Value, + Image = ogTags?.FirstOrDefault(a => a.Key.Equals("og:image")).Value + }; + + await _database.SetAsync(key, obj, TimeSpan.FromDays(7)); + return obj; + } + } + catch (HttpRequestException hx) + { + _logger.LogWarning("{url} returned {status}", url, hx.StatusCode); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch content at {url}", url); + } + + // cache no result in memory to avoid wasting time on repeated attempts + _memoryCache.Set(keyEmpty, true, TimeSpan.FromMinutes(10)); + return default; + } +} + +[ProtoContract] +public class LinkPreviewData +{ + [ProtoMember(1)] + [System.Text.Json.Serialization.JsonIgnore] + public List> OgTags { get; init; } = new(); + + [ProtoIgnore] + [JsonProperty("og_tags")] + public List OgTagsJson => OgTags.Select(a => new[] {a.Key, a.Value}).ToList(); + + [ProtoMember(2)] + [JsonProperty("title")] + public string? Title { get; init; } + + [ProtoMember(3)] + [JsonProperty("description")] + public string? Description { get; init; } + + [ProtoMember(4)] + [JsonProperty("image")] + public string? Image { get; init; } +} diff --git a/NostrServices/Controllers/OpenGraphController.cs b/NostrServices/Controllers/OpenGraphController.cs new file mode 100644 index 0000000..c2c00c6 --- /dev/null +++ b/NostrServices/Controllers/OpenGraphController.cs @@ -0,0 +1,235 @@ +using AngleSharp; +using AngleSharp.Dom; +using Microsoft.AspNetCore.Mvc; +using Nostr.Client.Identifiers; +using Nostr.Client.Messages; +using Nostr.Client.Utils; +using NostrServices.Services; +using ProtoBuf; + +namespace NostrServices.Controllers; + +/// +/// Add OpenGraph tags to html documents +/// +[Route("/api/v1/opengraph")] +public class OpenGraphController : Controller +{ + private readonly ILogger _logger; + private readonly RedisStore _redisStore; + + public OpenGraphController(ILogger logger, RedisStore redisStore) + { + _logger = logger; + _redisStore = redisStore; + } + + /// + /// Inject opengraph tags into provided html + /// + /// Nostr identifier npub/note/nevent/naddr/nprofile + /// Url format for canonical tag https://example.com/%s + /// + [HttpPost("{id}")] + [Consumes("text/html")] + [Produces("text/html")] + public async Task TagPage([FromRoute] string id, [FromQuery] string? canonical) + { + var cts = HttpContext.RequestAborted; + using var sr = new StreamReader(Request.Body); + var html = await sr.ReadToEndAsync(cts); + + void AddCanonical(List tags) + { + if (!string.IsNullOrEmpty(canonical) && canonical.Contains("%s")) + { + var uc = new Uri(canonical.Replace("%s", id)); + tags.Add(new HeadElement("link", [ + new("rel", "canonical"), + new("href", uc.ToString()) + ])); + } + } + + if (NostrIdentifierParser.TryParse(id, out var nid)) + { + try + { + if (nid!.Hrp is "nevent" or "note") + { + var ev = await _redisStore.GetEvent(nid); + if (ev != default) + { + var tags = MetaTagsToElements(await GetEventTags(ev)); + AddCanonical(tags); + + var doc = await InjectTags(html, tags); + return Content(doc?.ToHtml() ?? html, "text/html"); + } + } + else if (nid.Hrp is "nprofile" or "npub") + { + var meta = await GetProfileMeta(id); + if (meta != default) + { + var tags = MetaTagsToElements([ + new("og:type", "profile"), + new("og:title", meta.Title ?? ""), + new("og:description", meta.Description ?? ""), + new("og:image", meta.Image ?? ""), + new("og:profile:username", meta.Profile?.Name ?? "") + ]); + + AddCanonical(tags); + var doc = await InjectTags(html, tags); + + return Content(doc?.ToHtml() ?? html, "text/html"); + } + } + } + catch (Exception ex) when (ex is not TaskCanceledException) + { + _logger.LogWarning("Failed to inject event tags: {Message}", ex.Message); + } + } + + return Content(html, "text/html"); + } + + private async Task>> GetEventTags(CompactEvent ev) + { + var ret = new List>(); + + var profile = await _redisStore.GetProfile(ev.PubKey.ToHex()); + var name = profile?.Name ?? "Nostrich"; + switch (ev.Kind) + { + case (long)NostrKind.LiveEvent: + { + var host = ev.Tags.FirstOrDefault(a => a.Key is "p" && a.Values[3] is "host")?.Values[1] ?? ev.PubKey.ToHex(); + var hostProfile = await _redisStore.GetProfile(host); + var hostName = hostProfile?.Name ?? profile?.Name ?? "Nostrich"; + var stream = ev.GetFirstTagValue("streaming") ?? ev.GetFirstTagValue("recording") ?? ""; + ret.AddRange(new KeyValuePair[] + { + new("og:type", "video.other"), + new("og:title", $"{hostName} is streaming"), + new("og:description", ev.GetFirstTagValue("title") ?? ""), + new("og:image", + ev.GetFirstTagValue("image") ?? hostProfile?.Picture ?? $"https://robohash.v0l.io/{ev.PubKey.ToHex()}.png"), + new("og:video", stream), + new("og:video:secure_url", stream), + new("og:video:type", "application/vnd.apple.mpegurl"), + }); + + break; + } + case 1_313: + { + var stream = ev.GetFirstTagValue("r")!; + ret.AddRange(new KeyValuePair[] + { + new("og:type", "video.other"), + new("og:title", $"{name} created a clip"), + new("og:description", ev.GetFirstTagValue("title") ?? ""), + new("og:image", + ev.GetFirstTagValue("image") ?? profile?.Picture ?? $"https://robohash.v0l.io/{ev.PubKey.ToHex()}.png"), + new("og:video", stream), + new("og:video:secure_url", stream), + new("og:video:type", "video/mp4"), + }); + + break; + } + default: + { + const int maxLen = 160; + var trimmedContent = ev.Content.Length > maxLen ? ev.Content[..maxLen] : ev.Content; + var titleContent = $"{profile}: {trimmedContent}"; + ret.AddRange(new KeyValuePair[] + { + new("og:type", "article"), + new("og:title", titleContent), + new("og:description", ""), + new("og:image", profile?.Picture ?? $"https://robohash.v0l.io/{ev.PubKey.ToHex()}.png"), + new("og:article:published_time", ev.Created.ToString("o")), + new("og:article:author:username", profile?.Name ?? ""), + }); + + break; + } + } + + return ret; + } + + private async Task GetProfileMeta(string id) + { + var profile = await _redisStore.GetProfile(id); + var titleContent = $"Snort - {profile?.Name ?? "Nostrich"}'s Profile"; + var aboutContent = profile?.About?.Length > 160 ? profile.About[..160] : profile?.About ?? ""; + var imageContent = profile?.Picture ?? "https://snort.social/nostrich_512.png"; + return new CachedMeta + { + Title = titleContent, + Description = aboutContent, + Image = imageContent, + Profile = profile + }; + } + + private async Task InjectTags(string html, List tags) + { + var config = Configuration.Default; + var context = BrowsingContext.New(config); + var doc = await context.OpenAsync(c => c.Content(html)); + + foreach (var ex in tags) + { + var tag = doc.CreateElement(ex.Element); + foreach (var attr in ex.Attributes) + { + tag.SetAttribute(attr.Key, attr.Value); + } + + doc.Head?.AppendChild(tag); + } + + return doc; + } + + private List MetaTagsToElements(List> tags) + { + var ret = new List(); + foreach (var tag in tags) + { + ret.Add(new("meta", [ + new("property", tag.Key), + new("content", tag.Value) + ])); + } + + return ret; + } + + record HeadElement(string Element, List> Attributes); +} + +[ProtoContract] +class CachedMeta +{ + [ProtoMember(1)] + public string? Title { get; init; } + + [ProtoMember(2)] + public string? Description { get; init; } + + [ProtoMember(3)] + public string? Image { get; init; } + + [ProtoMember(4)] + public CompactProfile? Profile { get; init; } + + [ProtoMember(5)] + public CompactEvent? Event { get; init; } +} diff --git a/NostrServices/Database/Configuration/Relay.cs b/NostrServices/Database/Configuration/Relay.cs new file mode 100644 index 0000000..0ff68c8 --- /dev/null +++ b/NostrServices/Database/Configuration/Relay.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace NostrServices.Database.Configuration; + +public class RelayConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(a => a.Id); + builder.Property(a => a.Url) + .IsRequired(); + + builder.Property(a => a.FirstSeen) + .IsRequired(); + + builder.Property(a => a.LastSeen) + .IsRequired(); + + builder.Property(a => a.IsAnyCast) + .IsRequired(); + } +} diff --git a/NostrServices/Database/NostrServicesContext.cs b/NostrServices/Database/NostrServicesContext.cs new file mode 100644 index 0000000..f5ecafa --- /dev/null +++ b/NostrServices/Database/NostrServicesContext.cs @@ -0,0 +1,21 @@ +using Microsoft.EntityFrameworkCore; + +namespace NostrServices.Database; + +public class NostrServicesContext : DbContext +{ + public NostrServicesContext() + { + } + + public NostrServicesContext(DbContextOptions ctx) : base(ctx) + { + } + + public DbSet Relays => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.ApplyConfigurationsFromAssembly(typeof(NostrServicesContext).Assembly); + } +} diff --git a/NostrServices/Database/Relay.cs b/NostrServices/Database/Relay.cs new file mode 100644 index 0000000..4ac853f --- /dev/null +++ b/NostrServices/Database/Relay.cs @@ -0,0 +1,17 @@ +namespace NostrServices.Database; + +public class Relay +{ + public Guid Id { get; init; } = Guid.NewGuid(); + + public Uri Url { get; init; } = null!; + + public DateTime FirstSeen { get; init; } = DateTime.UtcNow; + + public DateTime LastSeen { get; init; } + + /// + /// If this relay uses any cast IPs which will naturally obscure its location + /// + public bool IsAnyCast { get; init; } +} diff --git a/NostrServices/Dockerfile b/NostrServices/Dockerfile new file mode 100644 index 0000000..c2c8afa --- /dev/null +++ b/NostrServices/Dockerfile @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base +USER $APP_UID +WORKDIR /app +EXPOSE 8080 +EXPOSE 8081 + +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["NostrServices/NostrServices.csproj", "NostrServices/"] +RUN dotnet restore "NostrServices/NostrServices.csproj" +COPY . . +WORKDIR "/src/NostrServices" +RUN dotnet build "NostrServices.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "NostrServices.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "NostrServices.dll"] diff --git a/NostrServices/Extensions.cs b/NostrServices/Extensions.cs new file mode 100644 index 0000000..99548b7 --- /dev/null +++ b/NostrServices/Extensions.cs @@ -0,0 +1,54 @@ +using Nostr.Client.Identifiers; +using Nostr.Client.Messages; +using Nostr.Client.Utils; +using NostrServices.Services; +using ProtoBuf; +using StackExchange.Redis; + +namespace NostrServices; + +public static class Extensions +{ + public static string? GetFirstTagValue(this CompactEvent ev, string key) + { + return ev.Tags.FirstOrDefault(a => a.Key == key)?.Values[0]; + } + + public static async Task SetAsync(this IDatabase db, RedisKey key, T val, TimeSpan? expire = null) + { + using var ms = new MemoryStream(); + Serializer.Serialize(ms, val); + return await db.StringSetAsync(key, ms.ToArray(), expire); + } + + public static async Task GetAsync(this IDatabase db, RedisKey key) where T : class + { + var data = await db.StringGetAsync(key); + if (data is {HasValue: true, IsNullOrEmpty: false}) + { + return Serializer.Deserialize(((byte[])data!).AsSpan()); + } + + return default; + } + + public static NostrIdentifier ToIdentifier(this NostrEvent ev) + { + if ((long)ev.Kind is >= 30_000 and < 40_000) + { + return new NostrAddressIdentifier(ev.Tags!.FindFirstTagValue("d")!, ev.Pubkey!, [], ev.Kind); + } + + return new NostrEventIdentifier(ev.Id!, ev.Pubkey, [], ev.Kind); + } + + public static NostrIdentifier ToIdentifier(this CompactEvent ev) + { + if (ev.Kind is >= 30_000 and < 40_000) + { + return new NostrAddressIdentifier(ev.GetFirstTagValue("d")!, ev.PubKey.ToHex(), [], (NostrKind)ev.Kind); + } + + return new NostrEventIdentifier(ev.Id.ToHex(), ev.PubKey.ToHex(), [], (NostrKind)ev.Kind); + } +} diff --git a/NostrServices/Migrations/20240110094022_Relay.Designer.cs b/NostrServices/Migrations/20240110094022_Relay.Designer.cs new file mode 100644 index 0000000..979c4ec --- /dev/null +++ b/NostrServices/Migrations/20240110094022_Relay.Designer.cs @@ -0,0 +1,54 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NostrServices.Database; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace NostrServices.Migrations +{ + [DbContext(typeof(NostrServicesContext))] + [Migration("20240110094022_Relay")] + partial class Relay + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.1") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("NostrServices.Database.Relay", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FirstSeen") + .HasColumnType("timestamp with time zone"); + + b.Property("IsAnyCast") + .HasColumnType("boolean"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Relays"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/NostrServices/Migrations/20240110094022_Relay.cs b/NostrServices/Migrations/20240110094022_Relay.cs new file mode 100644 index 0000000..f86fd59 --- /dev/null +++ b/NostrServices/Migrations/20240110094022_Relay.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace NostrServices.Migrations +{ + /// + public partial class Relay : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "Relays", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Url = table.Column(type: "text", nullable: false), + FirstSeen = table.Column(type: "timestamp with time zone", nullable: false), + LastSeen = table.Column(type: "timestamp with time zone", nullable: false), + IsAnyCast = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Relays", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "Relays"); + } + } +} diff --git a/NostrServices/NostrServices.csproj b/NostrServices/NostrServices.csproj new file mode 100644 index 0000000..af885d5 --- /dev/null +++ b/NostrServices/NostrServices.csproj @@ -0,0 +1,32 @@ + + + + net8.0 + enable + enable + Linux + + + + + .dockerignore + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + diff --git a/NostrServices/Program.cs b/NostrServices/Program.cs new file mode 100644 index 0000000..e0fe8b2 --- /dev/null +++ b/NostrServices/Program.cs @@ -0,0 +1,99 @@ +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; +using Nostr.Client.Client; +using NostrServices.Database; +using NostrServices.Services; +using NostrServices.Services.EventHandlers; +using StackExchange.Redis; + +namespace NostrServices; + +public static class Program +{ + public static async Task Main(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + var config = builder.Configuration.GetSection("Config").Get()!; + + var cx = await ConnectionMultiplexer.ConnectAsync(config.Redis); + builder.Services.AddSingleton(cx); + + //builder.Services.AddSingleton(_ => new DatabaseReader(cfg.GeoIpDatabase)); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(s => s.GetRequiredService()); + builder.Services.AddSingleton(); + + builder.Services.AddTransient(svc => svc.GetRequiredService().GetDatabase()); + builder.Services.AddTransient(svc => svc.GetRequiredService().GetSubscriber()); + builder.Services.AddTransient(); + builder.Services.AddNostrEventHandlers(); + + builder.Services.AddControllers().AddNewtonsoftJson(opt => + { + opt.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; + opt.SerializerSettings.Formatting = Formatting.None; + opt.SerializerSettings.NullValueHandling = NullValueHandling.Ignore; + opt.SerializerSettings.ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor; + opt.SerializerSettings.Converters = + [ + new UnixDateTimeConverter() + ]; + + opt.SerializerSettings.ContractResolver = new CamelCasePropertyNamesContractResolver(); + }); + + builder.Services.AddEndpointsApiExplorer(); + builder.Services.AddSwaggerGen(); + builder.Services.AddHttpClient(); + builder.Services.AddMemoryCache(); + builder.Services.AddResponseCaching(); + + builder.Services.AddHostedService(); + + ConfigureDb(builder.Services, builder.Configuration); + + var app = builder.Build(); + using (var scope = app.Services.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + await db.Database.MigrateAsync(); + } + + app.UseResponseCaching(); + app.UseSwagger(); + app.UseSwaggerUI(); + app.UseCors(o => + { + o.AllowAnyOrigin(); + o.AllowAnyHeader(); + o.AllowAnyMethod(); + }); + + app.UseRouting(); + app.MapControllers(); + + await app.RunAsync(); + } + + private static void ConfigureDb(IServiceCollection services, IConfiguration configuration) + { + services.AddDbContext(o => + o.UseNpgsql(configuration.GetConnectionString("Database"))); + } + + /// + /// Dummy method for EF core migrations + /// + /// + /// + // ReSharper disable once UnusedMember.Global + public static IHostBuilder CreateHostBuilder(string[] args) + { + var dummyHost = Host.CreateDefaultBuilder(args); + dummyHost.ConfigureServices((ctx, svc) => { ConfigureDb(svc, ctx.Configuration); }); + + return dummyHost; + } +} diff --git a/NostrServices/Properties/launchSettings.json b/NostrServices/Properties/launchSettings.json new file mode 100644 index 0000000..f48d81d --- /dev/null +++ b/NostrServices/Properties/launchSettings.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5228", + "launchUrl": "http://localhost:5228/swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/NostrServices/Services/EventHandlers/IEventHandler.cs b/NostrServices/Services/EventHandlers/IEventHandler.cs new file mode 100644 index 0000000..48f1330 --- /dev/null +++ b/NostrServices/Services/EventHandlers/IEventHandler.cs @@ -0,0 +1,23 @@ +using Nostr.Client.Messages; + +namespace NostrServices.Services.EventHandlers; + +public interface IEventHandler +{ + /// + /// Handle an event from the network + /// + /// + /// + ValueTask HandleEvent(NostrEvent ev); +} + + +public static class EventHandlerStartup +{ + public static void AddNostrEventHandlers(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + } +} \ No newline at end of file diff --git a/NostrServices/Services/EventHandlers/RedisEventCache.cs b/NostrServices/Services/EventHandlers/RedisEventCache.cs new file mode 100644 index 0000000..6077930 --- /dev/null +++ b/NostrServices/Services/EventHandlers/RedisEventCache.cs @@ -0,0 +1,29 @@ +using Nostr.Client.Messages; + +namespace NostrServices.Services.EventHandlers; + +public class RedisEventCache : IEventHandler +{ + private readonly RedisStore _redisStore; + + public RedisEventCache(RedisStore redisStore) + { + _redisStore = redisStore; + } + + public async ValueTask HandleEvent(NostrEvent ev) + { + if (ev.Kind is NostrKind.Metadata) + { + var p = CompactProfile.FromNostrEvent(ev); + if (p != default) + { + await _redisStore.StoreProfile(p); + } + } + else if (ev.Kind is NostrKind.ShortTextNote or NostrKind.LongFormContent or NostrKind.LiveEvent or NostrKind.LiveChatMessage) + { + await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev)); + } + } +} diff --git a/NostrServices/Services/EventHandlers/RedisStreamPublisher.cs b/NostrServices/Services/EventHandlers/RedisStreamPublisher.cs new file mode 100644 index 0000000..07f33a3 --- /dev/null +++ b/NostrServices/Services/EventHandlers/RedisStreamPublisher.cs @@ -0,0 +1,22 @@ +using Newtonsoft.Json; +using Nostr.Client.Json; +using Nostr.Client.Messages; +using StackExchange.Redis; + +namespace NostrServices.Services.EventHandlers; + +public class RedisStreamPublisher : IEventHandler +{ + private readonly IDatabase _redis; + + public RedisStreamPublisher(IDatabase redis) + { + _redis = redis; + } + + public async ValueTask HandleEvent(NostrEvent ev) + { + var json = JsonConvert.SerializeObject(ev, NostrSerializer.Settings); + await _redis.PublishAsync("event-stream", json); + } +} diff --git a/NostrServices/Services/NostrListener.cs b/NostrServices/Services/NostrListener.cs new file mode 100644 index 0000000..0731578 --- /dev/null +++ b/NostrServices/Services/NostrListener.cs @@ -0,0 +1,136 @@ +using System.Net.WebSockets; +using System.Reflection; +using Microsoft.EntityFrameworkCore; +using Nostr.Client.Client; +using Nostr.Client.Communicator; +using Nostr.Client.Requests; +using NostrServices.Database; +using Websocket.Client.Models; + +namespace NostrServices.Services; + +public class NostrListener : IDisposable +{ + private readonly NostrMultiWebsocketClient _client; + private readonly ILogger _logger; + + private readonly Dictionary _subscriptionToFilter = new(); + + public NostrListener(NostrMultiWebsocketClient client, ILogger logger) + { + _client = client; + _logger = logger; + } + + public NostrClientStreams Streams => _client.Streams; + + public void Dispose() + { + _client.Dispose(); + } + + public void RegisterFilter(string subscription, NostrFilter filter) + { + _subscriptionToFilter[subscription] = filter; + _client.Send(new NostrRequest(subscription, filter)); + } + + private void Stop() + { + foreach (var nostrWebsocketClient in _client.Clients) + { + _ = nostrWebsocketClient.Communicator.Stop(WebSocketCloseStatus.NormalClosure, string.Empty); + } + } + + private void AddCommunicators(List relays) + { + var missing = relays.Where(a => _client.FindClient(ClientName(a)) == null); + var newComm = missing.Select(CreateCommunicator); + foreach (var comm in newComm) + { + _client.RegisterCommunicator(comm); + comm.Start(); + } + } + + private INostrCommunicator CreateCommunicator(Uri uri) + { + var comm = new NostrWebsocketCommunicator(uri, () => + { + var client = new ClientWebSocket(); + client.Options.SetRequestHeader("Origin", "https://nostr-api.v0l.io/swagger"); + client.Options.SetRequestHeader("User-Agent", $"v0l/NostrServices ({Assembly.GetExecutingAssembly().GetName().Version})"); + return client; + }); + + comm.Name = ClientName(uri); + comm.ReconnectTimeout = null; //TimeSpan.FromSeconds(30); + comm.ErrorReconnectTimeout = TimeSpan.FromSeconds(60); + + comm.ReconnectionHappened.Subscribe(info => OnCommunicatorReconnection(info, comm.Name)); + comm.DisconnectionHappened.Subscribe(info => + _logger.LogWarning("[{relay}] Disconnected, type: {type}, reason: {reason}", comm.Name, info.Type, info.CloseStatus)); + + return comm; + } + + private string ClientName(Uri u) => $"{u.Host}{u.AbsolutePath}"; + + private void OnCommunicatorReconnection(ReconnectionInfo info, string communicatorName) + { + try + { + _logger.LogInformation("[{relay}] Reconnected, sending Nostr filters ({filterCount})", communicatorName, + _subscriptionToFilter.Count); + + var client = _client.FindClient(communicatorName); + if (client == null) + { + _logger.LogWarning("[{relay}] Cannot find client", communicatorName); + return; + } + + foreach (var (sub, filter) in _subscriptionToFilter) + { + client.Send(new NostrRequest(sub, filter)); + } + } + catch (Exception e) + { + _logger.LogError(e, "[{relay}] Failed to process reconnection, error: {error}", communicatorName, e.Message); + } + } + + public class NostrListenerLifetime : BackgroundService + { + private readonly NostrListener _nostrListener; + private readonly IServiceScopeFactory _scopeFactory; + + public NostrListenerLifetime(NostrListener nostrListener, IServiceScopeFactory scopeFactory) + { + _nostrListener = nostrListener; + _scopeFactory = scopeFactory; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + using var scope = _scopeFactory.CreateScope(); + await using var db = scope.ServiceProvider.GetRequiredService(); + + var relays = await db.Relays.Select(a => a.Url).ToListAsync(cancellationToken: stoppingToken); + _nostrListener.AddCommunicators(relays); + + await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken); + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await base.StopAsync(cancellationToken); + _nostrListener.Stop(); + } + } +} diff --git a/NostrServices/Services/RedisStore.cs b/NostrServices/Services/RedisStore.cs new file mode 100644 index 0000000..42936c4 --- /dev/null +++ b/NostrServices/Services/RedisStore.cs @@ -0,0 +1,155 @@ +using NBitcoin; +using Nostr.Client.Identifiers; +using Nostr.Client.Json; +using Nostr.Client.Messages; +using Nostr.Client.Messages.Metadata; +using Nostr.Client.Utils; +using ProtoBuf; +using StackExchange.Redis; + +namespace NostrServices.Services; + +public class RedisStore +{ + private static readonly TimeSpan DefaultExpire = TimeSpan.FromDays(30); + private readonly IDatabase _database; + + public RedisStore(IDatabase database) + { + _database = database; + } + + public async Task StoreEvent(CompactEvent ev, TimeSpan? expiry = null) + { + return await _database.SetAsync(EventKey(ev.ToIdentifier()), ev, expiry ?? DefaultExpire); + } + + public async Task GetEvent(NostrIdentifier id) + { + return await _database.GetAsync(EventKey(id)); + } + + public async Task StoreProfile(CompactProfile meta, TimeSpan? expiry = null) + { + await _database.SetAsync(ProfileKey(meta.PubKey.ToHex()), meta, expiry ?? DefaultExpire); + } + + public async Task GetProfile(string id) + { + return await _database.GetAsync(ProfileKey(id)); + } + + private string EventKey(NostrIdentifier id) => $"event:{id}"; + private string ProfileKey(string id) => $"profile:{id}"; +} + +[ProtoContract] +public class CompactEventTag +{ + [ProtoMember(1)] + public string Key { get; init; } = null!; + + [ProtoMember(2)] + public List Values { get; init; } = null!; +} + +[ProtoContract] +public class CompactEvent +{ + [ProtoMember(1)] + public byte[] Id { get; init; } = null!; + + [ProtoMember(2)] + public byte[] PubKey { get; init; } = null!; + + [ProtoMember(3)] + public long Created { get; init; } + + [ProtoMember(4)] + public string Content { get; init; } = null!; + + [ProtoMember(5)] + public List Tags { get; init; } = new(); + + [ProtoMember(6)] + public byte[] Sig { get; init; } = null!; + + [ProtoMember(7)] + public long Kind { get; init; } + + public static CompactEvent FromNostrEvent(NostrEvent ev) + { + return new CompactEvent + { + Id = Convert.FromHexString(ev.Id!), + PubKey = Convert.FromHexString(ev.Pubkey!), + Kind = (long)ev.Kind, + Created = ev.CreatedAt!.Value.ToUnixTimestamp(), + Content = ev.Content!, + Tags = ev.Tags!.Select(a => new CompactEventTag + { + Key = a.TagIdentifier!, + Values = a.AdditionalData.Cast().ToList() + }).ToList(), + Sig = Convert.FromHexString(ev.Sig!) + }; + } + + public NostrEvent ToNostrEvent() + { + return new NostrEvent() + { + Id = Id.ToHex(), + Pubkey = PubKey.ToHex(), + CreatedAt = DateTimeOffset.FromUnixTimeSeconds(Created).UtcDateTime, + Content = Content, + Tags = new(Tags.Select(a => new NostrEventTag(a.Key, a.Values.ToArray()))), + Sig = Sig.ToHex() + }; + } +} + +[ProtoContract] +public class CompactProfile +{ + [ProtoMember(1)] + public byte[] PubKey { get; init; } = null!; + + [ProtoMember(2)] + public string? Name { get; init; } + + [ProtoMember(3)] + public string? About { get; init; } + + [ProtoMember(4)] + public string? Picture { get; init; } + + [ProtoMember(5)] + public string? Nip05 { get; init; } + + [ProtoMember(6)] + public string? Lud16 { get; init; } + + [ProtoMember(7)] + public string? Banner { get; init; } + + public static CompactProfile? FromNostrEvent(NostrEvent ev) + { + var meta = NostrJson.Deserialize(ev.Content); + if (meta != default) + { + return new() + { + PubKey = Convert.FromHexString(ev.Pubkey!), + Name = meta.Name, + About = meta.About, + Picture = meta.Picture, + Nip05 = meta.Nip05, + Lud16 = meta.Lud16, + Banner = meta.Banner + }; + } + + return default; + } +} diff --git a/NostrServices/Services/RelayListener.cs b/NostrServices/Services/RelayListener.cs new file mode 100644 index 0000000..3f4a049 --- /dev/null +++ b/NostrServices/Services/RelayListener.cs @@ -0,0 +1,105 @@ +using System.Threading.Tasks.Dataflow; +using Microsoft.Extensions.Caching.Memory; +using Newtonsoft.Json; +using Nostr.Client.Json; +using Nostr.Client.Messages; +using Nostr.Client.Requests; +using NostrServices.Services.EventHandlers; + +namespace NostrServices.Services; + +public class RelayListener : IHostedService +{ + private readonly NostrListener _nostr; + private readonly IMemoryCache _cache; + private readonly CancellationTokenSource _cts = new(); + private readonly BufferBlock _queue = new(); + private readonly ILogger _logger; + private readonly IEnumerable _handlers; + + public RelayListener(NostrListener nostr, IMemoryCache cache, ILogger logger, IEnumerable handlers) + { + _nostr = nostr; + _cache = cache; + _logger = logger; + _handlers = handlers; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + const string subId = "notifications"; + var evs = _nostr.Streams.EventStream.Subscribe(a => + { + if (a.Subscription == subId) + { + _queue.Post(a.Event!); + } + }); + + _nostr.RegisterFilter(subId, new NostrFilter() + { + Kinds = new[] + { + NostrKind.Metadata, + NostrKind.Reaction, + NostrKind.Zap, + NostrKind.EncryptedDm, + NostrKind.GenericRepost, + NostrKind.LiveEvent, + NostrKind.LiveChatMessage, + NostrKind.ShortTextNote, + NostrKind.LongFormContent + }, + Limit = 0 + }); + + _cts.Token.Register(() => { evs.Dispose(); }); + _ = HandleEvents(); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + private async Task HandleEvents() + { + while (!_cts.IsCancellationRequested) + { + try + { + var msg = await _queue.ReceiveAsync(); + if (Math.Abs((DateTime.UtcNow - msg.CreatedAt!.Value).TotalMinutes) > 10) + { + // skip events older/newer than 10mins from now + continue; + } + + var seenKey = $"seen-{msg.Id}"; + var hasSeen = _cache.Get(seenKey); + if (hasSeen != default) continue; + + _logger.LogDebug(JsonConvert.SerializeObject(msg, NostrSerializer.Settings)); + _cache.Set(seenKey, true, TimeSpan.FromMinutes(10)); + + foreach (var eventHandler in _handlers) + { + try + { + await eventHandler.HandleEvent(msg); + } + catch (Exception ex) + { + _logger.LogError(ex, "Handler failed {msg}", ex.Message); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error in handle events {msg}", ex.Message); + } + } + } +} diff --git a/NostrServices/appsettings.Development.json b/NostrServices/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/NostrServices/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/NostrServices/appsettings.json b/NostrServices/appsettings.json new file mode 100644 index 0000000..288324b --- /dev/null +++ b/NostrServices/appsettings.json @@ -0,0 +1,15 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "ConnectionStrings": { + "Database": "User ID=postgres;Password=postgres;Database=nostr-services;Pooling=true;Host=127.0.0.1:25438" + }, + "Config": { + "Redis": "localhost:6377" + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..88c2061 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +services: + postgres: + image: "postgres:15" + ports: + - "25438:5432" + environment: + - "POSTGRES_DB=nostr-services" + - "POSTGRES_HOST_AUTH_METHOD=trust" + redis: + image: redis + ports: + - "6377:6379" \ No newline at end of file