diff --git a/NostrServices/Config.cs b/NostrServices/Config.cs index 133929d..be65bf8 100644 --- a/NostrServices/Config.cs +++ b/NostrServices/Config.cs @@ -3,4 +3,6 @@ namespace NostrServices; public class Config { public string Redis { get; init; } = null!; + + public string GeoIpDatabase { get; init; } = null!; } diff --git a/NostrServices/Controllers/RelaysController.cs b/NostrServices/Controllers/RelaysController.cs new file mode 100644 index 0000000..bd4fa4f --- /dev/null +++ b/NostrServices/Controllers/RelaysController.cs @@ -0,0 +1,113 @@ +using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json; +using NostrServices.Services; + +namespace NostrServices.Controllers; + +[Route("/api/v1/relays")] +public class RelaysController : Controller +{ + private readonly RedisStore _database; + + public RelaysController(RedisStore database) + { + _database = database; + } + + [HttpPost] + public async Task GetCloseRelays([FromBody] LatLonReq pos, [FromQuery] int count = 5) + { + const int distance = 5000 * 1000; // 5,000km + var relays = await _database.FindCloseRelays(pos.Lat, pos.Lon, distance, count); + + return Json(relays.Select(RelayDistance.FromDistance)); + } + + [HttpGet("top")] + public async Task GetTop([FromQuery] int count = 10) + { + var top = await _database.CountRelayUsers(); + var topSelected = top.OrderByDescending(a => a.Value).Take(count); + + var infos = await Task.WhenAll(topSelected.Select(a => _database.GetRelay(a.Key))); + return Json(infos.Where(a => a != default).Select(a => RelayDistance.FromInfo(a!))); + } +} + +public class LatLonReq +{ + [JsonProperty("lat")] + public double Lat { get; init; } + + [JsonProperty("lon")] + public double Lon { get; init; } +} + +class RelayDistance +{ + [JsonProperty("url")] + public Uri Url { get; init; } = null!; + + [JsonProperty("distance")] + public double Distance { get; init; } + + [JsonProperty("users")] + public long? Users { get; init; } + + [JsonProperty("country")] + public string? Country { get; init; } + + [JsonProperty("city")] + public string? City { get; init; } + + [JsonProperty("description")] + public string? Description { get; init; } + + [JsonProperty("is_paid")] + public bool? IsPaid { get; init; } + + [JsonProperty("is_write_restricted")] + public bool? IsWriteRestricted { get; init; } + + [JsonProperty("lat")] + public double? Latitude { get; init; } + + [JsonProperty("lon")] + public double? Longitude { get; init; } + + public static RelayDistance FromDistance(Services.RelayDistance x) + { + var rp = x.Relay.Positions.FirstOrDefault(a => a.IpAddress == x.IpAddress) ?? x.Relay.Positions.First(); + return new RelayDistance() + { + Url = x.Relay.Url, + Distance = x.Distance, + Users = x.Relay.Users, + Country = rp.Country, + City = rp.City, + Description = x.Relay.Description, + IsPaid = x.Relay.IsPaid, + IsWriteRestricted = x.Relay.IsWriteRestricted, + Latitude = rp.Lat, + Longitude = rp.Lon + }; + } + + public static RelayDistance FromInfo(RelayInfo x) + { + var rp = x.Positions.First(); + return new RelayDistance() + { + Url = x.Url, + Distance = 0, + Users = x.Users, + Country = rp.Country, + City = rp.City, + Description = x.Description, + IsPaid = x.IsPaid, + IsWriteRestricted = x.IsWriteRestricted, + Latitude = rp.Lat, + Longitude = rp.Lon + }; + } +} diff --git a/NostrServices/Database/Configuration/Relay.cs b/NostrServices/Database/Configuration/Relay.cs deleted file mode 100644 index 0ff68c8..0000000 --- a/NostrServices/Database/Configuration/Relay.cs +++ /dev/null @@ -1,23 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata.Builders; - -namespace NostrServices.Database.Configuration; - -public class RelayConfiguration : IEntityTypeConfiguration -{ - public void Configure(EntityTypeBuilder builder) - { - builder.HasKey(a => a.Id); - builder.Property(a => a.Url) - .IsRequired(); - - builder.Property(a => a.FirstSeen) - .IsRequired(); - - builder.Property(a => a.LastSeen) - .IsRequired(); - - builder.Property(a => a.IsAnyCast) - .IsRequired(); - } -} diff --git a/NostrServices/Database/NostrServicesContext.cs b/NostrServices/Database/NostrServicesContext.cs deleted file mode 100644 index f5ecafa..0000000 --- a/NostrServices/Database/NostrServicesContext.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Microsoft.EntityFrameworkCore; - -namespace NostrServices.Database; - -public class NostrServicesContext : DbContext -{ - public NostrServicesContext() - { - } - - public NostrServicesContext(DbContextOptions ctx) : base(ctx) - { - } - - public DbSet Relays => Set(); - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - modelBuilder.ApplyConfigurationsFromAssembly(typeof(NostrServicesContext).Assembly); - } -} diff --git a/NostrServices/Database/Relay.cs b/NostrServices/Database/Relay.cs deleted file mode 100644 index 4ac853f..0000000 --- a/NostrServices/Database/Relay.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace NostrServices.Database; - -public class Relay -{ - public Guid Id { get; init; } = Guid.NewGuid(); - - public Uri Url { get; init; } = null!; - - public DateTime FirstSeen { get; init; } = DateTime.UtcNow; - - public DateTime LastSeen { get; init; } - - /// - /// If this relay uses any cast IPs which will naturally obscure its location - /// - public bool IsAnyCast { get; init; } -} diff --git a/NostrServices/Migrations/20240110094022_Relay.Designer.cs b/NostrServices/Migrations/20240110094022_Relay.Designer.cs deleted file mode 100644 index 979c4ec..0000000 --- a/NostrServices/Migrations/20240110094022_Relay.Designer.cs +++ /dev/null @@ -1,54 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using NostrServices.Database; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace NostrServices.Migrations -{ - [DbContext(typeof(NostrServicesContext))] - [Migration("20240110094022_Relay")] - partial class Relay - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "8.0.1") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("NostrServices.Database.Relay", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("FirstSeen") - .HasColumnType("timestamp with time zone"); - - b.Property("IsAnyCast") - .HasColumnType("boolean"); - - b.Property("LastSeen") - .HasColumnType("timestamp with time zone"); - - b.Property("Url") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("Relays"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/NostrServices/Migrations/20240110094022_Relay.cs b/NostrServices/Migrations/20240110094022_Relay.cs deleted file mode 100644 index f86fd59..0000000 --- a/NostrServices/Migrations/20240110094022_Relay.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace NostrServices.Migrations -{ - /// - public partial class Relay : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "Relays", - columns: table => new - { - Id = table.Column(type: "uuid", nullable: false), - Url = table.Column(type: "text", nullable: false), - FirstSeen = table.Column(type: "timestamp with time zone", nullable: false), - LastSeen = table.Column(type: "timestamp with time zone", nullable: false), - IsAnyCast = table.Column(type: "boolean", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_Relays", x => x.Id); - }); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "Relays"); - } - } -} diff --git a/NostrServices/Migrations/NostrServicesContextModelSnapshot.cs b/NostrServices/Migrations/NostrServicesContextModelSnapshot.cs deleted file mode 100644 index 582f0ed..0000000 --- a/NostrServices/Migrations/NostrServicesContextModelSnapshot.cs +++ /dev/null @@ -1,51 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using NostrServices.Database; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace NostrServices.Migrations -{ - [DbContext(typeof(NostrServicesContext))] - partial class NostrServicesContextModelSnapshot : ModelSnapshot - { - protected override void BuildModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "8.0.1") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("NostrServices.Database.Relay", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("FirstSeen") - .HasColumnType("timestamp with time zone"); - - b.Property("IsAnyCast") - .HasColumnType("boolean"); - - b.Property("LastSeen") - .HasColumnType("timestamp with time zone"); - - b.Property("Url") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("Relays"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/NostrServices/NostrServices.csproj b/NostrServices/NostrServices.csproj index cb5f26c..fa7c680 100644 --- a/NostrServices/NostrServices.csproj +++ b/NostrServices/NostrServices.csproj @@ -7,6 +7,7 @@ Linux true $(NoWarn);1591 + true @@ -17,16 +18,11 @@ + - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - diff --git a/NostrServices/Program.cs b/NostrServices/Program.cs index 65218db..b237f95 100644 --- a/NostrServices/Program.cs +++ b/NostrServices/Program.cs @@ -1,12 +1,11 @@ using System.Reflection; -using Microsoft.EntityFrameworkCore; +using MaxMind.GeoIP2; using Microsoft.OpenApi.Models; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; using Nostr.Client.Client; using NostrRelay; -using NostrServices.Database; using NostrServices.Services; using NostrServices.Services.EventHandlers; using Prometheus; @@ -24,7 +23,7 @@ public static class Program var cx = await ConnectionMultiplexer.ConnectAsync(config.Redis); builder.Services.AddSingleton(cx); - //builder.Services.AddSingleton(_ => new DatabaseReader(cfg.GeoIpDatabase)); + builder.Services.AddSingleton(_ => new DatabaseReader(config.GeoIpDatabase)); builder.Services.AddSingleton(); builder.Services.AddSingleton(s => s.GetRequiredService()); builder.Services.AddSingleton(); @@ -81,17 +80,10 @@ public static class Program builder.Services.AddCors(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddTransient(); - ConfigureDb(builder.Services, builder.Configuration); - var app = builder.Build(); - using (var scope = app.Services.CreateScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - await db.Database.MigrateAsync(); - } - app.UseResponseCaching(); app.UseSwagger(); app.UseSwaggerUI(); @@ -112,24 +104,4 @@ public static class Program await app.RunAsync(); } - - private static void ConfigureDb(IServiceCollection services, IConfiguration configuration) - { - services.AddDbContext(o => - o.UseNpgsql(configuration.GetConnectionString("Database"))); - } - - /// - /// Dummy method for EF core migrations - /// - /// - /// - // ReSharper disable once UnusedMember.Global - public static IHostBuilder CreateHostBuilder(string[] args) - { - var dummyHost = Host.CreateDefaultBuilder(args); - dummyHost.ConfigureServices((ctx, svc) => { ConfigureDb(svc, ctx.Configuration); }); - - return dummyHost; - } } diff --git a/NostrServices/Services/CacheRelay.cs b/NostrServices/Services/CacheRelay.cs index 74a2b07..f540d13 100644 --- a/NostrServices/Services/CacheRelay.cs +++ b/NostrServices/Services/CacheRelay.cs @@ -1,18 +1,19 @@ using Nostr.Client.Messages; using Nostr.Client.Requests; using NostrRelay; +using NostrServices.Services.EventHandlers; namespace NostrServices.Services; public class CacheRelay : INostrRelay, IDisposable { private readonly ILogger _logger; - private readonly RedisStore _redisStore; + private readonly IEnumerable _handlers; - public CacheRelay(RedisStore redisStore, ILogger logger) + public CacheRelay(ILogger logger, IEnumerable handlers) { - _redisStore = redisStore; _logger = logger; + _handlers = handlers; } public ValueTask AcceptConnection(NostrClientContext context) @@ -31,13 +32,16 @@ public class CacheRelay : INostrRelay, IDisposable { if (RelayListener.AcceptedKinds.Contains(ev.Kind)) { - if (ev.Kind is NostrKind.Metadata) + foreach (var eventHandler in _handlers) { - await _redisStore.StoreProfile(CompactProfile.FromNostrEvent(ev)!); - } - else - { - await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev)); + try + { + await eventHandler.HandleEvent(ev); + } + catch (Exception ex) + { + _logger.LogError(ex, "Handler failed {msg}", ex.Message); + } } return new(true, null); diff --git a/NostrServices/Services/EventHandlers/RedisEventCache.cs b/NostrServices/Services/EventHandlers/RedisEventCache.cs index a3093c8..d7df081 100644 --- a/NostrServices/Services/EventHandlers/RedisEventCache.cs +++ b/NostrServices/Services/EventHandlers/RedisEventCache.cs @@ -1,3 +1,5 @@ +using NBitcoin; +using Newtonsoft.Json; using Nostr.Client.Messages; namespace NostrServices.Services.EventHandlers; @@ -21,9 +23,54 @@ public class RedisEventCache : IEventHandler await _redisStore.StoreProfile(p); } } + else if (ev.Kind is NostrKind.Contacts) + { + var relays = JsonConvert.DeserializeObject>(ev.Content!); + if (relays == default) return; + + var compact = relays.Select(a => new RelaySetting + { + Url = new Uri(a.Key.ToLower()), + Read = a.Value.Read, + Write = a.Value.Write + }); + + await _redisStore.StoreUserRelays(ev.Pubkey!, new() + { + Created = ev.CreatedAt!.Value.ToUnixTimestamp(), + Relays = compact.ToList() + }); + } + else if (ev.Kind is NostrKind.RelayListMetadata) + { + var rTags = ev.Tags!.Where(a => a.TagIdentifier == "r").ToList(); + if (rTags.Count > 50) return; + + var compact = rTags.Select(a => new RelaySetting + { + Url = new Uri(a.AdditionalData[0].ToLower()), + Read = a.AdditionalData.Length == 1 || a.AdditionalData[1] == "read", + Write = a.AdditionalData.Length == 1 || a.AdditionalData[1] == "write" + }); + + await _redisStore.StoreUserRelays(ev.Pubkey!, new() + { + Created = ev.CreatedAt!.Value.ToUnixTimestamp(), + Relays = compact.ToList() + }); + } else { await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev)); } } + + class ContactsRelays + { + [JsonProperty("read")] + public bool Read { get; init; } + + [JsonProperty("write")] + public bool Write { get; init; } + } } diff --git a/NostrServices/Services/NostrListener.cs b/NostrServices/Services/NostrListener.cs index 7323ca0..2cefc1a 100644 --- a/NostrServices/Services/NostrListener.cs +++ b/NostrServices/Services/NostrListener.cs @@ -1,10 +1,8 @@ using System.Net.WebSockets; using System.Reflection; -using Microsoft.EntityFrameworkCore; using Nostr.Client.Client; using Nostr.Client.Communicator; using Nostr.Client.Requests; -using NostrServices.Database; using Websocket.Client.Models; namespace NostrServices.Services; @@ -126,10 +124,11 @@ public class NostrListener : IDisposable while (!stoppingToken.IsCancellationRequested) { using var scope = _scopeFactory.CreateScope(); - await using var db = scope.ServiceProvider.GetRequiredService(); + var db = scope.ServiceProvider.GetRequiredService(); - var relays = await db.Relays.Select(a => a.Url).ToListAsync(cancellationToken: stoppingToken); - _nostrListener.AddCommunicators(relays); + var relays = await db.CountRelayUsers(); + var topRelays = relays.OrderByDescending(a => a.Value).Take(20).Select(a => a.Key).ToList(); + _nostrListener.AddCommunicators(topRelays); await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken); } diff --git a/NostrServices/Services/RedisStore.cs b/NostrServices/Services/RedisStore.cs index 37e668e..bda9da4 100644 --- a/NostrServices/Services/RedisStore.cs +++ b/NostrServices/Services/RedisStore.cs @@ -38,13 +38,15 @@ public class RedisStore return ev; } - public async Task StoreProfile(CompactProfile meta, TimeSpan? expiry = null) + public async Task StoreProfile(CompactProfile meta, TimeSpan? expiry = null) { var oldProfile = await GetProfile(meta.PubKey.ToHex()); if ((oldProfile?.Created ?? new DateTime()) < meta.Created) { - await _database.SetAsync(ProfileKey(meta.PubKey.ToHex()), meta, expiry ?? DefaultExpire); + return await _database.SetAsync(ProfileKey(meta.PubKey.ToHex()), meta, expiry ?? DefaultExpire); } + + return false; } public async Task GetProfile(string id) @@ -59,6 +61,93 @@ public class RedisStore return profile; } + public async Task> GetKnownRelays() + { + var ret = new HashSet(); + await foreach (var r in _database.SetScanAsync("relays")) + { + if (r.HasValue) + { + ret.Add(new Uri(r!)); + } + } + + return ret; + } + + public async Task StoreRelay(RelayInfo cri) + { + return await _database.SetAsync(RelayKey(cri.Url), cri, DefaultExpire); + } + + public async Task GetRelay(Uri u) + { + return await _database.GetAsync(RelayKey(u)); + } + + public async Task StoreUserRelays(string pubkey, UserRelaySettings relays) + { + var old = await GetUserRelays(pubkey); + if ((old?.Created ?? 0) < relays.Created) + { + var removed = old?.Relays.Where(a => relays.Relays.All(b => b.Url != a.Url)); + var added = old == default ? relays.Relays : relays.Relays.Where(a => old.Relays.All(b => b.Url != a.Url)).ToList(); + if (removed != default) + { + await Task.WhenAll(removed.Select(a => _database.SetRemoveAsync(RelayUsersKey(a.Url), Convert.FromHexString(pubkey)))); + } + + await Task.WhenAll(added.Select(a => _database.SetAddAsync(RelayUsersKey(a.Url), Convert.FromHexString(pubkey)))); + await _database.SetAddAsync("relays", added.Select(a => (RedisValue)a.Url.ToString()).ToArray()); + + return await _database.SetAsync(UserRelays(pubkey), relays, DefaultExpire); + } + + return false; + } + + public async Task GetUserRelays(string pubkey) + { + return await _database.GetAsync(UserRelays(pubkey)); + } + + public async Task> CountRelayUsers() + { + var allRelays = await GetKnownRelays(); + var tasks = allRelays.Select(a => (Url: a, Task: _database.SetLengthAsync(RelayUsersKey(a)))).ToList(); + await Task.WhenAll(tasks.Select(a => a.Task)); + + return tasks.ToDictionary(a => a.Url, b => b.Task.Result); + } + + public async Task StoreRelayPosition(Uri u, string ipAddress, double lat, double lon) + { + await _database.GeoAddAsync(RelayPositionKey(), lon, lat, $"{u}\x1{ipAddress}"); + } + + public async Task> FindCloseRelays(double lat, double lon, int radius = 50_000, int count = 10) + { + var ret = new List(); + var geoRelays = await _database.GeoSearchAsync(RelayPositionKey(), lon, lat, new GeoSearchCircle(radius), count); + foreach (var gr in geoRelays) + { + var id = ((string)gr.Member!).Split('\x1'); + var u = new Uri(id[0]); + var info = await GetRelay(u); + if (info != default) + { + ret.Add(new() + { + Distance = gr.Distance.HasValue ? (long)gr.Distance.Value : 0, + Relay = info, + IpAddress = id[1] + }); + } + } + + return ret; + } + private string EventKey(NostrIdentifier id) { if (id is NostrAddressIdentifier naddr) @@ -69,7 +158,11 @@ public class RedisStore return $"event:{id.Special}"; } - private string ProfileKey(string id) => $"profile:{id}"; + private static string ProfileKey(string id) => $"profile:{id}"; + private static string RelayKey(Uri relay) => $"relay:${relay}"; + private static string RelayPositionKey() => $"relays:geo"; + private static string RelayUsersKey(Uri relay) => $"relay:{relay}:users"; + private static string UserRelays(string pubkey) => $"profile:{pubkey}:relays"; } [ProtoContract] @@ -117,8 +210,8 @@ public class CompactEvent Content = ev.Content!, Tags = ev.Tags!.Select(a => new CompactEventTag { - Key = a.TagIdentifier!, - Values = a.AdditionalData.Cast().ToList() + Key = a.TagIdentifier, + Values = a.AdditionalData.ToList() }).ToList(), Sig = Convert.FromHexString(ev.Sig!) }; @@ -195,3 +288,96 @@ public class CompactProfile return default; } } + +[ProtoContract] +public class RelayDistance +{ + [ProtoMember(1)] + public long Distance { get; init; } + + [ProtoMember(2)] + public string IpAddress { get; init; } = null!; + + [ProtoMember(3)] + public RelayInfo Relay { get; init; } = null!; +} + +[ProtoContract] +public class RelayInfo +{ + [ProtoMember(1)] + public Uri Url { get; init; } = null!; + + [ProtoMember(2)] + public long FirstSeen { get; init; } + + [ProtoMember(3)] + public long LastSeen { get; init; } + + [ProtoMember(4)] + public long Users { get; init; } + + [ProtoMember(5)] + public List Positions { get; init; } = new(); + + [ProtoMember(6)] + public bool? IsPaid { get; init; } + + [ProtoMember(7)] + public bool? IsWriteRestricted { get; init; } + + [ProtoMember(8)] + public string? Description { get; init; } +} + +[ProtoContract] +public class Position +{ + [ProtoMember(1)] + public double Lat { get; init; } + + [ProtoMember(2)] + public double Lon { get; init; } + + [ProtoMember(3)] + public string? Country { get; init; } + + [ProtoMember(4)] + public string? City { get; init; } + + [ProtoMember(6)] + public string IpAddress { get; init; } = null!; +} + +[ProtoContract] +public class RelayEndpoint +{ + [ProtoMember(1)] + public Uri Relay { get; init; } = null!; + + [ProtoMember(2)] + public string IpAddress { get; init; } = null!; +} + +[ProtoContract] +public class UserRelaySettings +{ + [ProtoMember(1)] + public long Created { get; init; } + + [ProtoMember(2)] + public List Relays { get; init; } = new(); +} + +[ProtoContract] +public class RelaySetting +{ + [ProtoMember(1)] + public Uri Url { get; init; } = null!; + + [ProtoMember(2)] + public bool Read { get; init; } + + [ProtoMember(3)] + public bool Write { get; init; } +} diff --git a/NostrServices/Services/RelayListener.cs b/NostrServices/Services/RelayListener.cs index cfe3306..b70b5b1 100644 --- a/NostrServices/Services/RelayListener.cs +++ b/NostrServices/Services/RelayListener.cs @@ -20,7 +20,9 @@ public class RelayListener : IHostedService NostrKind.LiveChatMessage, (NostrKind)1_313, // stream clip NostrKind.LongFormContent, - NostrKind.ClassifiedListing + NostrKind.ClassifiedListing, + NostrKind.RelayListMetadata, + NostrKind.Contacts ]; private readonly NostrListener _nostr; diff --git a/NostrServices/Services/RelayScraperService.cs b/NostrServices/Services/RelayScraperService.cs new file mode 100644 index 0000000..493c6db --- /dev/null +++ b/NostrServices/Services/RelayScraperService.cs @@ -0,0 +1,167 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using MaxMind.GeoIP2; +using NBitcoin; +using Newtonsoft.Json; + +namespace NostrServices.Services; + +public class RelayScraperService : BackgroundService +{ + private readonly RedisStore _redisStore; + private readonly IGeoIP2DatabaseReader _geoIp2Database; + private readonly ILogger _logger; + private readonly HttpClient _client; + + public RelayScraperService(IGeoIP2DatabaseReader geoIp2Database, ILogger logger, HttpClient client, + RedisStore redisStore) + { + _geoIp2Database = geoIp2Database; + _logger = logger; + _client = client; + _redisStore = redisStore; + _client.Timeout = TimeSpan.FromSeconds(10); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + var sw = Stopwatch.StartNew(); + await UpdateRelays(stoppingToken); + sw.Stop(); + + _logger.LogInformation("Caching relay locations took {n:#,##0.00} seconds", sw.Elapsed.TotalSeconds); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to cache relay-geo info {msg}", ex.Message); + } + + await Task.Delay(TimeSpan.FromMinutes(60), stoppingToken); + } + } + + private async Task UpdateRelays(CancellationToken cts) + { + const int batchSize = 20; + + var relays = await _redisStore.CountRelayUsers(); + foreach (var batch in relays.Select((v, i) => (Batch: i / batchSize, Value: v)).GroupBy(a => a.Batch)) + { + await Task.WhenAll(batch.Select(a => ScrapeRelay(a.Value, cts))); + } + } + + private async Task ScrapeRelay(KeyValuePair relay, CancellationToken cts) + { + try + { + _logger.LogInformation("Scraping relay: {url}", relay); + if (relay.Key.AbsolutePath.Contains("%20wss") || relay.Key.AbsolutePath.StartsWith("/npub1")) + { + return; + } + + // only store relays with info doc (lazy up status) + var info = await GetRelayInfo(relay.Key); + if (info == default) return; + + var ret = new RelayInfo() + { + Url = relay.Key, + Users = relay.Value, + IsPaid = info?.Limitations?.PaymentRequired, + IsWriteRestricted = info?.Limitations?.RestrictedWrites, + Description = info?.Description, + LastSeen = DateTime.UtcNow.ToUnixTimestamp(), + FirstSeen = DateTime.UtcNow.ToUnixTimestamp() + }; + + var ips = await Dns.GetHostAddressesAsync(relay.Key.DnsSafeHost, cts); + foreach (var ip in ips) + { + if (!ip.IsRoutable(false)) continue; + + if (_geoIp2Database.TryCity(ip, out var city) && city!.Location.Longitude.HasValue && city.Location.Latitude.HasValue) + { + ret.Positions.Add(new() + { + Lat = city.Location.Latitude.Value, + Lon = city.Location.Longitude.Value, + Country = city.Country.Name, + City = city.City.Name, + IpAddress = ip.ToString() + }); + } + } + + await _redisStore.StoreRelay(ret); + foreach (var pos in ret.Positions) + { + await _redisStore.StoreRelayPosition(ret.Url, pos.IpAddress, pos.Lat, pos.Lon); + } + } + catch (Exception ex) + { + // ignored + _logger.LogWarning(ex, "Failed to load relay geo info {relay} {message}", relay, ex.Message); + } + } + + private async Task GetRelayInfo(Uri r) + { + try + { + var ub = new UriBuilder() + { + Scheme = r.Scheme == "wss" ? "https" : "http", + Host = r.Host, + Path = r.AbsolutePath, + Port = r.Port, + }; + + var rsp = await _client.SendAsync(new(HttpMethod.Get, ub.Uri) + { + Headers = + { + Accept = {new MediaTypeWithQualityHeaderValue("application/nostr+json")} + } + }); + + if (rsp.IsSuccessStatusCode) + { + var json = await rsp.Content.ReadAsStringAsync(); + var doc = JsonConvert.DeserializeObject(json); + return doc; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load relay doc for {url} {message}", r, ex.Message); + } + + return default; + } +} + +public class RelayDoc +{ + [JsonProperty("description")] + public string? Description { get; init; } + + [JsonProperty("limitation")] + public RelayDocLimitation? Limitations { get; init; } +} + +public class RelayDocLimitation +{ + [JsonProperty("payment_required")] + public bool? PaymentRequired { get; init; } + + [JsonProperty("restricted_writes")] + public bool? RestrictedWrites { get; init; } +} diff --git a/NostrServices/appsettings.json b/NostrServices/appsettings.json index 288324b..232500e 100644 --- a/NostrServices/appsettings.json +++ b/NostrServices/appsettings.json @@ -2,7 +2,8 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "System.Net.Http": "Warning" } }, "AllowedHosts": "*", @@ -10,6 +11,7 @@ "Database": "User ID=postgres;Password=postgres;Database=nostr-services;Pooling=true;Host=127.0.0.1:25438" }, "Config": { - "Redis": "localhost:6377" + "Redis": "localhost:6377", + "GeoIpDatabase": "/Users/kieran/Downloads/GeoLite2-City_20230801/GeoLite2-City.mmdb" } } diff --git a/docker-compose.yml b/docker-compose.yml index 88c2061..d4d6d9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,4 @@ services: - postgres: - image: "postgres:15" - ports: - - "25438:5432" - environment: - - "POSTGRES_DB=nostr-services" - - "POSTGRES_HOST_AUTH_METHOD=trust" redis: image: redis ports: