Relay Scraper services

NostrServices/Config.cs

@@ -3,4 +3,6 @@ namespace NostrServices;
 public class Config
 {
     public string Redis { get; init; } = null!;
+
+    public string GeoIpDatabase { get; init; } = null!;
 }

NostrServices/Controllers/RelaysController.cs

@@ -0,0 +1,113 @@
+using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json;
+using NostrServices.Services;
+
+namespace NostrServices.Controllers;
+
+[Route("/api/v1/relays")]
+public class RelaysController : Controller
+{
+    private readonly RedisStore _database;
+
+    public RelaysController(RedisStore database)
+    {
+        _database = database;
+    }
+
+    [HttpPost]
+    public async Task<IActionResult> GetCloseRelays([FromBody] LatLonReq pos, [FromQuery] int count = 5)
+    {
+        const int distance = 5000 * 1000; // 5,000km
+        var relays = await _database.FindCloseRelays(pos.Lat, pos.Lon, distance, count);
+
+        return Json(relays.Select(RelayDistance.FromDistance));
+    }
+
+    [HttpGet("top")]
+    public async Task<IActionResult> GetTop([FromQuery] int count = 10)
+    {
+        var top = await _database.CountRelayUsers();
+        var topSelected = top.OrderByDescending(a => a.Value).Take(count);
+
+        var infos = await Task.WhenAll(topSelected.Select(a => _database.GetRelay(a.Key)));
+        return Json(infos.Where(a => a != default).Select(a => RelayDistance.FromInfo(a!)));
+    }
+}
+
+public class LatLonReq
+{
+    [JsonProperty("lat")]
+    public double Lat { get; init; }
+
+    [JsonProperty("lon")]
+    public double Lon { get; init; }
+}
+
+class RelayDistance
+{
+    [JsonProperty("url")]
+    public Uri Url { get; init; } = null!;
+
+    [JsonProperty("distance")]
+    public double Distance { get; init; }
+
+    [JsonProperty("users")]
+    public long? Users { get; init; }
+
+    [JsonProperty("country")]
+    public string? Country { get; init; }
+
+    [JsonProperty("city")]
+    public string? City { get; init; }
+
+    [JsonProperty("description")]
+    public string? Description { get; init; }
+
+    [JsonProperty("is_paid")]
+    public bool? IsPaid { get; init; }
+
+    [JsonProperty("is_write_restricted")]
+    public bool? IsWriteRestricted { get; init; }
+
+    [JsonProperty("lat")]
+    public double? Latitude { get; init; }
+
+    [JsonProperty("lon")]
+    public double? Longitude { get; init; }
+
+    public static RelayDistance FromDistance(Services.RelayDistance x)
+    {
+        var rp = x.Relay.Positions.FirstOrDefault(a => a.IpAddress == x.IpAddress) ?? x.Relay.Positions.First();
+        return new RelayDistance()
+        {
+            Url = x.Relay.Url,
+            Distance = x.Distance,
+            Users = x.Relay.Users,
+            Country = rp.Country,
+            City = rp.City,
+            Description = x.Relay.Description,
+            IsPaid = x.Relay.IsPaid,
+            IsWriteRestricted = x.Relay.IsWriteRestricted,
+            Latitude = rp.Lat,
+            Longitude = rp.Lon
+        };
+    }
+
+    public static RelayDistance FromInfo(RelayInfo x)
+    {
+        var rp = x.Positions.First();
+        return new RelayDistance()
+        {
+            Url = x.Url,
+            Distance = 0,
+            Users = x.Users,
+            Country = rp.Country,
+            City = rp.City,
+            Description = x.Description,
+            IsPaid = x.IsPaid,
+            IsWriteRestricted = x.IsWriteRestricted,
+            Latitude = rp.Lat,
+            Longitude = rp.Lon
+        };
+    }
+}

NostrServices/Database/Configuration/Relay.cs

@@ -1,23 +0,0 @@
-using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Metadata.Builders;
-
-namespace NostrServices.Database.Configuration;
-
-public class RelayConfiguration : IEntityTypeConfiguration<Relay>
-{
-    public void Configure(EntityTypeBuilder<Relay> builder)
-    {
-        builder.HasKey(a => a.Id);
-        builder.Property(a => a.Url)
-            .IsRequired();
-
-        builder.Property(a => a.FirstSeen)
-            .IsRequired();
-
-        builder.Property(a => a.LastSeen)
-            .IsRequired();
-
-        builder.Property(a => a.IsAnyCast)
-            .IsRequired();
-    }
-}

NostrServices/Database/NostrServicesContext.cs

@@ -1,21 +0,0 @@
-using Microsoft.EntityFrameworkCore;
-
-namespace NostrServices.Database;
-
-public class NostrServicesContext : DbContext
-{
-    public NostrServicesContext()
-    {
-    }
-
-    public NostrServicesContext(DbContextOptions<NostrServicesContext> ctx) : base(ctx)
-    {
-    }
-
-    public DbSet<Relay> Relays => Set<Relay>();
-    
-    protected override void OnModelCreating(ModelBuilder modelBuilder)
-    {
-        modelBuilder.ApplyConfigurationsFromAssembly(typeof(NostrServicesContext).Assembly);
-    }
-}

NostrServices/Database/Relay.cs

@@ -1,17 +0,0 @@
-namespace NostrServices.Database;
-
-public class Relay
-{
-    public Guid Id { get; init; } = Guid.NewGuid();
-
-    public Uri Url { get; init; } = null!;
-
-    public DateTime FirstSeen { get; init; } = DateTime.UtcNow;
-    
-    public DateTime LastSeen { get; init; }
-    
-    /// <summary>
-    /// If this relay uses any cast IPs which will naturally obscure its location
-    /// </summary>
-    public bool IsAnyCast { get; init; }
-}

NostrServices/Migrations/20240110094022_Relay.Designer.cs

@@ -1,54 +0,0 @@
-// <auto-generated />
-using System;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Infrastructure;
-using Microsoft.EntityFrameworkCore.Migrations;
-using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
-using NostrServices.Database;
-using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
-
-#nullable disable
-
-namespace NostrServices.Migrations
-{
-    [DbContext(typeof(NostrServicesContext))]
-    [Migration("20240110094022_Relay")]
-    partial class Relay
-    {
-        /// <inheritdoc />
-        protected override void BuildTargetModel(ModelBuilder modelBuilder)
-        {
-#pragma warning disable 612, 618
-            modelBuilder
-                .HasAnnotation("ProductVersion", "8.0.1")
-                .HasAnnotation("Relational:MaxIdentifierLength", 63);
-
-            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
-
-            modelBuilder.Entity("NostrServices.Database.Relay", b =>
-                {
-                    b.Property<Guid>("Id")
-                        .ValueGeneratedOnAdd()
-                        .HasColumnType("uuid");
-
-                    b.Property<DateTime>("FirstSeen")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.Property<bool>("IsAnyCast")
-                        .HasColumnType("boolean");
-
-                    b.Property<DateTime>("LastSeen")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.Property<string>("Url")
-                        .IsRequired()
-                        .HasColumnType("text");
-
-                    b.HasKey("Id");
-
-                    b.ToTable("Relays");
-                });
-#pragma warning restore 612, 618
-        }
-    }
-}

NostrServices/Migrations/20240110094022_Relay.cs

@@ -1,37 +0,0 @@
-using System;
-using Microsoft.EntityFrameworkCore.Migrations;
-
-#nullable disable
-
-namespace NostrServices.Migrations
-{
-    /// <inheritdoc />
-    public partial class Relay : Migration
-    {
-        /// <inheritdoc />
-        protected override void Up(MigrationBuilder migrationBuilder)
-        {
-            migrationBuilder.CreateTable(
-                name: "Relays",
-                columns: table => new
-                {
-                    Id = table.Column<Guid>(type: "uuid", nullable: false),
-                    Url = table.Column<string>(type: "text", nullable: false),
-                    FirstSeen = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
-                    LastSeen = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
-                    IsAnyCast = table.Column<bool>(type: "boolean", nullable: false)
-                },
-                constraints: table =>
-                {
-                    table.PrimaryKey("PK_Relays", x => x.Id);
-                });
-        }
-
-        /// <inheritdoc />
-        protected override void Down(MigrationBuilder migrationBuilder)
-        {
-            migrationBuilder.DropTable(
-                name: "Relays");
-        }
-    }
-}

NostrServices/Migrations/NostrServicesContextModelSnapshot.cs

@@ -1,51 +0,0 @@
-// <auto-generated />
-using System;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore.Infrastructure;
-using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
-using NostrServices.Database;
-using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
-
-#nullable disable
-
-namespace NostrServices.Migrations
-{
-    [DbContext(typeof(NostrServicesContext))]
-    partial class NostrServicesContextModelSnapshot : ModelSnapshot
-    {
-        protected override void BuildModel(ModelBuilder modelBuilder)
-        {
-#pragma warning disable 612, 618
-            modelBuilder
-                .HasAnnotation("ProductVersion", "8.0.1")
-                .HasAnnotation("Relational:MaxIdentifierLength", 63);
-
-            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
-
-            modelBuilder.Entity("NostrServices.Database.Relay", b =>
-                {
-                    b.Property<Guid>("Id")
-                        .ValueGeneratedOnAdd()
-                        .HasColumnType("uuid");
-
-                    b.Property<DateTime>("FirstSeen")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.Property<bool>("IsAnyCast")
-                        .HasColumnType("boolean");
-
-                    b.Property<DateTime>("LastSeen")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.Property<string>("Url")
-                        .IsRequired()
-                        .HasColumnType("text");
-
-                    b.HasKey("Id");
-
-                    b.ToTable("Relays");
-                });
-#pragma warning restore 612, 618
-        }
-    }
-}

NostrServices/NostrServices.csproj

@@ -7,6 +7,7 @@
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <NoWarn>$(NoWarn);1591</NoWarn>
+        <ServerGarbageCollection>true</ServerGarbageCollection>
     </PropertyGroup>
 
     <ItemGroup>
@@ -17,16 +18,11 @@
 
     <ItemGroup>
       <PackageReference Include="AngleSharp" Version="1.0.7" />
+      <PackageReference Include="MaxMind.GeoIP2" Version="5.2.0" />
       <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.1" />
-      <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.1" />
-      <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.1">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-      </PackageReference>
       <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.1" />
       <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
       <PackageReference Include="Nostr.Client" Version="2.0.0" />
-      <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.0" />
       <PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
       <PackageReference Include="protobuf-net" Version="3.2.30" />
       <PackageReference Include="StackExchange.Redis" Version="2.7.10" />

NostrServices/Program.cs

@@ -1,12 +1,11 @@
 using System.Reflection;
-using Microsoft.EntityFrameworkCore;
+using MaxMind.GeoIP2;
 using Microsoft.OpenApi.Models;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Serialization;
 using Nostr.Client.Client;
 using NostrRelay;
-using NostrServices.Database;
 using NostrServices.Services;
 using NostrServices.Services.EventHandlers;
 using Prometheus;
@@ -24,7 +23,7 @@ public static class Program
         var cx = await ConnectionMultiplexer.ConnectAsync(config.Redis);
         builder.Services.AddSingleton(cx);
 
-        //builder.Services.AddSingleton<IGeoIP2DatabaseReader>(_ => new DatabaseReader(cfg.GeoIpDatabase));
+        builder.Services.AddSingleton<IGeoIP2DatabaseReader>(_ => new DatabaseReader(config.GeoIpDatabase));
         builder.Services.AddSingleton<NostrMultiWebsocketClient>();
         builder.Services.AddSingleton<INostrClient>(s => s.GetRequiredService<NostrMultiWebsocketClient>());
         builder.Services.AddSingleton<NostrListener>();
@@ -81,17 +80,10 @@ public static class Program
         builder.Services.AddCors();
 
         builder.Services.AddHostedService<NostrListener.NostrListenerLifetime>();
+        builder.Services.AddHostedService<RelayScraperService>();
         builder.Services.AddTransient<CacheRelay>();
 
-        ConfigureDb(builder.Services, builder.Configuration);
-
         var app = builder.Build();
-        using (var scope = app.Services.CreateScope())
-        {
-            var db = scope.ServiceProvider.GetRequiredService<NostrServicesContext>();
-            await db.Database.MigrateAsync();
-        }
-
         app.UseResponseCaching();
         app.UseSwagger();
         app.UseSwaggerUI();
@@ -112,24 +104,4 @@ public static class Program
 
         await app.RunAsync();
     }
-
-    private static void ConfigureDb(IServiceCollection services, IConfiguration configuration)
-    {
-        services.AddDbContext<NostrServicesContext>(o =>
-            o.UseNpgsql(configuration.GetConnectionString("Database")));
-    }
-
-    /// <summary>
-    /// Dummy method for EF core migrations
-    /// </summary>
-    /// <param name="args"></param>
-    /// <returns></returns>
-    // ReSharper disable once UnusedMember.Global
-    public static IHostBuilder CreateHostBuilder(string[] args)
-    {
-        var dummyHost = Host.CreateDefaultBuilder(args);
-        dummyHost.ConfigureServices((ctx, svc) => { ConfigureDb(svc, ctx.Configuration); });
-
-        return dummyHost;
-    }
 }

NostrServices/Services/CacheRelay.cs

@@ -1,18 +1,19 @@
 using Nostr.Client.Messages;
 using Nostr.Client.Requests;
 using NostrRelay;
+using NostrServices.Services.EventHandlers;
 
 namespace NostrServices.Services;
 
 public class CacheRelay : INostrRelay, IDisposable
 {
     private readonly ILogger<CacheRelay> _logger;
-    private readonly RedisStore _redisStore;
+    private readonly IEnumerable<IEventHandler> _handlers;
 
-    public CacheRelay(RedisStore redisStore, ILogger<CacheRelay> logger)
+    public CacheRelay(ILogger<CacheRelay> logger, IEnumerable<IEventHandler> handlers)
     {
-        _redisStore = redisStore;
         _logger = logger;
+        _handlers = handlers;
     }
 
     public ValueTask<bool> AcceptConnection(NostrClientContext context)
@@ -31,13 +32,16 @@ public class CacheRelay : INostrRelay, IDisposable
     {
         if (RelayListener.AcceptedKinds.Contains(ev.Kind))
         {
-            if (ev.Kind is NostrKind.Metadata)
+            foreach (var eventHandler in _handlers)
             {
-                await _redisStore.StoreProfile(CompactProfile.FromNostrEvent(ev)!);
-            }
-            else
-            {
-                await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev));
+                try
+                {
+                    await eventHandler.HandleEvent(ev);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Handler failed {msg}", ex.Message);
+                }
             }
 
             return new(true, null);

NostrServices/Services/EventHandlers/RedisEventCache.cs

@@ -1,3 +1,5 @@
+using NBitcoin;
+using Newtonsoft.Json;
 using Nostr.Client.Messages;
 
 namespace NostrServices.Services.EventHandlers;
@@ -21,9 +23,54 @@ public class RedisEventCache : IEventHandler
                 await _redisStore.StoreProfile(p);
             }
         }
+        else if (ev.Kind is NostrKind.Contacts)
+        {
+            var relays = JsonConvert.DeserializeObject<Dictionary<string, ContactsRelays>>(ev.Content!);
+            if (relays == default) return;
+
+            var compact = relays.Select(a => new RelaySetting
+            {
+                Url = new Uri(a.Key.ToLower()),
+                Read = a.Value.Read,
+                Write = a.Value.Write
+            });
+
+            await _redisStore.StoreUserRelays(ev.Pubkey!, new()
+            {
+                Created = ev.CreatedAt!.Value.ToUnixTimestamp(),
+                Relays = compact.ToList()
+            });
+        }
+        else if (ev.Kind is NostrKind.RelayListMetadata)
+        {
+            var rTags = ev.Tags!.Where(a => a.TagIdentifier == "r").ToList();
+            if (rTags.Count > 50) return;
+
+            var compact = rTags.Select(a => new RelaySetting
+            {
+                Url = new Uri(a.AdditionalData[0].ToLower()),
+                Read = a.AdditionalData.Length == 1 || a.AdditionalData[1] == "read",
+                Write = a.AdditionalData.Length == 1 || a.AdditionalData[1] == "write"
+            });
+
+            await _redisStore.StoreUserRelays(ev.Pubkey!, new()
+            {
+                Created = ev.CreatedAt!.Value.ToUnixTimestamp(),
+                Relays = compact.ToList()
+            });
+        }
         else
         {
             await _redisStore.StoreEvent(CompactEvent.FromNostrEvent(ev));
         }
     }
+
+    class ContactsRelays
+    {
+        [JsonProperty("read")]
+        public bool Read { get; init; }
+
+        [JsonProperty("write")]
+        public bool Write { get; init; }
+    }
 }

NostrServices/Services/NostrListener.cs

@@ -1,10 +1,8 @@
 using System.Net.WebSockets;
 using System.Reflection;
-using Microsoft.EntityFrameworkCore;
 using Nostr.Client.Client;
 using Nostr.Client.Communicator;
 using Nostr.Client.Requests;
-using NostrServices.Database;
 using Websocket.Client.Models;
 
 namespace NostrServices.Services;
@@ -126,10 +124,11 @@ public class NostrListener : IDisposable
             while (!stoppingToken.IsCancellationRequested)
             {
                 using var scope = _scopeFactory.CreateScope();
-                await using var db = scope.ServiceProvider.GetRequiredService<NostrServicesContext>();
+                var db = scope.ServiceProvider.GetRequiredService<RedisStore>();
 
-                var relays = await db.Relays.Select(a => a.Url).ToListAsync(cancellationToken: stoppingToken);
-                _nostrListener.AddCommunicators(relays);
+                var relays = await db.CountRelayUsers();
+                var topRelays = relays.OrderByDescending(a => a.Value).Take(20).Select(a => a.Key).ToList();
+                _nostrListener.AddCommunicators(topRelays);
 
                 await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken);
             }

NostrServices/Services/RedisStore.cs

@@ -38,13 +38,15 @@ public class RedisStore
         return ev;
     }
 
-    public async Task StoreProfile(CompactProfile meta, TimeSpan? expiry = null)
+    public async Task<bool> StoreProfile(CompactProfile meta, TimeSpan? expiry = null)
     {
         var oldProfile = await GetProfile(meta.PubKey.ToHex());
         if ((oldProfile?.Created ?? new DateTime()) < meta.Created)
         {
-            await _database.SetAsync(ProfileKey(meta.PubKey.ToHex()), meta, expiry ?? DefaultExpire);
+            return await _database.SetAsync(ProfileKey(meta.PubKey.ToHex()), meta, expiry ?? DefaultExpire);
         }
+
+        return false;
     }
 
     public async Task<CompactProfile?> GetProfile(string id)
@@ -59,6 +61,93 @@ public class RedisStore
         return profile;
     }
 
+    public async Task<HashSet<Uri>> GetKnownRelays()
+    {
+        var ret = new HashSet<Uri>();
+        await foreach (var r in _database.SetScanAsync("relays"))
+        {
+            if (r.HasValue)
+            {
+                ret.Add(new Uri(r!));
+            }
+        }
+
+        return ret;
+    }
+
+    public async Task<bool> StoreRelay(RelayInfo cri)
+    {
+        return await _database.SetAsync(RelayKey(cri.Url), cri, DefaultExpire);
+    }
+
+    public async Task<RelayInfo?> GetRelay(Uri u)
+    {
+        return await _database.GetAsync<RelayInfo>(RelayKey(u));
+    }
+
+    public async Task<bool> StoreUserRelays(string pubkey, UserRelaySettings relays)
+    {
+        var old = await GetUserRelays(pubkey);
+        if ((old?.Created ?? 0) < relays.Created)
+        {
+            var removed = old?.Relays.Where(a => relays.Relays.All(b => b.Url != a.Url));
+            var added = old == default ? relays.Relays : relays.Relays.Where(a => old.Relays.All(b => b.Url != a.Url)).ToList();
+            if (removed != default)
+            {
+                await Task.WhenAll(removed.Select(a => _database.SetRemoveAsync(RelayUsersKey(a.Url), Convert.FromHexString(pubkey))));
+            }
+
+            await Task.WhenAll(added.Select(a => _database.SetAddAsync(RelayUsersKey(a.Url), Convert.FromHexString(pubkey))));
+            await _database.SetAddAsync("relays", added.Select(a => (RedisValue)a.Url.ToString()).ToArray());
+
+            return await _database.SetAsync(UserRelays(pubkey), relays, DefaultExpire);
+        }
+
+        return false;
+    }
+
+    public async Task<UserRelaySettings?> GetUserRelays(string pubkey)
+    {
+        return await _database.GetAsync<UserRelaySettings>(UserRelays(pubkey));
+    }
+
+    public async Task<Dictionary<Uri, long>> CountRelayUsers()
+    {
+        var allRelays = await GetKnownRelays();
+        var tasks = allRelays.Select(a => (Url: a, Task: _database.SetLengthAsync(RelayUsersKey(a)))).ToList();
+        await Task.WhenAll(tasks.Select(a => a.Task));
+
+        return tasks.ToDictionary(a => a.Url, b => b.Task.Result);
+    }
+
+    public async Task StoreRelayPosition(Uri u, string ipAddress, double lat, double lon)
+    {
+        await _database.GeoAddAsync(RelayPositionKey(), lon, lat, $"{u}\x1{ipAddress}");
+    }
+
+    public async Task<List<RelayDistance>> FindCloseRelays(double lat, double lon, int radius = 50_000, int count = 10)
+    {
+        var ret = new List<RelayDistance>();
+        var geoRelays = await _database.GeoSearchAsync(RelayPositionKey(), lon, lat, new GeoSearchCircle(radius), count);
+        foreach (var gr in geoRelays)
+        {
+            var id = ((string)gr.Member!).Split('\x1');
+            var u = new Uri(id[0]);
+            var info = await GetRelay(u);
+            if (info != default)
+            {
+                ret.Add(new()
+                {
+                    Distance = gr.Distance.HasValue ? (long)gr.Distance.Value : 0,
+                    Relay = info,
+                    IpAddress = id[1]
+                });
+            }
+        }
+
+        return ret;
+    }
+
     private string EventKey(NostrIdentifier id)
     {
         if (id is NostrAddressIdentifier naddr)
@@ -69,7 +158,11 @@ public class RedisStore
         return $"event:{id.Special}";
     }
 
-    private string ProfileKey(string id) => $"profile:{id}";
+    private static string ProfileKey(string id) => $"profile:{id}";
+    private static string RelayKey(Uri relay) => $"relay:${relay}";
+    private static string RelayPositionKey() => $"relays:geo";
+    private static string RelayUsersKey(Uri relay) => $"relay:{relay}:users";
+    private static string UserRelays(string pubkey) => $"profile:{pubkey}:relays";
 }
 
 [ProtoContract]
@@ -117,8 +210,8 @@ public class CompactEvent
             Content = ev.Content!,
             Tags = ev.Tags!.Select(a => new CompactEventTag
             {
-                Key = a.TagIdentifier!,
-                Values = a.AdditionalData.Cast<string>().ToList()
+                Key = a.TagIdentifier,
+                Values = a.AdditionalData.ToList()
             }).ToList(),
             Sig = Convert.FromHexString(ev.Sig!)
         };
@@ -195,3 +288,96 @@ public class CompactProfile
         return default;
     }
 }
+
+[ProtoContract]
+public class RelayDistance
+{
+    [ProtoMember(1)]
+    public long Distance { get; init; }
+
+    [ProtoMember(2)]
+    public string IpAddress { get; init; } = null!;
+
+    [ProtoMember(3)]
+    public RelayInfo Relay { get; init; } = null!;
+}
+
+[ProtoContract]
+public class RelayInfo
+{
+    [ProtoMember(1)]
+    public Uri Url { get; init; } = null!;
+
+    [ProtoMember(2)]
+    public long FirstSeen { get; init; }
+
+    [ProtoMember(3)]
+    public long LastSeen { get; init; }
+
+    [ProtoMember(4)]
+    public long Users { get; init; }
+
+    [ProtoMember(5)]
+    public List<Position> Positions { get; init; } = new();
+
+    [ProtoMember(6)]
+    public bool? IsPaid { get; init; }
+
+    [ProtoMember(7)]
+    public bool? IsWriteRestricted { get; init; }
+
+    [ProtoMember(8)]
+    public string? Description { get; init; }
+}
+
+[ProtoContract]
+public class Position
+{
+    [ProtoMember(1)]
+    public double Lat { get; init; }
+
+    [ProtoMember(2)]
+    public double Lon { get; init; }
+
+    [ProtoMember(3)]
+    public string? Country { get; init; }
+
+    [ProtoMember(4)]
+    public string? City { get; init; }
+
+    [ProtoMember(6)]
+    public string IpAddress { get; init; } = null!;
+}
+
+[ProtoContract]
+public class RelayEndpoint
+{
+    [ProtoMember(1)]
+    public Uri Relay { get; init; } = null!;
+
+    [ProtoMember(2)]
+    public string IpAddress { get; init; } = null!;
+}
+
+[ProtoContract]
+public class UserRelaySettings
+{
+    [ProtoMember(1)]
+    public long Created { get; init; }
+
+    [ProtoMember(2)]
+    public List<RelaySetting> Relays { get; init; } = new();
+}
+
+[ProtoContract]
+public class RelaySetting
+{
+    [ProtoMember(1)]
+    public Uri Url { get; init; } = null!;
+
+    [ProtoMember(2)]
+    public bool Read { get; init; }
+
+    [ProtoMember(3)]
+    public bool Write { get; init; }
+}

NostrServices/Services/RelayListener.cs

@@ -20,7 +20,9 @@ public class RelayListener : IHostedService
         NostrKind.LiveChatMessage,
         (NostrKind)1_313, // stream clip
         NostrKind.LongFormContent,
-        NostrKind.ClassifiedListing
+        NostrKind.ClassifiedListing,
+        NostrKind.RelayListMetadata,
+        NostrKind.Contacts
     ];
 
     private readonly NostrListener _nostr;

NostrServices/Services/RelayScraperService.cs

@@ -0,0 +1,167 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Http.Headers;
+using MaxMind.GeoIP2;
+using NBitcoin;
+using Newtonsoft.Json;
+
+namespace NostrServices.Services;
+
+public class RelayScraperService : BackgroundService
+{
+    private readonly RedisStore _redisStore;
+    private readonly IGeoIP2DatabaseReader _geoIp2Database;
+    private readonly ILogger<RelayScraperService> _logger;
+    private readonly HttpClient _client;
+
+    public RelayScraperService(IGeoIP2DatabaseReader geoIp2Database, ILogger<RelayScraperService> logger, HttpClient client,
+        RedisStore redisStore)
+    {
+        _geoIp2Database = geoIp2Database;
+        _logger = logger;
+        _client = client;
+        _redisStore = redisStore;
+        _client.Timeout = TimeSpan.FromSeconds(10);
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                var sw = Stopwatch.StartNew();
+                await UpdateRelays(stoppingToken);
+                sw.Stop();
+
+                _logger.LogInformation("Caching relay locations took {n:#,##0.00} seconds", sw.Elapsed.TotalSeconds);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to cache relay-geo info {msg}", ex.Message);
+            }
+
+            await Task.Delay(TimeSpan.FromMinutes(60), stoppingToken);
+        }
+    }
+
+    private async Task UpdateRelays(CancellationToken cts)
+    {
+        const int batchSize = 20;
+
+        var relays = await _redisStore.CountRelayUsers();
+        foreach (var batch in relays.Select((v, i) => (Batch: i / batchSize, Value: v)).GroupBy(a => a.Batch))
+        {
+            await Task.WhenAll(batch.Select(a => ScrapeRelay(a.Value, cts)));
+        }
+    }
+
+    private async Task ScrapeRelay(KeyValuePair<Uri, long> relay, CancellationToken cts)
+    {
+        try
+        {
+            _logger.LogInformation("Scraping relay: {url}", relay);
+            if (relay.Key.AbsolutePath.Contains("%20wss") || relay.Key.AbsolutePath.StartsWith("/npub1"))
+            {
+                return;
+            }
+
+            // only store relays with info doc (lazy up status)
+            var info = await GetRelayInfo(relay.Key);
+            if (info == default) return;
+
+            var ret = new RelayInfo()
+            {
+                Url = relay.Key,
+                Users = relay.Value,
+                IsPaid = info?.Limitations?.PaymentRequired,
+                IsWriteRestricted = info?.Limitations?.RestrictedWrites,
+                Description = info?.Description,
+                LastSeen = DateTime.UtcNow.ToUnixTimestamp(),
+                FirstSeen = DateTime.UtcNow.ToUnixTimestamp()
+            };
+
+            var ips = await Dns.GetHostAddressesAsync(relay.Key.DnsSafeHost, cts);
+            foreach (var ip in ips)
+            {
+                if (!ip.IsRoutable(false)) continue;
+
+                if (_geoIp2Database.TryCity(ip, out var city) && city!.Location.Longitude.HasValue && city.Location.Latitude.HasValue)
+                {
+                    ret.Positions.Add(new()
+                    {
+                        Lat = city.Location.Latitude.Value,
+                        Lon = city.Location.Longitude.Value,
+                        Country = city.Country.Name,
+                        City = city.City.Name,
+                        IpAddress = ip.ToString()
+                    });
+                }
+            }
+
+            await _redisStore.StoreRelay(ret);
+            foreach (var pos in ret.Positions)
+            {
+                await _redisStore.StoreRelayPosition(ret.Url, pos.IpAddress, pos.Lat, pos.Lon);
+            }
+        }
+        catch (Exception ex)
+        {
+            // ignored
+            _logger.LogWarning(ex, "Failed to load relay geo info {relay} {message}", relay, ex.Message);
+        }
+    }
+
+    private async Task<RelayDoc?> GetRelayInfo(Uri r)
+    {
+        try
+        {
+            var ub = new UriBuilder()
+            {
+                Scheme = r.Scheme == "wss" ? "https" : "http",
+                Host = r.Host,
+                Path = r.AbsolutePath,
+                Port = r.Port,
+            };
+
+            var rsp = await _client.SendAsync(new(HttpMethod.Get, ub.Uri)
+            {
+                Headers =
+                {
+                    Accept = {new MediaTypeWithQualityHeaderValue("application/nostr+json")}
+                }
+            });
+
+            if (rsp.IsSuccessStatusCode)
+            {
+                var json = await rsp.Content.ReadAsStringAsync();
+                var doc = JsonConvert.DeserializeObject<RelayDoc>(json);
+                return doc;
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to load relay doc for {url} {message}", r, ex.Message);
+        }
+
+        return default;
+    }
+}
+
+public class RelayDoc
+{
+    [JsonProperty("description")]
+    public string? Description { get; init; }
+
+    [JsonProperty("limitation")]
+    public RelayDocLimitation? Limitations { get; init; }
+}
+
+public class RelayDocLimitation
+{
+    [JsonProperty("payment_required")]
+    public bool? PaymentRequired { get; init; }
+
+    [JsonProperty("restricted_writes")]
+    public bool? RestrictedWrites { get; init; }
+}

NostrServices/appsettings.json

@@ -2,7 +2,8 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Warning",
+      "System.Net.Http": "Warning"
     }
   },
   "AllowedHosts": "*",
@@ -10,6 +11,7 @@
     "Database": "User ID=postgres;Password=postgres;Database=nostr-services;Pooling=true;Host=127.0.0.1:25438"
   },
   "Config": {
-    "Redis": "localhost:6377"
+    "Redis": "localhost:6377",
+    "GeoIpDatabase": "/Users/kieran/Downloads/GeoLite2-City_20230801/GeoLite2-City.mmdb"
   }
 }

docker-compose.yml

@@ -1,11 +1,4 @@
 services:
-  postgres:
-    image: "postgres:15"
-    ports:
-      - "25438:5432"
-    environment:
-      - "POSTGRES_DB=nostr-services"
-      - "POSTGRES_HOST_AUTH_METHOD=trust"
   redis:
     image: redis
     ports: