Collect pubkey stats

NostrServices.Client/NostrServices.Client.csproj

@@ -6,7 +6,7 @@
         <Nullable>enable</Nullable>
         <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
         <RepositoryUrl>https://git.v0l.io/Kieran/NostrServices</RepositoryUrl>
-        <Version>1.0.3</Version>
+        <Version>1.0.4</Version>
         <Authors>Kieran</Authors>
         <Description>Client wrapper for https://nostr.api.v0l.io</Description>
         <PackageProjectUrl>https://git.v0l.io/Kieran/NostrServices</PackageProjectUrl>

NostrServices.Client/NostrServicesClient.cs

@@ -41,6 +41,11 @@ public class NostrServicesClient
         return await Get<List<RelayDistance>>(HttpMethod.Get, $"/api/v1/relays/top?count={count}");
     }
 
+    public async Task<List<PubKeyStat>?> GetStats()
+    {
+        return await Get<List<PubKeyStat>>(HttpMethod.Get, "/api/v1/pubkeys");
+    }
+
     private async Task<T?> Get<T>(HttpMethod method, string path, object? body = null)
     {
         var req = new HttpRequestMessage(method, path);

NostrServices.Client/Profile.cs

@@ -1,5 +1,6 @@
 using NBitcoin.JsonConverters;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Nostr.Client.Json;
 using Nostr.Client.Messages;
 using Nostr.Client.Messages.Metadata;
@@ -41,6 +42,7 @@ public class CompactProfile
 
     [ProtoMember(8)]
     [JsonProperty("created")]
+    [JsonConverter(typeof(UnixDateTimeConverter))]
     public DateTime Created { get; init; }
 
     public static CompactProfile? FromNostrEvent(NostrEvent ev)

NostrServices.Client/PubkeyStat.cs

@@ -0,0 +1,29 @@
+using NBitcoin.JsonConverters;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using ProtoBuf;
+
+namespace NostrServices.Client;
+
+[ProtoContract]
+public class PubKeyStat
+{
+    [ProtoMember(1)]
+    [JsonProperty("pubkey")]
+    [JsonConverter(typeof(HexJsonConverter))]
+    public byte[] PubKey { get; init; } = null!;
+
+    [ProtoMember(2)]
+    [JsonProperty("last_event")]
+    [JsonConverter(typeof(UnixDateTimeConverter))]
+    public DateTime? LastEvent { get; init; }
+
+    [ProtoMember(3)]
+    [JsonProperty("total_events")]
+    public long TotalEvents { get; init; }
+    
+    [ProtoMember(4)]
+    [JsonProperty("updated")]
+    [JsonConverter(typeof(UnixDateTimeConverter))]
+    public DateTime LastUpdate { get; init; }
+}

NostrServices/Controllers/ExportController.cs

@@ -1,5 +1,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Nostr.Client.Json;
+using Nostr.Client.Messages;
+using NostrServices.Client;
 using NostrServices.Services;
 
 namespace NostrServices.Controllers;
@@ -20,7 +22,7 @@ public class ExportController : Controller
     /// <param name="id">The pubkey of the user</param>
     /// <returns></returns>
     [HttpGet("profile/{id}")]
-    [Produces("application/json")]
+    [Produces("application/json", Type = typeof(CompactProfile))]
     public async Task<IActionResult> GetProfile([FromRoute] string id)
     {
         var nid = await Extensions.TryParseIdentifier(id);
@@ -47,7 +49,7 @@ public class ExportController : Controller
     /// <param name="id">note1/nevent/naddr</param>
     /// <returns></returns>
     [HttpGet("{id}")]
-    [Produces("application/json")]
+    [Produces("application/json", Type = typeof(NostrEvent))]
     public async Task<IActionResult> GetEvent([FromRoute] string id)
     {
         var nid = await Extensions.TryParseIdentifier(id);

NostrServices/Controllers/PubkeyStatsController.cs

@@ -0,0 +1,22 @@
+using Microsoft.AspNetCore.Mvc;
+using NostrServices.Client;
+using NostrServices.Services;
+
+namespace NostrServices.Controllers;
+
+[Route("/api/v1/pubkeys")]
+public class PubkeyStatsController : Controller
+{
+    private readonly RedisStore _store;
+
+    public PubkeyStatsController(RedisStore store)
+    {
+        _store = store;
+    }
+
+    [HttpGet]
+    public async Task<List<PubKeyStat>?> GetStats()
+    {
+        return await _store.GetPubkeyStats();
+    }
+}

NostrServices/Controllers/RelaysController.cs

@@ -14,23 +14,25 @@ public class RelaysController : Controller
         _database = database;
     }
 
+    [Produces("application/json")]
     [HttpPost]
-    public async Task<IActionResult> GetCloseRelays([FromBody] LatLonReq pos, [FromQuery] int count = 5)
+    public async Task<IEnumerable<Client.RelayDistance>> GetCloseRelays([FromBody] LatLonReq pos, [FromQuery] int count = 5)
     {
         const int distance = 5000 * 1000; // 5,000km
         var relays = await _database.FindCloseRelays(pos.Lat, pos.Lon, distance, count);
 
-        return Json(relays.Select(a => a.FromDistance()));
+        return relays.Select(a => a.FromDistance());
     }
 
+    [Produces("application/json")]
     [HttpGet("top")]
-    public async Task<IActionResult> GetTop([FromQuery] int count = 10)
+    public async Task<IEnumerable<Client.RelayDistance>> GetTop([FromQuery] int count = 10)
     {
         var top = await _database.CountRelayUsers();
         var topSelected = top.OrderByDescending(a => a.Value).Take(count);
 
         var infos = await Task.WhenAll(topSelected.Select(a => _database.GetRelay(a.Key)));
-        return Json(infos.Where(a => a != default).Select(a => a!.FromInfo()));
+        return infos.Where(a => a != default).Select(a => a!.FromInfo());
     }
 }
 

NostrServices/Program.cs

@@ -32,7 +32,11 @@ public static class Program
         builder.Services.AddTransient<IDatabase>(svc => svc.GetRequiredService<ConnectionMultiplexer>().GetDatabase());
         builder.Services.AddTransient<ISubscriber>(svc => svc.GetRequiredService<ConnectionMultiplexer>().GetSubscriber());
         builder.Services.AddTransient<RedisStore>();
+        builder.Services.AddTransient<CacheRelay>();
         builder.Services.AddNostrEventHandlers();
+        builder.Services.AddHostedService<NostrListener.NostrListenerLifetime>();
+        builder.Services.AddHostedService<RelayScraperService>();
+        builder.Services.AddHostedService<PubkeyStatsService>();
 
         builder.Services.AddControllers().AddNewtonsoftJson(opt =>
         {
@@ -79,9 +83,6 @@ public static class Program
         builder.Services.AddHealthChecks();
         builder.Services.AddCors();
 
-        builder.Services.AddHostedService<NostrListener.NostrListenerLifetime>();
-        builder.Services.AddHostedService<RelayScraperService>();
-        builder.Services.AddTransient<CacheRelay>();
 
         var app = builder.Build();
         app.UseResponseCaching();

NostrServices/Services/PubkeyStatsService.cs

@@ -0,0 +1,58 @@
+using System.Diagnostics;
+using NostrServices.Client;
+
+namespace NostrServices.Services;
+
+public class PubkeyStatsService : BackgroundService
+{
+    private readonly RedisStore _store;
+    private readonly ILogger<PubkeyStatsService> _logger;
+
+    public PubkeyStatsService(RedisStore store, ILogger<PubkeyStatsService> logger)
+    {
+        _store = store;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                var sw = Stopwatch.StartNew();
+                var results = new List<PubKeyStat>();
+                var allKeys = new HashSet<string>();
+                await foreach (var k in _store.EnumerateProfiles().WithCancellation(stoppingToken))
+                {
+                    allKeys.Add(k);
+                }
+
+                _logger.LogInformation("Collected pubkeys in: {n:#,##0}ms", sw.Elapsed.TotalMilliseconds);
+
+                sw.Restart();
+                foreach (var pk in allKeys)
+                {
+                    var stat = new PubKeyStat
+                    {
+                        PubKey = Convert.FromHexString(pk),
+                        LastUpdate = DateTime.UtcNow,
+                        TotalEvents = await _store.CountUserEvents(pk)
+                    };
+
+                    results.Add(stat);
+                }
+
+                _logger.LogInformation("Computed pubkey stats in: {n:#,##0}ms", sw.Elapsed.TotalMilliseconds);
+
+                await _store.StorePubkeyStats(results);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex.Message);
+            }
+
+            await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken);
+        }
+    }
+}

NostrServices/Services/RedisStore.cs

@@ -12,15 +12,22 @@ public class RedisStore
 {
     private static readonly TimeSpan DefaultExpire = TimeSpan.FromDays(90);
     private readonly IDatabase _database;
+    private readonly ConnectionMultiplexer _connection;
 
-    public RedisStore(IDatabase database)
+    public RedisStore(ConnectionMultiplexer connection)
     {
-        _database = database;
+        _connection = connection;
+        _database = connection.GetDatabase();
     }
 
     public async Task<bool> StoreEvent(CompactEvent ev, TimeSpan? expiry = null)
     {
-        return await _database.SetAsync(EventKey(ev.ToIdentifier()), ev, expiry ?? DefaultExpire);
+        if (await _database.SetAsync(EventKey(ev.ToIdentifier()), ev, expiry ?? DefaultExpire))
+        {
+            return await _database.SetAddAsync(UserEventsKey(ev.PubKey.ToHex()), ev.Id);
+        }
+
+        return false;
     }
 
     public async Task<CompactEvent?> GetEvent(NostrIdentifier id)
@@ -97,7 +104,7 @@ public class RedisStore
             await Task.WhenAll(added.Select(a => _database.SetAddAsync(RelayUsersKey(a.Url), Convert.FromHexString(pubkey))));
             await _database.SetAddAsync("relays", added.Select(a => (RedisValue)a.Url.ToString()).ToArray());
 
-            return await _database.SetAsync(UserRelays(pubkey), relays, DefaultExpire);
+            return await _database.SetAsync(UserRelaysKey(pubkey), relays, DefaultExpire);
         }
 
         return false;
@@ -105,7 +112,7 @@ public class RedisStore
 
     public async Task<UserRelaySettings?> GetUserRelays(string pubkey)
     {
-        return await _database.GetAsync<UserRelaySettings>(UserRelays(pubkey));
+        return await _database.GetAsync<UserRelaySettings>(UserRelaysKey(pubkey));
     }
 
     public async Task<Dictionary<Uri, long>> CountRelayUsers()
@@ -117,6 +124,11 @@ public class RedisStore
         return tasks.ToDictionary(a => a.Url, b => b.Task.Result);
     }
 
+    public async Task<long> CountUserEvents(string pubkey)
+    {
+        return await _database.SetLengthAsync(UserEventsKey(pubkey));
+    }
+
     public async Task StoreRelayPosition(Uri u, string ipAddress, double lat, double lon)
     {
         await _database.GeoAddAsync(RelayPositionKey(), lon, lat, $"{u}\x1{ipAddress}");
@@ -145,6 +157,32 @@ public class RedisStore
         return ret;
     }
 
+    public async IAsyncEnumerable<string> EnumerateProfiles()
+    {
+        var servers = _connection.GetServers();
+        foreach (var server in servers)
+        {
+            await foreach (var k in server.KeysAsync())
+            {
+                var stringKey = (string)k!;
+                if (stringKey.StartsWith("profile:") && stringKey.Length == 64 + 8)
+                {
+                    yield return stringKey[8..];
+                }
+            }
+        }
+    }
+
+    public async Task<bool> StorePubkeyStats(List<PubKeyStat> stats)
+    {
+        return await _database.SetAsync(UserStatsKey(), stats);
+    }
+
+    public async Task<List<PubKeyStat>?> GetPubkeyStats()
+    {
+        return await _database.GetAsync<List<PubKeyStat>>(UserStatsKey());
+    }
+
     private string EventKey(NostrIdentifier id)
     {
         if (id is NostrAddressIdentifier naddr)
@@ -159,7 +197,9 @@ public class RedisStore
     private static string RelayKey(Uri relay) => $"relay:${relay}";
     private static string RelayPositionKey() => $"relays:geo";
     private static string RelayUsersKey(Uri relay) => $"relay:{relay}:users";
-    private static string UserRelays(string pubkey) => $"profile:{pubkey}:relays";
+    private static string UserRelaysKey(string pubkey) => $"profile:{pubkey}:relays";
+    private static string UserEventsKey(string pubkey) => $"profile:{pubkey}:events";
+    private static string UserStatsKey() => "profile-stats";
 }
 
 [ProtoContract]