Manual load more

packages/system/src/gossip-model.ts

@@ -61,63 +61,26 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<
     ];
   }
 
-  const allRelays = unwrap(authors).map(a => {
-    return {
-      key: a,
-      relays: cache
-        .getFromCache(a)
-        ?.relays?.filter(a => a.settings.write)
-        .sort(() => (Math.random() < 0.5 ? 1 : -1)),
-    };
-  });
+  const topRelays = pickTopRelays(cache, unwrap(authors), PickNRelays);
+  const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
 
-  const missing = allRelays.filter(a => a.relays === undefined || a.relays.length === 0);
-  const hasRelays = allRelays.filter(a => a.relays !== undefined && a.relays.length > 0);
-  const relayUserMap = hasRelays.reduce((acc, v) => {
-    for (const r of unwrap(v.relays)) {
-      if (!acc.has(r.url)) {
-        acc.set(r.url, new Set([v.key]));
-      } else {
-        unwrap(acc.get(r.url)).add(v.key);
-      }
-    }
-    return acc;
-  }, new Map<string, Set<string>>());
-
-  // selection algo will just pick relays with the most users
-  const topRelays = [...relayUserMap.entries()].sort(([, v], [, v1]) => v1.size - v.size);
-
-  // <relay, key[]> - count keys per relay
-  // <key, relay[]> - pick n top relays
-  // <relay, key[]> - map keys per relay (for subscription filter)
-
-  const userPickedRelays = unwrap(authors).map(k => {
-    // pick top 3 relays for this key
-    const relaysForKey = topRelays
-      .filter(([, v]) => v.has(k))
-      .slice(0, PickNRelays)
-      .map(([k]) => k);
-    return { k, relaysForKey };
-  });
-
-  const pickedRelays = new Set(userPickedRelays.map(a => a.relaysForKey).flat());
-
-  const picked = [...pickedRelays].map(a => {
-    const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k));
+  const picked = pickedRelays.map(a => {
+    const keysOnPickedRelay = dedupe(topRelays.filter(b => b.relays.includes(a)).map(b => b.key));
     return {
       relay: a,
       filter: {
         ...filter,
-        authors: [...keysOnPickedRelay],
+        authors: keysOnPickedRelay,
       },
     } as RelayTaggedFilter;
   });
-  if (missing.length > 0) {
+  const noRelays = dedupe(topRelays.filter(a => a.relays.length === 0).map(a => a.key));
+  if (noRelays.length > 0) {
     picked.push({
       relay: "",
       filter: {
         ...filter,
-        authors: missing.map(a => a.key),
+        authors: noRelays,
       },
     });
   }
@@ -142,24 +105,20 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array<FlatReqFi
   const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
 
   const picked = pickedRelays.map(a => {
-    const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k));
+    const authorsOnRelay = new Set(topRelays.filter(v => v.relays.includes(a)).map(v => v.key));
     return {
       relay: a,
-      filter: {
-        ...filter,
-        authors: [...keysOnPickedRelay],
-      },
-    } as RelayTaggedFilter;
+      filters: input.filter(v => v.authors && authorsOnRelay.has(v.authors)),
+    } as RelayTaggedFlatFilters;
   });
-  if (missing.length > 0) {
+  const noRelays = new Set(topRelays.filter(v => v.relays.length === 0).map(v => v.key));
+  if (noRelays.size > 0) {
     picked.push({
       relay: "",
-      filter: {
-        ...filter,
-        authors: missing.map(a => a.key),
-      },
-    });
+      filters: input.filter(v => !v.authors || noRelays.has(v.authors)),
+    } as RelayTaggedFlatFilters);
   }
+
   debug("GOSSIP")("Picked %o", picked);
   return picked;
 }

packages/system/src/profile-cache.ts

@@ -126,7 +126,7 @@ export class ProfileLoaderService {
         const empty = couldNotFetch.map(a =>
           this.#cache.update({
             pubkey: a,
-            loaded: unixNowMs() - ProfileCacheExpire + 5_000, // expire in 5s
+            loaded: unixNowMs() - ProfileCacheExpire + 30_000, // expire in 30s
             created: 69,
           } as MetadataCache)
         );

packages/system/src/query.ts

@@ -250,7 +250,7 @@ export class Query implements QueryBase {
   #onProgress() {
     const isFinished = this.progress === 1;
     if (this.feed.loading !== isFinished) {
-      this.#log("%s loading=%s, progress=%d", this.id, this.feed.loading, this.progress);
+      this.#log("%s loading=%s, progress=%d, traces=%O", this.id, this.feed.loading, this.progress, this.#tracing);
       this.feed.loading = isFinished;
     }
   }

packages/system/src/request-builder.ts

@@ -4,8 +4,8 @@ import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared";
 
 import { ReqFilter, u256, HexKey, EventKind } from ".";
 import { diffFilters } from "./request-splitter";
-import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./gossip-model";
-import { mergeSimilar } from "./request-merger";
+import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model";
+import { flatMerge, mergeSimilar } from "./request-merger";
 import { FlatReqFilter, expandFilter } from "./request-expander";
 
 /**
@@ -111,10 +111,10 @@ export class RequestBuilder {
     const ts = unixNowMs() - start;
     this.#log("buildDiff %s %d ms", this.id, ts);
     if (diff.changed) {
-      return splitAllByWriteRelays(relays, diff.added).map(a => {
+      return splitFlatByWriteRelays(relays, diff.added).map(a => {
         return {
           strategy: RequestStrategy.AuthorsRelays,
-          filters: a.filters,
+          filters: flatMerge(a.filters),
           relay: a.relay,
         };
       });

packages/system/src/request-splitter.ts

@@ -1,13 +1,10 @@
 import { flatFilterEq } from "./utils";
 import { FlatReqFilter } from "./request-expander";
-import { flatMerge } from "./request-merger";
 
 export function diffFilters(prev: Array<FlatReqFilter>, next: Array<FlatReqFilter>, calcRemoved?: boolean) {
   const added = [];
   const removed = [];
 
-  prev = [...prev];
-  next = [...next];
   for (const n of next) {
     const px = prev.findIndex(a => flatFilterEq(a, n));
     if (px !== -1) {