diff --git a/packages/app/src/Element/Timeline.tsx b/packages/app/src/Element/Timeline.tsx index 3ecc01ce..0da862db 100644 --- a/packages/app/src/Element/Timeline.tsx +++ b/packages/app/src/Element/Timeline.tsx @@ -8,13 +8,11 @@ import Icon from "Icons/Icon"; import { dedupeByPubkey, findTag, tagFilterOfTextRepost } from "SnortUtils"; import ProfileImage from "Element/ProfileImage"; import useTimelineFeed, { TimelineFeed, TimelineSubject } from "Feed/TimelineFeed"; -import LoadMore from "Element/LoadMore"; import Zap from "Element/Zap"; import Note from "Element/Note"; import NoteReaction from "Element/NoteReaction"; import useModeration from "Hooks/useModeration"; import ProfilePreview from "Element/ProfilePreview"; -import Skeleton from "Element/Skeleton"; import { UserCache } from "Cache"; export interface TimelineProps { @@ -142,11 +140,11 @@ const Timeline = (props: TimelineProps) => { )} {mainFeed.map(eventElement)} {(props.loadMore === undefined || props.loadMore === true) && ( - feed.loadMore()} shouldLoadMore={!feed.loading}> - - - - +
+ +
)} ); diff --git a/packages/app/src/Feed/TimelineFeed.ts b/packages/app/src/Feed/TimelineFeed.ts index b2e498f7..862c0958 100644 --- a/packages/app/src/Feed/TimelineFeed.ts +++ b/packages/app/src/Feed/TimelineFeed.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo } from "react"; -import { EventKind, u256, FlatNoteStore, RequestBuilder } from "@snort/system"; +import { EventKind, FlatNoteStore, RequestBuilder } from "@snort/system"; import { useRequestBuilder } from "@snort/system-react"; import { unixNow, unwrap, tagFilterOfTextRepost } from "SnortUtils"; diff --git a/packages/system/package.json b/packages/system/package.json index 41993eb3..9d3c71b4 100644 --- a/packages/system/package.json +++ b/packages/system/package.json @@ -9,7 +9,7 @@ "license": "GPL-3.0-or-later", "scripts": { "build": "rm -rf dist && tsc", - "test": "jest" + "test": "jest --runInBand" }, "files": [ "src", diff --git a/packages/system/src/gossip-model.ts b/packages/system/src/gossip-model.ts index efda9a07..ea8243e6 100644 --- a/packages/system/src/gossip-model.ts +++ b/packages/system/src/gossip-model.ts @@ -61,63 +61,26 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array< ]; } - const allRelays = unwrap(authors).map(a => { - return { - key: a, - relays: cache - .getFromCache(a) - ?.relays?.filter(a => a.settings.write) - .sort(() => (Math.random() < 0.5 ? 1 : -1)), - }; - }); + const topRelays = pickTopRelays(cache, unwrap(authors), PickNRelays); + const pickedRelays = dedupe(topRelays.flatMap(a => a.relays)); - const missing = allRelays.filter(a => a.relays === undefined || a.relays.length === 0); - const hasRelays = allRelays.filter(a => a.relays !== undefined && a.relays.length > 0); - const relayUserMap = hasRelays.reduce((acc, v) => { - for (const r of unwrap(v.relays)) { - if (!acc.has(r.url)) { - acc.set(r.url, new Set([v.key])); - } else { - unwrap(acc.get(r.url)).add(v.key); - } - } - return acc; - }, new Map>()); - - // selection algo will just pick relays with the most users - const topRelays = [...relayUserMap.entries()].sort(([, v], [, v1]) => v1.size - v.size); - - // - count keys per relay - // - pick n top relays - // - map keys per relay (for subscription filter) - - const userPickedRelays = unwrap(authors).map(k => { - // pick top 3 relays for this key - const relaysForKey = topRelays - .filter(([, v]) => v.has(k)) - .slice(0, PickNRelays) - .map(([k]) => k); - return { k, relaysForKey }; - }); - - const pickedRelays = new Set(userPickedRelays.map(a => a.relaysForKey).flat()); - - const picked = [...pickedRelays].map(a => { - const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k)); + const picked = pickedRelays.map(a => { + const keysOnPickedRelay = dedupe(topRelays.filter(b => b.relays.includes(a)).map(b => b.key)); return { relay: a, filter: { ...filter, - authors: [...keysOnPickedRelay], + authors: keysOnPickedRelay, }, } as RelayTaggedFilter; }); - if (missing.length > 0) { + const noRelays = dedupe(topRelays.filter(a => a.relays.length === 0).map(a => a.key)); + if (noRelays.length > 0) { picked.push({ relay: "", filter: { ...filter, - authors: missing.map(a => a.key), + authors: noRelays, }, }); } @@ -142,24 +105,20 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array a.relays)); const picked = pickedRelays.map(a => { - const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k)); + const authorsOnRelay = new Set(topRelays.filter(v => v.relays.includes(a)).map(v => v.key)); return { relay: a, - filter: { - ...filter, - authors: [...keysOnPickedRelay], - }, - } as RelayTaggedFilter; + filters: input.filter(v => v.authors && authorsOnRelay.has(v.authors)), + } as RelayTaggedFlatFilters; }); - if (missing.length > 0) { + const noRelays = new Set(topRelays.filter(v => v.relays.length === 0).map(v => v.key)); + if (noRelays.size > 0) { picked.push({ relay: "", - filter: { - ...filter, - authors: missing.map(a => a.key), - }, - }); + filters: input.filter(v => !v.authors || noRelays.has(v.authors)), + } as RelayTaggedFlatFilters); } + debug("GOSSIP")("Picked %o", picked); return picked; } diff --git a/packages/system/src/profile-cache.ts b/packages/system/src/profile-cache.ts index aac74026..7f534fff 100644 --- a/packages/system/src/profile-cache.ts +++ b/packages/system/src/profile-cache.ts @@ -126,7 +126,7 @@ export class ProfileLoaderService { const empty = couldNotFetch.map(a => this.#cache.update({ pubkey: a, - loaded: unixNowMs() - ProfileCacheExpire + 5_000, // expire in 5s + loaded: unixNowMs() - ProfileCacheExpire + 30_000, // expire in 30s created: 69, } as MetadataCache) ); diff --git a/packages/system/src/query.ts b/packages/system/src/query.ts index a7421f9e..fce2fc19 100644 --- a/packages/system/src/query.ts +++ b/packages/system/src/query.ts @@ -250,7 +250,7 @@ export class Query implements QueryBase { #onProgress() { const isFinished = this.progress === 1; if (this.feed.loading !== isFinished) { - this.#log("%s loading=%s, progress=%d", this.id, this.feed.loading, this.progress); + this.#log("%s loading=%s, progress=%d, traces=%O", this.id, this.feed.loading, this.progress, this.#tracing); this.feed.loading = isFinished; } } diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts index 07f0ac1d..747a97a4 100644 --- a/packages/system/src/request-builder.ts +++ b/packages/system/src/request-builder.ts @@ -4,8 +4,8 @@ import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared"; import { ReqFilter, u256, HexKey, EventKind } from "."; import { diffFilters } from "./request-splitter"; -import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./gossip-model"; -import { mergeSimilar } from "./request-merger"; +import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model"; +import { flatMerge, mergeSimilar } from "./request-merger"; import { FlatReqFilter, expandFilter } from "./request-expander"; /** @@ -111,10 +111,10 @@ export class RequestBuilder { const ts = unixNowMs() - start; this.#log("buildDiff %s %d ms", this.id, ts); if (diff.changed) { - return splitAllByWriteRelays(relays, diff.added).map(a => { + return splitFlatByWriteRelays(relays, diff.added).map(a => { return { strategy: RequestStrategy.AuthorsRelays, - filters: a.filters, + filters: flatMerge(a.filters), relay: a.relay, }; }); diff --git a/packages/system/src/request-splitter.ts b/packages/system/src/request-splitter.ts index 5dd62579..7f51295d 100644 --- a/packages/system/src/request-splitter.ts +++ b/packages/system/src/request-splitter.ts @@ -1,13 +1,10 @@ import { flatFilterEq } from "./utils"; import { FlatReqFilter } from "./request-expander"; -import { flatMerge } from "./request-merger"; export function diffFilters(prev: Array, next: Array, calcRemoved?: boolean) { const added = []; const removed = []; - prev = [...prev]; - next = [...next]; for (const n of next) { const px = prev.findIndex(a => flatFilterEq(a, n)); if (px !== -1) { diff --git a/packages/system/tests/Query.test.ts b/packages/system/tests/Query.test.ts index 11a7f139..287ccbb8 100644 --- a/packages/system/tests/Query.test.ts +++ b/packages/system/tests/Query.test.ts @@ -1,9 +1,9 @@ import { Connection } from "../src"; import { describe, expect } from "@jest/globals"; -import { Query } from "../src/Query"; +import { Query } from "../src/query"; import { getRandomValues } from "crypto"; -import { FlatNoteStore } from "../src/NoteCollection"; -import { RequestStrategy } from "../src/RequestBuilder"; +import { FlatNoteStore } from "../src/note-collection"; +import { RequestStrategy } from "../src/request-builder"; window.crypto = {} as any; window.crypto.getRandomValues = getRandomValues as any; diff --git a/packages/system/tests/EventExt.test.ts b/packages/system/tests/event-ext.test.ts similarity index 97% rename from packages/system/tests/EventExt.test.ts rename to packages/system/tests/event-ext.test.ts index b12f8b0f..3e98c35e 100644 --- a/packages/system/tests/EventExt.test.ts +++ b/packages/system/tests/event-ext.test.ts @@ -1,4 +1,4 @@ -import { EventExt } from "../src/EventExt"; +import { EventExt } from "../src/event-ext"; describe("NIP-10", () => { it("should extract thread", () => { diff --git a/packages/system/tests/GossipModel.test.ts b/packages/system/tests/gossip-model.test.ts similarity index 91% rename from packages/system/tests/GossipModel.test.ts rename to packages/system/tests/gossip-model.test.ts index 72a9b8a9..8379b9cc 100644 --- a/packages/system/tests/GossipModel.test.ts +++ b/packages/system/tests/gossip-model.test.ts @@ -1,4 +1,4 @@ -import { splitAllByWriteRelays } from "../src/GossipModel"; +import { splitAllByWriteRelays } from "../src/gossip-model"; describe("GossipModel", () => { it("should not output empty", () => { diff --git a/packages/system/tests/NoteCollection.test.ts b/packages/system/tests/note-collection.test.ts similarity index 93% rename from packages/system/tests/NoteCollection.test.ts rename to packages/system/tests/note-collection.test.ts index 613bc63d..9d4223ac 100644 --- a/packages/system/tests/NoteCollection.test.ts +++ b/packages/system/tests/note-collection.test.ts @@ -1,6 +1,6 @@ -import { TaggedRawEvent } from "../src/Nostr"; +import { TaggedRawEvent } from "../src/nostr"; import { describe, expect } from "@jest/globals"; -import { FlatNoteStore, ReplaceableNoteStore } from "../src/NoteCollection"; +import { FlatNoteStore, ReplaceableNoteStore } from "../src/note-collection"; describe("NoteStore", () => { describe("flat", () => { diff --git a/packages/system/tests/RequestBuilder.test.ts b/packages/system/tests/request-builder.test.ts similarity index 96% rename from packages/system/tests/RequestBuilder.test.ts rename to packages/system/tests/request-builder.test.ts index 0fa3a78c..adc93561 100644 --- a/packages/system/tests/RequestBuilder.test.ts +++ b/packages/system/tests/request-builder.test.ts @@ -1,7 +1,7 @@ -import { RelayCache } from "../src/GossipModel"; -import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder"; +import { RelayCache } from "../src/gossip-model"; +import { RequestBuilder, RequestStrategy } from "../src/request-builder"; import { describe, expect } from "@jest/globals"; -import { expandFilter } from "../src/RequestExpander"; +import { expandFilter } from "../src/request-expander"; import { bytesToHex } from "@noble/curves/abstract/utils"; import { unixNow, unixNowMs } from "@snort/shared"; diff --git a/packages/system/tests/RequestExpander.test.ts b/packages/system/tests/request-expander.test.ts similarity index 97% rename from packages/system/tests/RequestExpander.test.ts rename to packages/system/tests/request-expander.test.ts index fa327146..f268fa15 100644 --- a/packages/system/tests/RequestExpander.test.ts +++ b/packages/system/tests/request-expander.test.ts @@ -1,4 +1,4 @@ -import { expandFilter } from "../src/RequestExpander"; +import { expandFilter } from "../src/request-expander"; describe("RequestExpander", () => { test("expand filter", () => { diff --git a/packages/system/tests/RequestMatcher.test.ts b/packages/system/tests/request-matcher.test.ts similarity index 87% rename from packages/system/tests/RequestMatcher.test.ts rename to packages/system/tests/request-matcher.test.ts index 0b7a4b39..e858bf6c 100644 --- a/packages/system/tests/RequestMatcher.test.ts +++ b/packages/system/tests/request-matcher.test.ts @@ -1,4 +1,4 @@ -import { eventMatchesFilter } from "../src/RequestMatcher"; +import { eventMatchesFilter } from "../src/request-matcher"; describe("RequestMatcher", () => { it("should match simple filter", () => { diff --git a/packages/system/tests/RequestMerger.test.ts b/packages/system/tests/request-merger.test.ts similarity index 97% rename from packages/system/tests/RequestMerger.test.ts rename to packages/system/tests/request-merger.test.ts index 30ff3a1a..981afb91 100644 --- a/packages/system/tests/RequestMerger.test.ts +++ b/packages/system/tests/request-merger.test.ts @@ -1,6 +1,6 @@ import { ReqFilter } from "../src"; -import { canMergeFilters, filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/RequestMerger"; -import { FlatReqFilter, expandFilter } from "../src/RequestExpander"; +import { canMergeFilters, filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/request-merger"; +import { FlatReqFilter, expandFilter } from "../src/request-expander"; describe("RequestMerger", () => { it("should simple merge authors", () => { diff --git a/packages/system/tests/RequestSplitter.test.ts b/packages/system/tests/request-splitter.test.ts similarity index 71% rename from packages/system/tests/RequestSplitter.test.ts rename to packages/system/tests/request-splitter.test.ts index 9d3ac867..f3553445 100644 --- a/packages/system/tests/RequestSplitter.test.ts +++ b/packages/system/tests/request-splitter.test.ts @@ -1,15 +1,15 @@ import { ReqFilter } from "../src"; import { describe, expect } from "@jest/globals"; -import { diffFilters } from "../src/RequestSplitter"; -import { expandFilter } from "../src/RequestExpander"; +import { diffFilters } from "../src/request-splitter"; +import { expandFilter } from "../src/request-expander"; describe("RequestSplitter", () => { test("single filter add value", () => { const a: Array = [{ kinds: [0], authors: ["a"] }]; const b: Array = [{ kinds: [0], authors: ["a", "b"] }]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ - added: [{ kinds: [0], authors: ["b"] }], + expect(diff).toMatchObject({ + added: [{ kinds: 0, authors: "b" }], removed: [], changed: true, }); @@ -18,9 +18,9 @@ describe("RequestSplitter", () => { const a: Array = [{ kinds: [0], authors: ["a"] }]; const b: Array = [{ kinds: [0], authors: ["b"] }]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ - added: [{ kinds: [0], authors: ["b"] }], - removed: [{ kinds: [0], authors: ["a"] }], + expect(diff).toMatchObject({ + added: [{ kinds: 0, authors: "b" }], + removed: [{ kinds: 0, authors: "a" }], changed: true, }); }); @@ -28,9 +28,12 @@ describe("RequestSplitter", () => { const a: Array = [{ kinds: [0], authors: ["a"], since: 100 }]; const b: Array = [{ kinds: [0], authors: ["a", "b"], since: 101 }]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ - added: [{ kinds: [0], authors: ["a", "b"], since: 101 }], - removed: [{ kinds: [0], authors: ["a"], since: 100 }], + expect(diff).toMatchObject({ + added: [ + { kinds: 0, authors: "a", since: 101 }, + { kinds: 0, authors: "b", since: 101 }, + ], + removed: [{ kinds: 0, authors: "a", since: 100 }], changed: true, }); }); @@ -44,10 +47,10 @@ describe("RequestSplitter", () => { { kinds: [69], authors: ["a", "c"] }, ]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ + expect(diff).toMatchObject({ added: [ - { kinds: [0], authors: ["b"] }, - { kinds: [69], authors: ["c"] }, + { kinds: 0, authors: "b" }, + { kinds: 69, authors: "c" }, ], removed: [], changed: true, @@ -63,12 +66,15 @@ describe("RequestSplitter", () => { { kinds: [69], authors: ["c"] }, ]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ + expect(diff).toMatchObject({ added: [ - { kinds: [0], authors: ["b"] }, - { kinds: [69], authors: ["c"] }, + { kinds: 0, authors: "b" }, + { kinds: 69, authors: "c" }, + ], + removed: [ + { kinds: 0, authors: "a" }, + { kinds: 69, authors: "a" }, ], - removed: [{ kinds: [0, 69], authors: ["a"] }], changed: true, }); }); @@ -79,8 +85,8 @@ describe("RequestSplitter", () => { { kinds: [69], authors: ["c"] }, ]; const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true); - expect(diff).toEqual({ - added: [{ kinds: [69], authors: ["c"] }], + expect(diff).toMatchObject({ + added: [{ kinds: 69, authors: "c" }], removed: [], changed: true, }); diff --git a/packages/system/tests/Util.test.ts b/packages/system/tests/utils.test.ts similarity index 94% rename from packages/system/tests/Util.test.ts rename to packages/system/tests/utils.test.ts index 9f1499b8..b7248ed4 100644 --- a/packages/system/tests/Util.test.ts +++ b/packages/system/tests/utils.test.ts @@ -1,5 +1,5 @@ -import { NostrPrefix } from "../src/Links"; -import { parseNostrLink, tryParseNostrLink } from "../src/NostrLink"; +import { NostrPrefix } from "../src/links"; +import { parseNostrLink, tryParseNostrLink } from "../src/nostr-link"; describe("tryParseNostrLink", () => { it("is a valid nostr link", () => {