diff --git a/packages/app/src/Element/Timeline.tsx b/packages/app/src/Element/Timeline.tsx
index 3ecc01ce..0da862db 100644
--- a/packages/app/src/Element/Timeline.tsx
+++ b/packages/app/src/Element/Timeline.tsx
@@ -8,13 +8,11 @@ import Icon from "Icons/Icon";
import { dedupeByPubkey, findTag, tagFilterOfTextRepost } from "SnortUtils";
import ProfileImage from "Element/ProfileImage";
import useTimelineFeed, { TimelineFeed, TimelineSubject } from "Feed/TimelineFeed";
-import LoadMore from "Element/LoadMore";
import Zap from "Element/Zap";
import Note from "Element/Note";
import NoteReaction from "Element/NoteReaction";
import useModeration from "Hooks/useModeration";
import ProfilePreview from "Element/ProfilePreview";
-import Skeleton from "Element/Skeleton";
import { UserCache } from "Cache";
export interface TimelineProps {
@@ -142,11 +140,11 @@ const Timeline = (props: TimelineProps) => {
)}
{mainFeed.map(eventElement)}
{(props.loadMore === undefined || props.loadMore === true) && (
- feed.loadMore()} shouldLoadMore={!feed.loading}>
-
-
-
-
+
+
+
)}
>
);
diff --git a/packages/app/src/Feed/TimelineFeed.ts b/packages/app/src/Feed/TimelineFeed.ts
index b2e498f7..862c0958 100644
--- a/packages/app/src/Feed/TimelineFeed.ts
+++ b/packages/app/src/Feed/TimelineFeed.ts
@@ -1,5 +1,5 @@
import { useCallback, useEffect, useMemo } from "react";
-import { EventKind, u256, FlatNoteStore, RequestBuilder } from "@snort/system";
+import { EventKind, FlatNoteStore, RequestBuilder } from "@snort/system";
import { useRequestBuilder } from "@snort/system-react";
import { unixNow, unwrap, tagFilterOfTextRepost } from "SnortUtils";
diff --git a/packages/system/package.json b/packages/system/package.json
index 41993eb3..9d3c71b4 100644
--- a/packages/system/package.json
+++ b/packages/system/package.json
@@ -9,7 +9,7 @@
"license": "GPL-3.0-or-later",
"scripts": {
"build": "rm -rf dist && tsc",
- "test": "jest"
+ "test": "jest --runInBand"
},
"files": [
"src",
diff --git a/packages/system/src/gossip-model.ts b/packages/system/src/gossip-model.ts
index efda9a07..ea8243e6 100644
--- a/packages/system/src/gossip-model.ts
+++ b/packages/system/src/gossip-model.ts
@@ -61,63 +61,26 @@ export function splitByWriteRelays(cache: RelayCache, filter: ReqFilter): Array<
];
}
- const allRelays = unwrap(authors).map(a => {
- return {
- key: a,
- relays: cache
- .getFromCache(a)
- ?.relays?.filter(a => a.settings.write)
- .sort(() => (Math.random() < 0.5 ? 1 : -1)),
- };
- });
+ const topRelays = pickTopRelays(cache, unwrap(authors), PickNRelays);
+ const pickedRelays = dedupe(topRelays.flatMap(a => a.relays));
- const missing = allRelays.filter(a => a.relays === undefined || a.relays.length === 0);
- const hasRelays = allRelays.filter(a => a.relays !== undefined && a.relays.length > 0);
- const relayUserMap = hasRelays.reduce((acc, v) => {
- for (const r of unwrap(v.relays)) {
- if (!acc.has(r.url)) {
- acc.set(r.url, new Set([v.key]));
- } else {
- unwrap(acc.get(r.url)).add(v.key);
- }
- }
- return acc;
- }, new Map>());
-
- // selection algo will just pick relays with the most users
- const topRelays = [...relayUserMap.entries()].sort(([, v], [, v1]) => v1.size - v.size);
-
- // - count keys per relay
- // - pick n top relays
- // - map keys per relay (for subscription filter)
-
- const userPickedRelays = unwrap(authors).map(k => {
- // pick top 3 relays for this key
- const relaysForKey = topRelays
- .filter(([, v]) => v.has(k))
- .slice(0, PickNRelays)
- .map(([k]) => k);
- return { k, relaysForKey };
- });
-
- const pickedRelays = new Set(userPickedRelays.map(a => a.relaysForKey).flat());
-
- const picked = [...pickedRelays].map(a => {
- const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k));
+ const picked = pickedRelays.map(a => {
+ const keysOnPickedRelay = dedupe(topRelays.filter(b => b.relays.includes(a)).map(b => b.key));
return {
relay: a,
filter: {
...filter,
- authors: [...keysOnPickedRelay],
+ authors: keysOnPickedRelay,
},
} as RelayTaggedFilter;
});
- if (missing.length > 0) {
+ const noRelays = dedupe(topRelays.filter(a => a.relays.length === 0).map(a => a.key));
+ if (noRelays.length > 0) {
picked.push({
relay: "",
filter: {
...filter,
- authors: missing.map(a => a.key),
+ authors: noRelays,
},
});
}
@@ -142,24 +105,20 @@ export function splitFlatByWriteRelays(cache: RelayCache, input: Array a.relays));
const picked = pickedRelays.map(a => {
- const keysOnPickedRelay = new Set(userPickedRelays.filter(b => b.relaysForKey.includes(a)).map(b => b.k));
+ const authorsOnRelay = new Set(topRelays.filter(v => v.relays.includes(a)).map(v => v.key));
return {
relay: a,
- filter: {
- ...filter,
- authors: [...keysOnPickedRelay],
- },
- } as RelayTaggedFilter;
+ filters: input.filter(v => v.authors && authorsOnRelay.has(v.authors)),
+ } as RelayTaggedFlatFilters;
});
- if (missing.length > 0) {
+ const noRelays = new Set(topRelays.filter(v => v.relays.length === 0).map(v => v.key));
+ if (noRelays.size > 0) {
picked.push({
relay: "",
- filter: {
- ...filter,
- authors: missing.map(a => a.key),
- },
- });
+ filters: input.filter(v => !v.authors || noRelays.has(v.authors)),
+ } as RelayTaggedFlatFilters);
}
+
debug("GOSSIP")("Picked %o", picked);
return picked;
}
diff --git a/packages/system/src/profile-cache.ts b/packages/system/src/profile-cache.ts
index aac74026..7f534fff 100644
--- a/packages/system/src/profile-cache.ts
+++ b/packages/system/src/profile-cache.ts
@@ -126,7 +126,7 @@ export class ProfileLoaderService {
const empty = couldNotFetch.map(a =>
this.#cache.update({
pubkey: a,
- loaded: unixNowMs() - ProfileCacheExpire + 5_000, // expire in 5s
+ loaded: unixNowMs() - ProfileCacheExpire + 30_000, // expire in 30s
created: 69,
} as MetadataCache)
);
diff --git a/packages/system/src/query.ts b/packages/system/src/query.ts
index a7421f9e..fce2fc19 100644
--- a/packages/system/src/query.ts
+++ b/packages/system/src/query.ts
@@ -250,7 +250,7 @@ export class Query implements QueryBase {
#onProgress() {
const isFinished = this.progress === 1;
if (this.feed.loading !== isFinished) {
- this.#log("%s loading=%s, progress=%d", this.id, this.feed.loading, this.progress);
+ this.#log("%s loading=%s, progress=%d, traces=%O", this.id, this.feed.loading, this.progress, this.#tracing);
this.feed.loading = isFinished;
}
}
diff --git a/packages/system/src/request-builder.ts b/packages/system/src/request-builder.ts
index 07f0ac1d..747a97a4 100644
--- a/packages/system/src/request-builder.ts
+++ b/packages/system/src/request-builder.ts
@@ -4,8 +4,8 @@ import { appendDedupe, sanitizeRelayUrl, unixNowMs } from "@snort/shared";
import { ReqFilter, u256, HexKey, EventKind } from ".";
import { diffFilters } from "./request-splitter";
-import { RelayCache, splitAllByWriteRelays, splitByWriteRelays } from "./gossip-model";
-import { mergeSimilar } from "./request-merger";
+import { RelayCache, splitByWriteRelays, splitFlatByWriteRelays } from "./gossip-model";
+import { flatMerge, mergeSimilar } from "./request-merger";
import { FlatReqFilter, expandFilter } from "./request-expander";
/**
@@ -111,10 +111,10 @@ export class RequestBuilder {
const ts = unixNowMs() - start;
this.#log("buildDiff %s %d ms", this.id, ts);
if (diff.changed) {
- return splitAllByWriteRelays(relays, diff.added).map(a => {
+ return splitFlatByWriteRelays(relays, diff.added).map(a => {
return {
strategy: RequestStrategy.AuthorsRelays,
- filters: a.filters,
+ filters: flatMerge(a.filters),
relay: a.relay,
};
});
diff --git a/packages/system/src/request-splitter.ts b/packages/system/src/request-splitter.ts
index 5dd62579..7f51295d 100644
--- a/packages/system/src/request-splitter.ts
+++ b/packages/system/src/request-splitter.ts
@@ -1,13 +1,10 @@
import { flatFilterEq } from "./utils";
import { FlatReqFilter } from "./request-expander";
-import { flatMerge } from "./request-merger";
export function diffFilters(prev: Array, next: Array, calcRemoved?: boolean) {
const added = [];
const removed = [];
- prev = [...prev];
- next = [...next];
for (const n of next) {
const px = prev.findIndex(a => flatFilterEq(a, n));
if (px !== -1) {
diff --git a/packages/system/tests/Query.test.ts b/packages/system/tests/Query.test.ts
index 11a7f139..287ccbb8 100644
--- a/packages/system/tests/Query.test.ts
+++ b/packages/system/tests/Query.test.ts
@@ -1,9 +1,9 @@
import { Connection } from "../src";
import { describe, expect } from "@jest/globals";
-import { Query } from "../src/Query";
+import { Query } from "../src/query";
import { getRandomValues } from "crypto";
-import { FlatNoteStore } from "../src/NoteCollection";
-import { RequestStrategy } from "../src/RequestBuilder";
+import { FlatNoteStore } from "../src/note-collection";
+import { RequestStrategy } from "../src/request-builder";
window.crypto = {} as any;
window.crypto.getRandomValues = getRandomValues as any;
diff --git a/packages/system/tests/EventExt.test.ts b/packages/system/tests/event-ext.test.ts
similarity index 97%
rename from packages/system/tests/EventExt.test.ts
rename to packages/system/tests/event-ext.test.ts
index b12f8b0f..3e98c35e 100644
--- a/packages/system/tests/EventExt.test.ts
+++ b/packages/system/tests/event-ext.test.ts
@@ -1,4 +1,4 @@
-import { EventExt } from "../src/EventExt";
+import { EventExt } from "../src/event-ext";
describe("NIP-10", () => {
it("should extract thread", () => {
diff --git a/packages/system/tests/GossipModel.test.ts b/packages/system/tests/gossip-model.test.ts
similarity index 91%
rename from packages/system/tests/GossipModel.test.ts
rename to packages/system/tests/gossip-model.test.ts
index 72a9b8a9..8379b9cc 100644
--- a/packages/system/tests/GossipModel.test.ts
+++ b/packages/system/tests/gossip-model.test.ts
@@ -1,4 +1,4 @@
-import { splitAllByWriteRelays } from "../src/GossipModel";
+import { splitAllByWriteRelays } from "../src/gossip-model";
describe("GossipModel", () => {
it("should not output empty", () => {
diff --git a/packages/system/tests/NoteCollection.test.ts b/packages/system/tests/note-collection.test.ts
similarity index 93%
rename from packages/system/tests/NoteCollection.test.ts
rename to packages/system/tests/note-collection.test.ts
index 613bc63d..9d4223ac 100644
--- a/packages/system/tests/NoteCollection.test.ts
+++ b/packages/system/tests/note-collection.test.ts
@@ -1,6 +1,6 @@
-import { TaggedRawEvent } from "../src/Nostr";
+import { TaggedRawEvent } from "../src/nostr";
import { describe, expect } from "@jest/globals";
-import { FlatNoteStore, ReplaceableNoteStore } from "../src/NoteCollection";
+import { FlatNoteStore, ReplaceableNoteStore } from "../src/note-collection";
describe("NoteStore", () => {
describe("flat", () => {
diff --git a/packages/system/tests/RequestBuilder.test.ts b/packages/system/tests/request-builder.test.ts
similarity index 96%
rename from packages/system/tests/RequestBuilder.test.ts
rename to packages/system/tests/request-builder.test.ts
index 0fa3a78c..adc93561 100644
--- a/packages/system/tests/RequestBuilder.test.ts
+++ b/packages/system/tests/request-builder.test.ts
@@ -1,7 +1,7 @@
-import { RelayCache } from "../src/GossipModel";
-import { RequestBuilder, RequestStrategy } from "../src/RequestBuilder";
+import { RelayCache } from "../src/gossip-model";
+import { RequestBuilder, RequestStrategy } from "../src/request-builder";
import { describe, expect } from "@jest/globals";
-import { expandFilter } from "../src/RequestExpander";
+import { expandFilter } from "../src/request-expander";
import { bytesToHex } from "@noble/curves/abstract/utils";
import { unixNow, unixNowMs } from "@snort/shared";
diff --git a/packages/system/tests/RequestExpander.test.ts b/packages/system/tests/request-expander.test.ts
similarity index 97%
rename from packages/system/tests/RequestExpander.test.ts
rename to packages/system/tests/request-expander.test.ts
index fa327146..f268fa15 100644
--- a/packages/system/tests/RequestExpander.test.ts
+++ b/packages/system/tests/request-expander.test.ts
@@ -1,4 +1,4 @@
-import { expandFilter } from "../src/RequestExpander";
+import { expandFilter } from "../src/request-expander";
describe("RequestExpander", () => {
test("expand filter", () => {
diff --git a/packages/system/tests/RequestMatcher.test.ts b/packages/system/tests/request-matcher.test.ts
similarity index 87%
rename from packages/system/tests/RequestMatcher.test.ts
rename to packages/system/tests/request-matcher.test.ts
index 0b7a4b39..e858bf6c 100644
--- a/packages/system/tests/RequestMatcher.test.ts
+++ b/packages/system/tests/request-matcher.test.ts
@@ -1,4 +1,4 @@
-import { eventMatchesFilter } from "../src/RequestMatcher";
+import { eventMatchesFilter } from "../src/request-matcher";
describe("RequestMatcher", () => {
it("should match simple filter", () => {
diff --git a/packages/system/tests/RequestMerger.test.ts b/packages/system/tests/request-merger.test.ts
similarity index 97%
rename from packages/system/tests/RequestMerger.test.ts
rename to packages/system/tests/request-merger.test.ts
index 30ff3a1a..981afb91 100644
--- a/packages/system/tests/RequestMerger.test.ts
+++ b/packages/system/tests/request-merger.test.ts
@@ -1,6 +1,6 @@
import { ReqFilter } from "../src";
-import { canMergeFilters, filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/RequestMerger";
-import { FlatReqFilter, expandFilter } from "../src/RequestExpander";
+import { canMergeFilters, filterIncludes, flatMerge, mergeSimilar, simpleMerge } from "../src/request-merger";
+import { FlatReqFilter, expandFilter } from "../src/request-expander";
describe("RequestMerger", () => {
it("should simple merge authors", () => {
diff --git a/packages/system/tests/RequestSplitter.test.ts b/packages/system/tests/request-splitter.test.ts
similarity index 71%
rename from packages/system/tests/RequestSplitter.test.ts
rename to packages/system/tests/request-splitter.test.ts
index 9d3ac867..f3553445 100644
--- a/packages/system/tests/RequestSplitter.test.ts
+++ b/packages/system/tests/request-splitter.test.ts
@@ -1,15 +1,15 @@
import { ReqFilter } from "../src";
import { describe, expect } from "@jest/globals";
-import { diffFilters } from "../src/RequestSplitter";
-import { expandFilter } from "../src/RequestExpander";
+import { diffFilters } from "../src/request-splitter";
+import { expandFilter } from "../src/request-expander";
describe("RequestSplitter", () => {
test("single filter add value", () => {
const a: Array = [{ kinds: [0], authors: ["a"] }];
const b: Array = [{ kinds: [0], authors: ["a", "b"] }];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
- added: [{ kinds: [0], authors: ["b"] }],
+ expect(diff).toMatchObject({
+ added: [{ kinds: 0, authors: "b" }],
removed: [],
changed: true,
});
@@ -18,9 +18,9 @@ describe("RequestSplitter", () => {
const a: Array = [{ kinds: [0], authors: ["a"] }];
const b: Array = [{ kinds: [0], authors: ["b"] }];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
- added: [{ kinds: [0], authors: ["b"] }],
- removed: [{ kinds: [0], authors: ["a"] }],
+ expect(diff).toMatchObject({
+ added: [{ kinds: 0, authors: "b" }],
+ removed: [{ kinds: 0, authors: "a" }],
changed: true,
});
});
@@ -28,9 +28,12 @@ describe("RequestSplitter", () => {
const a: Array = [{ kinds: [0], authors: ["a"], since: 100 }];
const b: Array = [{ kinds: [0], authors: ["a", "b"], since: 101 }];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
- added: [{ kinds: [0], authors: ["a", "b"], since: 101 }],
- removed: [{ kinds: [0], authors: ["a"], since: 100 }],
+ expect(diff).toMatchObject({
+ added: [
+ { kinds: 0, authors: "a", since: 101 },
+ { kinds: 0, authors: "b", since: 101 },
+ ],
+ removed: [{ kinds: 0, authors: "a", since: 100 }],
changed: true,
});
});
@@ -44,10 +47,10 @@ describe("RequestSplitter", () => {
{ kinds: [69], authors: ["a", "c"] },
];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
+ expect(diff).toMatchObject({
added: [
- { kinds: [0], authors: ["b"] },
- { kinds: [69], authors: ["c"] },
+ { kinds: 0, authors: "b" },
+ { kinds: 69, authors: "c" },
],
removed: [],
changed: true,
@@ -63,12 +66,15 @@ describe("RequestSplitter", () => {
{ kinds: [69], authors: ["c"] },
];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
+ expect(diff).toMatchObject({
added: [
- { kinds: [0], authors: ["b"] },
- { kinds: [69], authors: ["c"] },
+ { kinds: 0, authors: "b" },
+ { kinds: 69, authors: "c" },
+ ],
+ removed: [
+ { kinds: 0, authors: "a" },
+ { kinds: 69, authors: "a" },
],
- removed: [{ kinds: [0, 69], authors: ["a"] }],
changed: true,
});
});
@@ -79,8 +85,8 @@ describe("RequestSplitter", () => {
{ kinds: [69], authors: ["c"] },
];
const diff = diffFilters(a.flatMap(expandFilter), b.flatMap(expandFilter), true);
- expect(diff).toEqual({
- added: [{ kinds: [69], authors: ["c"] }],
+ expect(diff).toMatchObject({
+ added: [{ kinds: 69, authors: "c" }],
removed: [],
changed: true,
});
diff --git a/packages/system/tests/Util.test.ts b/packages/system/tests/utils.test.ts
similarity index 94%
rename from packages/system/tests/Util.test.ts
rename to packages/system/tests/utils.test.ts
index 9f1499b8..b7248ed4 100644
--- a/packages/system/tests/Util.test.ts
+++ b/packages/system/tests/utils.test.ts
@@ -1,5 +1,5 @@
-import { NostrPrefix } from "../src/Links";
-import { parseNostrLink, tryParseNostrLink } from "../src/NostrLink";
+import { NostrPrefix } from "../src/links";
+import { parseNostrLink, tryParseNostrLink } from "../src/nostr-link";
describe("tryParseNostrLink", () => {
it("is a valid nostr link", () => {