Kieran/snort
5
14
Fork 14
Code Issues 37 Pull Requests 1 Packages Projects Releases 24 Wiki Activity

refactor: use nostr-social-graph
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
kieran 2024-09-21 23:22:21 +01:00
parent 304976d66d
commit 415e4e3705
Signed by: Kieran
GPG Key ID: DE71CEB3925BE941
30 changed files with 106 additions and 895 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/app/config/default.json
View File

@ -35,7 +35,7 @@
"list": "naddr1qq4xc6tnw3ez6vp58y6rywpjxckngdtyxukngwr9vckkze33vcknzcnrxcenje35xqmn2cczyp3lucccm3v9s087z6qslpkap8schltk427zfgqgrn3g2menq5zw6qcyqqq82vqprpmhxue69uhhyetvv9ujuumwdae8gtnnda3kjctv7rajfl"
},
"noteCreatorToast": false,
"hideFromNavbar": ["/graph"],
"hideFromNavbar": [],
"deckSubKind": 1,
"showPowIcon": true,
"eventLinkPrefix": "nevent",

2
packages/app/config/meku.json
View File

@ -34,7 +34,7 @@
},
"communityLeaders": null,
"noteCreatorToast": false,
"hideFromNavbar": ["/graph"],
"hideFromNavbar": [],
"deckSubKind": 1,
"showPowIcon": true,
"eventLinkPrefix": "nevent",

2
packages/app/config/nostr.json
View File

@ -33,7 +33,7 @@
},
"communityLeaders": null,
"noteCreatorToast": true,
"hideFromNavbar": ["/graph"],
"hideFromNavbar": [],
"deckSubKind": 1,
"showPowIcon": true,
"eventLinkPrefix": "nevent",

9
packages/app/src/Components/User/FollowDistanceIndicator.tsx
View File

@ -1,15 +1,16 @@
import { HexKey, socialGraphInstance } from "@snort/system";
import classNames from "classnames";
import Icon from "@/Components/Icons/Icon";
import useWoT from "@/Hooks/useWoT";
interface FollowDistanceIndicatorProps {
pubkey: HexKey;
pubkey: string;
className?: string;
}
export default function FollowDistanceIndicator({ pubkey, className }: FollowDistanceIndicatorProps) {
const followDistance = socialGraphInstance.getFollowDistance(pubkey);
const wot = useWoT();
const followDistance = wot.followDistance(pubkey);
let followDistanceColor = "";
let title = "";
@ -20,7 +21,7 @@ export default function FollowDistanceIndicator({ pubkey, className }: FollowDis
followDistanceColor = "success";
title = "Following";
} else if (followDistance === 2) {
const followedByFriendsCount = socialGraphInstance.followedByFriendsCount(pubkey);
const followedByFriendsCount = wot.followedByCount(pubkey);
if (followedByFriendsCount > 10) {
followDistanceColor = "text-nostr-orange";
}

8
packages/app/src/Components/User/FollowedBy.tsx
View File

@ -1,17 +1,19 @@
import { HexKey, socialGraphInstance } from "@snort/system";
import { HexKey } from "@snort/system";
import React, { Fragment, useMemo } from "react";
import { FormattedMessage } from "react-intl";
import { AvatarGroup } from "@/Components/User/AvatarGroup";
import DisplayName from "@/Components/User/DisplayName";
import { ProfileLink } from "@/Components/User/ProfileLink";
import useWoT from "@/Hooks/useWoT";
const MAX_FOLLOWED_BY_FRIENDS = 3;
export default function FollowedBy({ pubkey }: { pubkey: HexKey }) {
const followDistance = socialGraphInstance.getFollowDistance(pubkey);
const wot = useWoT();
const followDistance = wot.followDistance(pubkey);
const { followedByFriendsArray, totalFollowedByFriends } = useMemo(() => {
const followedByFriends = socialGraphInstance.followedByFriends(pubkey);
const followedByFriends = wot.followedByCount(pubkey);
return {
followedByFriendsArray: Array.from(followedByFriends).slice(0, MAX_FOLLOWED_BY_FRIENDS),
totalFollowedByFriends: followedByFriends.size,

6
packages/app/src/Components/User/ProfileLink.tsx
View File

@ -1,9 +1,7 @@
import { CachedMetadata, NostrLink, NostrPrefix, UserMetadata } from "@snort/system";
import { SnortContext } from "@snort/system-react";
import { ReactNode, useContext } from "react";
import { CachedMetadata, UserMetadata } from "@snort/system";
import { ReactNode } from "react";
import { Link, LinkProps } from "react-router-dom";
import { randomSample } from "@/Utils";
import { useProfileLink } from "@/Hooks/useProfileLink";
export function ProfileLink({

9
packages/app/src/Feed/FollowersFeed.ts
View File

@ -1,8 +1,11 @@
import { EventKind, HexKey, RequestBuilder, socialGraphInstance } from "@snort/system";
import { EventKind, HexKey, RequestBuilder } from "@snort/system";
import { useRequestBuilder } from "@snort/system-react";
import { useMemo } from "react";
import useWoT from "@/Hooks/useWoT";
export default function useFollowersFeed(pubkey?: HexKey) {
const wot = useWoT();
const sub = useMemo(() => {
const b = new RequestBuilder(`followers`);
if (pubkey) {
@ -17,9 +20,7 @@ export default function useFollowersFeed(pubkey?: HexKey) {
const contactLists = followersFeed?.filter(
a => a.kind === EventKind.ContactList && a.tags.some(b => b[0] === "p" && b[1] === pubkey),
);
return [...new Set(contactLists?.map(a => a.pubkey))].sort((a, b) => {
return socialGraphInstance.getFollowDistance(a) - socialGraphInstance.getFollowDistance(b);
});
return wot.sortEvents(contactLists);
}, [followersFeed, pubkey]);
return followers;

13
packages/app/src/Feed/LoginFeed.ts
View File

@ -1,3 +1,4 @@
import { unixNowMs } from "@snort/shared";
import { EventKind, RequestBuilder } from "@snort/system";
import { useRequestBuilder } from "@snort/system-react";
import { useEffect, useMemo } from "react";
@ -5,6 +6,7 @@ import { useEffect, useMemo } from "react";
import useEventPublisher from "@/Hooks/useEventPublisher";
import useLogin from "@/Hooks/useLogin";
import usePreferences from "@/Hooks/usePreferences";
import { System } from "@/system";
import { bech32ToHex, unwrap } from "@/Utils";
import { SnortPubKey } from "@/Utils/Const";
import { addSubscription } from "@/Utils/Login";
@ -23,6 +25,17 @@ export default function useLoginFeed() {
system.checkSigs = checkSigs;
}, [system, checkSigs]);
useEffect(() => {
if (pubKey) {
const start = unixNowMs();
system.config.socialGraphInstance.setRoot(pubKey);
console.debug(
`Social graph loaded in ${(unixNowMs() - start).toFixed(2)}ms`,
System.config.socialGraphInstance.size(),
);
}
}, [pubKey]);
useEffect(() => {
login.state.init(publisher?.signer, system).catch(console.error);
}, [login, publisher, system]);

14
packages/app/src/Hooks/useProfileSearch.tsx
View File

@ -1,10 +1,11 @@
import { socialGraphInstance } from "@snort/system";
import { useEffect, useMemo, useState } from "react";
import fuzzySearch from "@/Db/FuzzySearch";
import useTimelineFeed, { TimelineFeedOptions, TimelineSubject } from "@/Feed/TimelineFeed";
import { debounce } from "@/Utils";
import useWoT from "./useWoT";
const options: TimelineFeedOptions = { method: "LIMIT_UNTIL" };
export default function useProfileSearch(search: string) {
@ -34,6 +35,7 @@ export default function useProfileSearch(search: string) {
}
export function userSearch(search: string) {
const wot = useWoT();
const searchString = search.trim();
const fuseResults = fuzzySearch.search(searchString);
@ -44,7 +46,7 @@ export function userSearch(search: string) {
.map(result => {
const fuseScore = result.score === undefined ? 1 : result.score;
const followDistance = wotScore(result.item.pubkey) / followDistanceNormalizationFactor;
const followDistance = wot.followDistance(result.item.pubkey) / followDistanceNormalizationFactor;
const startsWithSearchString = [result.item.name, result.item.display_name, result.item.nip05].some(
field => field && field.toLowerCase?.().startsWith(searchString.toLowerCase()),
@ -72,11 +74,3 @@ export function userSearch(search: string) {
return combinedResults.map(r => r.item);
}
export function wotScore(pubkey: string) {
return socialGraphInstance.getFollowDistance(pubkey);
}
export function sortByWoT(pubkeys: Array<string>) {
return pubkeys.sort((a, b) => (wotScore(a) > wotScore(b) ? 1 : -1));
}

23
packages/app/src/Hooks/useWoT.ts
View File

@ -1,15 +1,28 @@
import { socialGraphInstance, TaggedNostrEvent } from "@snort/system";
import { useMemo } from "react";
import { TaggedNostrEvent } from "@snort/system";
import { SnortContext } from "@snort/system-react";
import { useContext, useMemo } from "react";
export default function useWoT() {
const system = useContext(SnortContext);
return useMemo(
() => ({
sortEvents: (events: Array<TaggedNostrEvent>) =>
events.sort(
(a, b) => socialGraphInstance.getFollowDistance(a.pubkey) - socialGraphInstance.getFollowDistance(b.pubkey),
(a, b) =>
system.config.socialGraphInstance.getFollowDistance(a.pubkey) -
system.config.socialGraphInstance.getFollowDistance(b.pubkey),
),
followDistance: (pk: string) => socialGraphInstance.getFollowDistance(pk),
sortPubkeys: (events: Array<string>) =>
events.sort(
(a, b) =>
system.config.socialGraphInstance.getFollowDistance(a) -
system.config.socialGraphInstance.getFollowDistance(b),
),
followDistance: (pk: string) => system.config.socialGraphInstance.getFollowDistance(pk),
followedByCount: (pk: string) => system.config.socialGraphInstance.followedByFriendsCount(pk),
followedBy: (pk: string) => system.config.socialGraphInstance.followedByFriends(pk),
instance: system.config.socialGraphInstance,
}),
[socialGraphInstance.root],
[system.config.socialGraphInstance],
);
}

5
packages/app/src/Pages/Layout/NavSidebar.tsx
View File

@ -44,11 +44,6 @@ const MENU_ITEMS = [
icon: "deck",
link: "/deck",
},
{
label: <FormattedMessage defaultMessage="Social Graph" />,
icon: "graph",
link: "/graph",
},
{
label: <FormattedMessage defaultMessage="About" />,
icon: "info",

6
packages/app/src/Pages/Layout/ProfileMenu.tsx
View File

@ -3,7 +3,6 @@ import classNames from "classnames";
import { FormattedMessage } from "react-intl";
import { useNavigate } from "react-router-dom";
import AsyncButton from "@/Components/Button/AsyncButton";
import Icon from "@/Components/Icons/Icon";
import ProfileImage from "@/Components/User/ProfileImage";
import ProfilePreview from "@/Components/User/ProfilePreview";
@ -79,11 +78,6 @@ export default function ProfileMenu({ className }: { className?: string }) {
/>
</MenuItem>
))}
<MenuItem>
<AsyncButton className="!bg-gray-light !text-white">
<FormattedMessage defaultMessage="Add Account" />
</AsyncButton>
</MenuItem>
</Menu>
</div>
);

13
packages/app/src/Pages/NetworkGraph/Avatar.tsx
View File

@ -1,13 +0,0 @@
import { NodeObject } from "react-force-graph-3d";
import { proxyImg } from "@/Hooks/useImgProxy";
import { GraphNode } from "@/Pages/NetworkGraph/types";
import { defaultAvatar } from "@/Utils";
import { LoginStore } from "@/Utils/Login";
export const avatar = (node: NodeObject<NodeObject<GraphNode>>) => {
const login = LoginStore.snapshot();
return node.profile?.picture
? proxyImg(node.profile?.picture, login.state.appdata?.preferences.imgProxyConfig)
: defaultAvatar(node.address);
};

248
packages/app/src/Pages/NetworkGraph/NetworkGraph.tsx
View File

@ -1,248 +0,0 @@
import { socialGraphInstance, STR, UID } from "@snort/system";
import { SnortContext } from "@snort/system-react";
import { useContext, useEffect, useState } from "react";
import { NodeObject } from "react-force-graph-3d";
import { FormattedMessage } from "react-intl";
import Icon from "@/Components/Icons/Icon";
import { avatar } from "@/Pages/NetworkGraph/Avatar";
import { Direction, GraphConfig, GraphData, GraphLink, GraphNode, NODE_LIMIT } from "@/Pages/NetworkGraph/types";
const NetworkGraph = () => {
const [graphData, setGraphData] = useState(null as GraphData | null);
const [graphConfig, setGraphConfig] = useState({
direction: Direction.OUTBOUND,
renderLimit: NODE_LIMIT,
showDistance: 2,
});
const [open, setOpen] = useState(false);
const system = useContext(SnortContext);
// const [showDistance, setShowDistance] = useState(2);
// const [direction, setDirection] = useState(Direction.OUTBOUND);
// const [renderLimit, setRenderLimit] = useState(NODE_LIMIT);
const [ForceGraph3D, setForceGraph3D] = useState(null);
const [THREE, setTHREE] = useState(null);
useEffect(() => {
// Dynamically import the modules
import("react-force-graph-3d").then(module => {
setForceGraph3D(module.default);
});
import("three").then(module => {
setTHREE(module);
});
}, []);
const handleCloseGraph = () => {
setOpen(false);
};
const handleKeyDown = (event: { key: string }) => {
if (event.key === "Escape") {
handleCloseGraph();
}
};
useEffect(() => {
if (open) {
window.addEventListener("keydown", handleKeyDown);
}
// Cleanup function to remove the event listener
return () => {
window.removeEventListener("keydown", handleKeyDown);
};
}, [open]);
const updateConfig = async (changes: Partial<GraphConfig>) => {
setGraphConfig(old => {
const newConfig = Object.assign({}, old, changes);
updateGraph(newConfig).then(graph => setGraphData(graph));
return newConfig;
});
};
const toggleConnections = () => {
if (graphConfig.direction === Direction.OUTBOUND) {
updateConfig({ direction: Direction.BOTH });
} else {
updateConfig({ direction: Direction.OUTBOUND });
}
};
const updateGraph = async (newConfig?: GraphConfig) => {
const { direction, renderLimit, showDistance } = newConfig ?? graphConfig;
const nodes = new Map<number, GraphNode>();
const links: GraphLink[] = [];
const nodesVisited = new Set<UID>();
const userCountByDistance = Array.from(
{ length: 6 },
(_, i) => socialGraphInstance.usersByFollowDistance.get(i)?.size || 0,
);
// Go through all the nodes
for (let distance = 0; distance <= showDistance; ++distance) {
const users = socialGraphInstance.usersByFollowDistance.get(distance);
if (!users) break;
for (const UID of users) {
if (renderLimit && nodes.size >= renderLimit) break; // Temporary hack
const inboundCount = socialGraphInstance.followersByUser.get(UID)?.size || 0;
const outboundCount = socialGraphInstance.followedByUser.get(UID)?.size || 0;
const pubkey = STR(UID);
const node = {
id: UID,
address: pubkey,
profile: system.profileLoader.cache.getFromCache(pubkey),
distance,
inboundCount,
outboundCount,
visible: true, // Setting to false only hides the rendered element, does not prevent calculations
// curvature: 0.6,
// Node size is based on the follower count
val: Math.log10(inboundCount) + 1, // 1 followers -> 1, 10 followers -> 2, 100 followers -> 3, etc.,
} as GraphNode;
// A visibility boost for the origin user:
if (node.distance === 0) {
node.val = 10; // they're always larger than life
node.color = "#603285";
}
nodes.set(UID, node);
}
}
// Add links
for (const node of nodes.values()) {
if (direction === Direction.OUTBOUND || direction === Direction.BOTH) {
for (const followedID of socialGraphInstance.followedByUser.get(node.id) ?? []) {
if (!nodes.has(followedID)) continue; // Skip links to nodes that we're not rendering
if (nodesVisited.has(followedID)) continue;
links.push({
source: node.id,
target: followedID,
distance: node.distance,
});
}
}
// TODO: Fix filtering
/* if (direction === Direction.INBOUND || direction === Direction.BOTH) {
for (const followerID of socialGraphInstance.followersByUser.get(node.id) ?? []) {
if (nodesVisited.has(followerID)) continue;
const follower = nodes.get(followerID);
if (!follower) continue; // Skip links to nodes that we're not rendering
links.push({
source: followerID,
target: node.id,
distance: follower.distance,
});
}
}*/
nodesVisited.add(node.id);
}
// Squash cases, where there are a lot of nodes
const graph: GraphData = {
nodes: [...nodes.values()],
links,
meta: {
nodes,
userCountByDistance,
},
};
// console.log('!!', graph);
// for (const l of links) {
// if (!nodes.has(l.source)) {
// console.log('source missing:', l.source);
// }
// if (!nodes.has(l.target)) {
// console.log('target missing:', l.target);
// }
// }
return graph;
};
const refreshData = async () => {
updateGraph().then(setGraphData);
};
useEffect(() => {
refreshData();
}, []);
if (!ForceGraph3D || !THREE) return null;
return (
<div>
{!open && (
<button
className="btn btn-primary"
onClick={() => {
setOpen(true);
refreshData();
}}>
<FormattedMessage defaultMessage="Show graph" />
</button>
)}
{open && graphData && (
<div className="fixed top-0 left-0 right-0 bottom-0 z-20">
<button className="absolute top-6 right-6 z-30 btn hover:bg-gray-900" onClick={handleCloseGraph}>
<Icon name="x" size={24} />
</button>
<div className="absolute top-6 right-0 left-0 z-20 flex flex-col content-center justify-center text-center">
<div className="text-center pb-2">Degrees of separation</div>
<div className="flex flex-row justify-center space-x-4">
{graphData.meta?.userCountByDistance?.map((value, i) => {
if (i === 0 || value <= 0) return null;
const isSelected = graphConfig.showDistance === i;
return (
<button
key={i}
className={`btn bg-gray-900 py-4 h-auto flex-col ${
isSelected ? "bg-gray-600 hover:bg-gray-600" : "hover:bg-gray-800"
}`}
onClick={() => isSelected || updateConfig({ showDistance: i })}>
<div className="text-lg block leading-none">{i}</div>
<div className="text-xs">({value})</div>
</button>
);
})}
</div>
</div>
<ForceGraph3D
graphData={graphData}
nodeThreeObject={(node: NodeObject<NodeObject<GraphNode>>) => {
const imgTexture = new THREE.TextureLoader().load(avatar(node));
imgTexture.colorSpace = THREE.SRGBColorSpace;
const material = new THREE.SpriteMaterial({ map: imgTexture });
const sprite = new THREE.Sprite(material);
sprite.scale.set(12, 12, 1);
return sprite;
}}
nodeLabel={node => `${node.profile?.name || node.address}`}
nodeAutoColorBy="distance"
linkAutoColorBy="distance"
linkDirectionalParticles={0}
nodeVisibility="visible"
numDimensions={3}
linkDirectionalArrowLength={0}
nodeOpacity={0.9}
/>
<div className="absolute bottom-6 right-6">
<button className="text-lg" onClick={() => toggleConnections()}>
Showing: {graphConfig.direction === Direction.OUTBOUND ? "Outbound" : "All"} connections
</button>
</div>
<div className="absolute bottom-6 left-6">
<span className="text-lg">Render limit: {graphConfig.renderLimit} nodes</span>
</div>
</div>
)}
</div>
);
};
export default NetworkGraph;

45
packages/app/src/Pages/NetworkGraph/types.ts
View File

@ -1,45 +0,0 @@
import { CachedMetadata, UID } from "@snort/system";
export interface GraphNode {
id: UID;
profile?: CachedMetadata;
distance: number;
val: number;
inboundCount: number;
outboundCount: number;
color?: string;
visible: boolean;
// curvature?: number;
}
export interface GraphLink {
source: UID;
target: UID;
distance: number;
}
interface GraphMetadata {
// usersByFollowDistance?: Map<number, Set<UID>>;
userCountByDistance: number[];
nodes?: Map<number, GraphNode>;
}
export interface GraphData {
nodes: GraphNode[];
links: GraphLink[];
meta?: GraphMetadata;
}
export enum Direction {
INBOUND,
OUTBOUND,
BOTH,
}
export interface GraphConfig {
direction: Direction;
renderLimit: number | null;
showDistance: number;
}
export const NODE_LIMIT = 500;

6
packages/app/src/Pages/Notifications/NotificationGroup.tsx
View File

@ -9,7 +9,7 @@ import { useNavigate } from "react-router-dom";
import NoteTime from "@/Components/Event/Note/NoteTime";
import Icon from "@/Components/Icons/Icon";
import ProfileImage from "@/Components/User/ProfileImage";
import { sortByWoT } from "@/Hooks/useProfileSearch";
import useWoT from "@/Hooks/useWoT";
import { dedupe, getDisplayName } from "@/Utils";
import { formatShort } from "@/Utils/Number";
@ -25,6 +25,7 @@ export function NotificationGroup({
}) {
const { ref, inView } = useInView({ triggerOnce: true });
const { formatMessage } = useIntl();
const wot = useWoT();
const kind = evs[0].kind;
const navigate = useNavigate();
@ -127,7 +128,8 @@ export function NotificationGroup({
<div className="flex flex-col w-max g12">
<div className="flex flex-row w-max overflow-hidden justify-between items-center">
<div className="flex flex-row">
{sortByWoT(pubkeys.filter(a => a !== "anon"))
{wot
.sortPubkeys(pubkeys.filter(a => a !== "anon"))
.slice(0, 12)
.map(v => (
<ProfileImage

9
packages/app/src/Utils/Login/MultiAccountStore.ts
View File

@ -9,7 +9,6 @@ import {
KeyStorage,
NotEncrypted,
RelaySettings,
socialGraphInstance,
UserState,
UserStateObject,
} from "@snort/system";
@ -84,10 +83,6 @@ export class MultiAccountStore extends ExternalStore<LoginSession> {
if (!this.#activeAccount) {
this.#activeAccount = this.#accounts.keys().next().value;
}
if (this.#activeAccount) {
const pubKey = this.#accounts.get(this.#activeAccount)?.publicKey;
socialGraphInstance.setRoot(pubKey || "");
}
for (const [, v] of this.#accounts) {
// reset readonly on load
if (v.type === LoginSessionType.PrivateKey && v.readonly) {
@ -146,8 +141,6 @@ export class MultiAccountStore extends ExternalStore<LoginSession> {
switchAccount(id: string) {
if (this.#accounts.has(id)) {
this.#activeAccount = id;
const pubKey = this.#accounts.get(id)?.publicKey || "";
socialGraphInstance.setRoot(pubKey);
this.#save();
}
}
@ -172,7 +165,6 @@ export class MultiAccountStore extends ExternalStore<LoginSession> {
if (this.#accounts.has(key)) {
throw new Error("Already logged in with this pubkey");
}
socialGraphInstance.setRoot(key);
const initRelays = this.decideInitRelays(relays);
const newSession = {
...LoggedOut,
@ -224,7 +216,6 @@ export class MultiAccountStore extends ExternalStore<LoginSession> {
if (this.#accounts.has(pubKey)) {
throw new Error("Already logged in with this pubkey");
}
socialGraphInstance.setRoot(pubKey);
const initRelays = this.decideInitRelays(relays);
const newSession = {
...LoggedOut,

13
packages/app/src/index.tsx
View File

@ -3,7 +3,6 @@ import "@szhsin/react-menu/dist/index.css";
import "@/assets/fonts/inter.css";
import { unixNow, unixNowMs } from "@snort/shared";
import { socialGraphInstance } from "@snort/system";
import { SnortContext } from "@snort/system-react";
import { StrictMode } from "react";
import * as ReactDOM from "react-dom/client";
@ -24,7 +23,6 @@ import HelpPage from "@/Pages/HelpPage";
import Layout from "@/Pages/Layout";
import { ListFeedPage } from "@/Pages/ListFeedPage";
import MessagesPage from "@/Pages/Messages/MessagesPage";
import NetworkGraph from "@/Pages/NetworkGraph/NetworkGraph";
import NostrAddressPage from "@/Pages/NostrAddressPage";
import NostrLinkHandler from "@/Pages/NostrLinkHandler";
import NotificationsPage from "@/Pages/Notifications/Notifications";
@ -63,11 +61,10 @@ async function initSite() {
preload(login.state.follows).then(async () => {
queueMicrotask(async () => {
const start = unixNowMs();
await System.PreloadSocialGraph(login.state.follows);
await System.PreloadSocialGraph(login.state.follows, login.publicKey);
console.debug(
`Social graph loaded in ${(unixNowMs() - start).toFixed(2)}ms, followDistances=${
socialGraphInstance.followDistanceByUser.size
}`,
`Social graph loaded in ${(unixNowMs() - start).toFixed(2)}ms`,
System.config.socialGraphInstance.size(),
);
});
@ -137,10 +134,6 @@ const mainRoutes = [
path: "/about",
element: <AboutPage />,
},
{
path: "/graph",
element: <NetworkGraph />,
},
{
path: "/wallet",
element: (

3
packages/app/src/lang.json
View File

@ -1776,9 +1776,6 @@
"hYOE+U": {
"defaultMessage": "Invite"
},
"ha8JKG": {
"defaultMessage": "Show graph"
},
"hicxcO": {
"defaultMessage": "Show replies"
},

1
packages/app/src/translations/en.json
View File

@ -589,7 +589,6 @@
"hRTfTR": "PRO",
"hY4lzx": "Supports",
"hYOE+U": "Invite",
"ha8JKG": "Show graph",
"hicxcO": "Show replies",
"hmZ3Bz": "Media",
"hniz8Z": "here",

3
packages/system/package.json
View File

@ -22,7 +22,6 @@
"@peculiar/webcrypto": "^1.4.6",
"@types/debug": "^4.1.8",
"@types/jest": "^29.5.11",
"@types/lokijs": "^1.5.14",
"@types/node": "^20.5.9",
"@types/uuid": "^9.0.2",
"@types/ws": "^8.5.5",
@ -43,8 +42,8 @@
"debug": "^4.3.4",
"eventemitter3": "^5.0.1",
"isomorphic-ws": "^5.0.0",
"lokijs": "^1.5.12",
"lru-cache": "^10.2.0",
"nostr-social-graph": "^1.0.3",
"uuid": "^9.0.0",
"ws": "^8.14.0"
}

210
packages/system/src/InMemoryDB.ts
View File

@ -1,210 +0,0 @@
import { ID, ReqFilter as Filter, STR, TaggedNostrEvent, UID } from ".";
import loki from "lokijs";
import debug from "debug";
type PackedNostrEvent = {
id: UID;
pubkey: number;
kind: number;
tags: Array<string | UID>[];
flatTags: string[];
sig: string;
created_at: number;
content?: string;
relays: string[];
saved_at: number;
};
const DEFAULT_MAX_SIZE = 5000;
class InMemoryDB {
private loki = new loki("EventDB");
private eventsCollection: Collection<PackedNostrEvent>;
private maxSize: number;
constructor(maxSize = DEFAULT_MAX_SIZE) {
this.maxSize = maxSize;
this.eventsCollection = this.loki.addCollection("events", {
unique: ["id"],
indices: ["pubkey", "kind", "flatTags", "created_at", "saved_at"],
});
this.startRemoveOldestInterval();
}
private startRemoveOldestInterval() {
const removeOldest = () => {
this.removeOldest();
setTimeout(() => removeOldest(), 3000);
};
setTimeout(() => removeOldest(), 3000);
}
#log = debug("InMemoryDB");
get(id: string): TaggedNostrEvent | undefined {
const event = this.eventsCollection.by("id", ID(id)); // throw if db not ready yet?
if (event) {
return this.unpack(event);
}
}
has(id: string): boolean {
return !!this.eventsCollection.by("id", ID(id));
}
// map to internal UIDs to save memory
private pack(event: TaggedNostrEvent): PackedNostrEvent {
return {
id: ID(event.id),
pubkey: ID(event.pubkey),
sig: event.sig,
kind: event.kind,
tags: event.tags.map(tag => {
if (["e", "p"].includes(tag[0]) && typeof tag[1] === "string") {
return [tag[0], ID(tag[1] as string), ...tag.slice(2)];
} else {
return tag;
}
}),
flatTags: event.tags.filter(tag => ["e", "p", "d"].includes(tag[0])).map(tag => `${tag[0]}_${ID(tag[1])}`),
created_at: event.created_at,
content: event.content,
relays: event.relays,
saved_at: Date.now(),
};
}
private unpack(packedEvent: PackedNostrEvent): TaggedNostrEvent {
return <TaggedNostrEvent>{
id: STR(packedEvent.id),
pubkey: STR(packedEvent.pubkey),
sig: packedEvent.sig,
kind: packedEvent.kind,
tags: packedEvent.tags.map(tag => {
if (["e", "p"].includes(tag[0] as string) && typeof tag[1] === "number") {
return [tag[0], STR(tag[1] as number), ...tag.slice(2)];
} else {
return tag;
}
}),
created_at: packedEvent.created_at,
content: packedEvent.content,
relays: packedEvent.relays,
};
}
handleEvent(event: TaggedNostrEvent): boolean {
if (!event || !event.id || !event.created_at) {
throw new Error("Invalid event");
}
const id = ID(event.id);
if (this.eventsCollection.by("id", id)) {
return false; // this prevents updating event.relays?
}
const packed = this.pack(event);
// we might want to limit the kinds of events we save, e.g. no kind 0, 3 or only 1, 6
try {
this.eventsCollection.insert(packed);
} catch (e) {
return false;
}
return true;
}
remove(eventId: string): void {
const id = ID(eventId);
this.eventsCollection.findAndRemove({ id });
}
removeOldest(): void {
const count = this.eventsCollection.count();
this.#log("InMemoryDB: count", count, this.maxSize);
if (count > this.maxSize) {
this.#log("InMemoryDB: removing oldest events", count - this.maxSize);
this.eventsCollection
.chain()
.simplesort("saved_at")
.limit(count - this.maxSize)
.remove();
}
}
find(filter: Filter, callback: (event: TaggedNostrEvent) => void): void {
this.findArray(filter).forEach(event => {
callback(event);
});
}
findArray(filter: Filter): TaggedNostrEvent[] {
const query = this.constructQuery(filter);
const searchRegex = filter.search ? new RegExp(filter.search, "i") : undefined;
let chain = this.eventsCollection
.chain()
.find(query)
.where((e: PackedNostrEvent) => {
if (searchRegex && !e.content?.match(searchRegex)) {
return false;
}
return true;
})
.simplesort("created_at", true);
if (filter.limit) {
chain = chain.limit(filter.limit);
}
return chain.data().map(e => this.unpack(e));
}
findAndRemove(filter: Filter) {
const query = this.constructQuery(filter);
this.eventsCollection.findAndRemove(query);
}
private constructQuery(filter: Filter): LokiQuery<PackedNostrEvent> {
const query: LokiQuery<PackedNostrEvent> = {};
if (filter.ids) {
query.id = { $in: filter.ids.map(ID) };
} else {
if (filter.authors) {
query.pubkey = { $in: filter.authors.map(ID) };
}
if (filter.kinds) {
query.kind = { $in: filter.kinds };
}
if (filter["#e"]) {
query.flatTags = { $contains: "e_" + filter["#e"]!.map(ID) };
} else if (filter["#p"]) {
query.flatTags = { $contains: "p_" + filter["#p"]!.map(ID) };
} else if (filter["#d"]) {
query.flatTags = { $contains: "d_" + filter["#d"]!.map(ID) };
}
if (filter.since && filter.until) {
query.created_at = { $between: [filter.since, filter.until] };
}
if (filter.since) {
query.created_at = { $gte: filter.since };
}
if (filter.until) {
query.created_at = { $lte: filter.until };
}
}
return query;
}
findOne(filter: Filter): TaggedNostrEvent | undefined {
return this.findArray(filter)[0];
}
}
export { InMemoryDB };
export default new InMemoryDB();

251
packages/system/src/SocialGraph/SocialGraph.ts
View File

@ -1,251 +0,0 @@
import { ID, STR, UID } from "./UniqueIds";
import { HexKey, NostrEvent } from "..";
import EventEmitter from "eventemitter3";
import { unixNowMs } from "@snort/shared";
import debug from "debug";
export interface SocialGraphEvents {
changeRoot: () => void;
}
export default class SocialGraph extends EventEmitter<SocialGraphEvents> {
#log = debug("SocialGraph");
root: UID;
followDistanceByUser = new Map<UID, number>();
usersByFollowDistance = new Map<number, Set<UID>>();
followedByUser = new Map<UID, Set<UID>>();
followersByUser = new Map<UID, Set<UID>>();
latestFollowEventTimestamps = new Map<UID, number>();
constructor(root: HexKey) {
super();
this.root = ID(root);
this.followDistanceByUser.set(this.root, 0);
this.usersByFollowDistance.set(0, new Set([this.root]));
}
setRoot(root: HexKey) {
const rootId = ID(root);
if (rootId === this.root) {
return;
}
const start = unixNowMs();
this.root = rootId;
this.followDistanceByUser.clear();
this.usersByFollowDistance.clear();
this.followDistanceByUser.set(this.root, 0);
this.usersByFollowDistance.set(0, new Set([this.root]));
const queue = [this.root];
while (queue.length > 0) {
const user = queue.shift()!;
const distance = this.followDistanceByUser.get(user)!;
const followers = this.followersByUser.get(user) || new Set<UID>();
for (const follower of followers) {
if (!this.followDistanceByUser.has(follower)) {
const newFollowDistance = distance + 1;
this.followDistanceByUser.set(follower, newFollowDistance);
if (!this.usersByFollowDistance.has(newFollowDistance)) {
this.usersByFollowDistance.set(newFollowDistance, new Set());
}
this.usersByFollowDistance.get(newFollowDistance)!.add(follower);
queue.push(follower);
}
}
}
this.emit("changeRoot");
this.#log(`Rebuilding root took ${(unixNowMs() - start).toFixed(2)} ms`);
}
handleEvent(evs: NostrEvent | Array<NostrEvent>) {
const filtered = (Array.isArray(evs) ? evs : [evs]).filter(a => a.kind === 3);
if (filtered.length === 0) {
return;
}
queueMicrotask(() => {
try {
for (const event of filtered) {
const author = ID(event.pubkey);
const timestamp = event.created_at;
const existingTimestamp = this.latestFollowEventTimestamps.get(author);
if (existingTimestamp && timestamp <= existingTimestamp) {
return;
}
this.latestFollowEventTimestamps.set(author, timestamp);
// Collect all users followed in the new event.
const followedInEvent = new Set<UID>();
for (const tag of event.tags) {
if (tag[0] === "p") {
const followedUser = ID(tag[1]);
if (followedUser !== author) {
followedInEvent.add(followedUser);
}
}
}
// Get the set of users currently followed by the author.
const currentlyFollowed = this.followedByUser.get(author) || new Set<UID>();
// Find users that need to be removed.
for (const user of currentlyFollowed) {
if (!followedInEvent.has(user)) {
this.removeFollower(user, author);
}
}
// Add or update the followers based on the new event.
for (const user of followedInEvent) {
this.addFollower(user, author);
}
}
} catch (e) {
// might not be logged in or sth
}
});
}
isFollowing(follower: HexKey, followedUser: HexKey): boolean {
const followedUserId = ID(followedUser);
const followerId = ID(follower);
return !!this.followedByUser.get(followerId)?.has(followedUserId);
}
getFollowDistance(user: HexKey): number {
try {
const userId = ID(user);
if (userId === this.root) {
return 0;
}
const distance = this.followDistanceByUser.get(userId);
return distance === undefined ? 1000 : distance;
} catch (e) {
// might not be logged in or sth
return 1000;
}
}
addUserByFollowDistance(distance: number, user: UID) {
if (!this.usersByFollowDistance.has(distance)) {
this.usersByFollowDistance.set(distance, new Set());
}
this.usersByFollowDistance.get(distance)?.add(user);
// remove from higher distances
for (const d of this.usersByFollowDistance.keys()) {
if (d > distance) {
this.usersByFollowDistance.get(d)?.delete(user);
}
}
}
addFollower(followedUser: UID, follower: UID) {
if (typeof followedUser !== "number" || typeof follower !== "number") {
throw new Error("Invalid user id");
}
if (!this.followersByUser.has(followedUser)) {
this.followersByUser.set(followedUser, new Set<UID>());
}
this.followersByUser.get(followedUser)?.add(follower);
if (!this.followedByUser.has(follower)) {
this.followedByUser.set(follower, new Set<UID>());
}
if (followedUser !== this.root) {
let newFollowDistance;
if (follower === this.root) {
// basically same as the next "else" block, but faster
newFollowDistance = 1;
this.addUserByFollowDistance(newFollowDistance, followedUser);
this.followDistanceByUser.set(followedUser, newFollowDistance);
} else {
const existingFollowDistance = this.followDistanceByUser.get(followedUser);
const followerDistance = this.followDistanceByUser.get(follower);
newFollowDistance = followerDistance && followerDistance + 1;
if (existingFollowDistance === undefined || (newFollowDistance && newFollowDistance < existingFollowDistance)) {
this.followDistanceByUser.set(followedUser, newFollowDistance!);
this.addUserByFollowDistance(newFollowDistance!, followedUser);
}
}
}
this.followedByUser.get(follower)?.add(followedUser);
}
removeFollower(unfollowedUser: UID, follower: UID) {
this.followersByUser.get(unfollowedUser)?.delete(follower);
this.followedByUser.get(follower)?.delete(unfollowedUser);
if (unfollowedUser === this.root) {
return;
}
// iterate over remaining followers and set the smallest follow distance
let smallest = Infinity;
for (const follower of this.followersByUser.get(unfollowedUser) || []) {
const followerDistance = this.followDistanceByUser.get(follower);
if (followerDistance !== undefined && followerDistance + 1 < smallest) {
smallest = followerDistance + 1;
}
}
if (smallest === Infinity) {
this.followDistanceByUser.delete(unfollowedUser);
} else {
this.followDistanceByUser.set(unfollowedUser, smallest);
}
}
// TODO subscription methods for followersByUser and followedByUser. and maybe messagesByTime. and replies
followerCount(address: HexKey) {
const id = ID(address);
return this.followersByUser.get(id)?.size ?? 0;
}
followedByFriendsCount(address: HexKey) {
let count = 0;
const id = ID(address);
for (const follower of this.followersByUser.get(id) ?? []) {
if (this.followedByUser.get(this.root)?.has(follower)) {
count++; // should we stop at 10?
}
}
return count;
}
followedByFriends(address: HexKey) {
const id = ID(address);
const set = new Set<HexKey>();
for (const follower of this.followersByUser.get(id) ?? []) {
if (this.followedByUser.get(this.root)?.has(follower)) {
set.add(STR(follower));
}
}
return set;
}
getFollowedByUser(user: HexKey, includeSelf = false): Set<HexKey> {
const userId = ID(user);
const set = new Set<HexKey>();
for (const id of this.followedByUser.get(userId) || []) {
set.add(STR(id));
}
if (includeSelf) {
set.add(user);
}
return set;
}
getFollowersByUser(address: HexKey): Set<HexKey> {
const userId = ID(address);
const set = new Set<HexKey>();
for (const id of this.followersByUser.get(userId) || []) {
set.add(STR(id));
}
return set;
}
}
export const socialGraphInstance = new SocialGraph("");

38
packages/system/src/SocialGraph/UniqueIds.ts
View File

@ -1,38 +0,0 @@
// should this be a class instead? convert all strings to internal representation, enable comparison
export type UID = number;
// save space by mapping strs to internal unique ids
export class UniqueIds {
static strToUniqueId = new Map<string, UID>();
static uniqueIdToStr = new Map<UID, string>();
static currentUniqueId = 0;
static id(str: string): UID {
if (str.startsWith("npub")) {
throw new Error("use hex instead of npub " + str);
}
const existing = UniqueIds.strToUniqueId.get(str);
if (existing) {
return existing;
}
const newId = UniqueIds.currentUniqueId++;
UniqueIds.strToUniqueId.set(str, newId);
UniqueIds.uniqueIdToStr.set(newId, str);
return newId;
}
static str(id: UID): string {
const pub = UniqueIds.uniqueIdToStr.get(id);
if (!pub) {
throw new Error("pub: invalid id " + id);
}
return pub;
}
static has(str: string): boolean {
return UniqueIds.strToUniqueId.has(str);
}
}
export const STR = UniqueIds.str;
export const ID = UniqueIds.id;

2
packages/system/src/index.ts
View File

@ -1,9 +1,7 @@
export { NostrSystem } from "./nostr-system";
export { NDKSystem } from "./ndk-system";
export { default as EventKind } from "./event-kind";
export { default as SocialGraph, socialGraphInstance } from "./SocialGraph/SocialGraph";
export * from "./system";
export * from "./SocialGraph/UniqueIds";
export * from "./nostr";
export * from "./links";
export * from "./nips";

36
packages/system/src/nostr-system.ts
View File

@ -11,9 +11,7 @@ import {
SystemSnapshot,
QueryLike,
OutboxModel,
socialGraphInstance,
EventKind,
ID,
SystemConfig,
} from ".";
import { RelayMetadataLoader } from "./outbox";
@ -21,6 +19,7 @@ import { ConnectionPool, DefaultConnectionPool } from "./connection-pool";
import { QueryManager } from "./query-manager";
import { RequestRouter } from "./request-router";
import { SystemBase } from "./system-base";
import { SerializedSocialGraph, SocialGraph, UniqueIds } from "nostr-social-graph";
/**
* Manages nostr content retrieval system
@ -72,7 +71,7 @@ export class NostrSystem extends SystemBase implements SystemInterface {
evBuf.push(ev);
if (!t) {
t = setTimeout(() => {
socialGraphInstance.handleEvent(evBuf);
this.config.socialGraphInstance.handleEvent(evBuf);
evBuf = [];
}, 500);
}
@ -134,18 +133,35 @@ export class NostrSystem extends SystemBase implements SystemInterface {
await this.PreloadSocialGraph(follows);
}
async PreloadSocialGraph(follows?: Array<string>) {
async PreloadSocialGraph(follows?: Array<string>, root?: string) {
// Insert data to socialGraph from cache
if (this.config.buildFollowGraph) {
for (const list of this.userFollowsCache.snapshot()) {
if (follows && !follows.includes(list.pubkey)) continue;
const user = ID(list.pubkey);
for (const fx of list.follows) {
if (fx[0] === "p" && fx[1]?.length === 64) {
socialGraphInstance.addFollower(ID(fx[1]), user);
// load saved social graph
if ("localStorage" in globalThis) {
const saved = localStorage.getItem("social-graph");
if (saved) {
try {
const data = JSON.parse(saved) as SerializedSocialGraph;
this.config.socialGraphInstance = new SocialGraph(root ?? "", data);
} catch (e) {
this.#log("Failed to load serialzied social-graph: %O", e);
localStorage.removeItem("social-graph");
}
}
}
this.config.socialGraphInstance.setRoot(root ?? "");
for (const list of this.userFollowsCache.snapshot()) {
if (follows && !follows.includes(list.pubkey)) continue;
this.config.socialGraphInstance.handleEvent({
id: "",
sig: "",
content: "",
kind: 3,
pubkey: list.pubkey,
created_at: list.created,
tags: list.follows,
});
}
}
}

8
packages/system/src/sync/connection.ts
View File

@ -44,13 +44,7 @@ export class DefaultSyncModule implements ConnectionSyncModule {
// if the event is replaceable there is no need to use any special sync query,
// just send the filters directly
const isReplaceableSync = filters.every(
a =>
a.kinds?.every(
b =>
EventExt.getType(b) === EventType.Replaceable || EventExt.getType(b) === EventType.ParameterizedReplaceable,
) ?? false,
);
const isReplaceableSync = filters.every(a => a.kinds?.every(b => EventExt.isReplaceable(b) ?? false));
if (filters.some(a => a.since || a.until || a.ids || a.limit) || isReplaceableSync) {
c.request(["REQ", id, ...filters], cb);
} else if (this.method === "since") {

2
packages/system/src/system-base.ts
View File

@ -7,6 +7,7 @@ import { UserRelaysCache, UserProfileCache, RelayMetricCache, NostrEvent } from
import { DefaultOptimizer, Optimizer } from "./query-optimizer";
import { NostrSystemEvents, SystemConfig } from "./system";
import { EventEmitter } from "eventemitter3";
import { SocialGraph } from "nostr-social-graph";
export abstract class SystemBase extends EventEmitter<NostrSystemEvents> {
#config: SystemConfig;
@ -31,6 +32,7 @@ export abstract class SystemBase extends EventEmitter<NostrSystemEvents> {
automaticOutboxModel: props.automaticOutboxModel ?? true,
buildFollowGraph: props.buildFollowGraph ?? false,
fallbackSync: props.fallbackSync ?? "since",
socialGraphInstance: props.socialGraphInstance ?? new SocialGraph(""),
};
}

6
packages/system/src/system.ts
View File

@ -11,6 +11,7 @@ import { BuiltRawReqFilter, RequestBuilder } from "./request-builder";
import { RequestRouter } from "./request-router";
import { QueryEvents } from "./query";
import EventEmitter from "eventemitter3";
import { SocialGraph } from "nostr-social-graph";
export type QueryLike = {
get progress(): number;
@ -96,6 +97,11 @@ export interface SystemConfig {
* Pick a fallback sync method when negentropy is not available
*/
fallbackSync: "since" | "range-sync";
/**
* Internal social graph used for WoT filtering
*/
socialGraphInstance: SocialGraph;
}
export interface SystemInterface {

8
yarn.lock
View File

@ -4899,6 +4899,7 @@ __metadata:
jest-environment-jsdom: "npm:^29.5.0"
lokijs: "npm:^1.5.12"
lru-cache: "npm:^10.2.0"
nostr-social-graph: "npm:^1.0.3"
ts-jest: "npm:^29.1.0"
ts-node: "npm:^10.9.1"
typescript: "npm:^5.2.2"
@ -11889,6 +11890,13 @@ __metadata:
languageName: node
linkType: hard
"nostr-social-graph@npm:^1.0.3":
version: 1.0.3
resolution: "nostr-social-graph@npm:1.0.3"
checksum: 10/dde599a75a743cc21710640ba9daf4b90aab3e54999c2d8a4ed9e2ee43abaf5d9d67d93994fc95575262f44e7ffed8caa8756fb576aabe0ad1be92ee17d1ee63
languageName: node
linkType: hard
"nostr-tools@npm:^1.15.0":
version: 1.17.0
resolution: "nostr-tools@npm:1.17.0"