feat: auto-ingest / config

Cargo.lock

@@ -27,6 +27,18 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -36,6 +48,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@@ -51,12 +69,67 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
 
+[[package]]
+name = "arraydeque"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.6"
@@ -111,6 +184,7 @@ dependencies = [
  "js-sys",
  "tokio",
  "tokio-rustls",
+ "tokio-socks",
  "tokio-tungstenite 0.24.0",
  "url",
  "wasm-bindgen",
@@ -265,6 +339,15 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -376,6 +459,100 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "clap"
+version = "4.5.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+
+[[package]]
+name = "config"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf"
+dependencies = [
+ "async-trait",
+ "convert_case",
+ "json5",
+ "nom",
+ "pathdiff",
+ "ron",
+ "rust-ini",
+ "serde",
+ "serde_json",
+ "toml",
+ "yaml-rust2",
+]
+
+[[package]]
+name = "const-random"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
+dependencies = [
+ "const-random-macro",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "tiny-keccak",
+]
+
+[[package]]
+name = "convert_case"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
@@ -415,6 +592,12 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -454,6 +637,21 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f"
+dependencies = [
+ "const-random",
+]
+
+[[package]]
+name = "either"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.35"
@@ -641,12 +839,31 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+dependencies = [
+ "ahash",
+ "allocator-api2",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 
+[[package]]
+name = "hashlink"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
+dependencies = [
+ "hashbrown 0.14.5",
+]
+
 [[package]]
 name = "headers"
 version = "0.3.9"
@@ -671,6 +888,12 @@ dependencies = [
  "http 0.2.12",
 ]
 
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
 [[package]]
 name = "hermit-abi"
 version = "0.4.0"
@@ -951,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
 dependencies = [
  "equivalent",
- "hashbrown",
+ "hashbrown 0.15.2",
 ]
 
 [[package]]
@@ -987,6 +1210,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
 [[package]]
 name = "itoa"
 version = "1.0.14"
@@ -1012,6 +1241,17 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "json5"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1"
+dependencies = [
+ "pest",
+ "pest_derive",
+ "serde",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.168"
@@ -1062,6 +1302,12 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.0"
@@ -1112,10 +1358,20 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43a88da9dd148bbcdce323dd6ac47d369b4769d4a3b78c6c52389b9269f77932"
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nostr"
 version = "0.37.0"
-source = "git+https://git.v0l.io/nostr/rust-nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
+source = "git+https://github.com/rust-nostr/nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -1140,7 +1396,7 @@ dependencies = [
 [[package]]
 name = "nostr-database"
 version = "0.37.0"
-source = "git+https://git.v0l.io/nostr/rust-nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
+source = "git+https://github.com/rust-nostr/nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
 dependencies = [
  "async-trait",
  "nostr",
@@ -1150,7 +1406,7 @@ dependencies = [
 [[package]]
 name = "nostr-relay-builder"
 version = "0.37.0"
-source = "git+https://git.v0l.io/nostr/rust-nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
+source = "git+https://github.com/rust-nostr/nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
 dependencies = [
  "async-utility",
  "async-wsocket",
@@ -1158,7 +1414,37 @@ dependencies = [
  "negentropy 0.4.3",
  "nostr",
  "nostr-database",
- "thiserror",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "nostr-relay-pool"
+version = "0.37.0"
+source = "git+https://github.com/rust-nostr/nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
+dependencies = [
+ "async-utility",
+ "async-wsocket",
+ "atomic-destructor",
+ "negentropy 0.3.1",
+ "negentropy 0.4.3",
+ "nostr",
+ "nostr-database",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "nostr-sdk"
+version = "0.37.0"
+source = "git+https://github.com/rust-nostr/nostr.git?rev=f21ffbd2de4e9ad87cd8345158039754cee05031#f21ffbd2de4e9ad87cd8345158039754cee05031"
+dependencies = [
+ "async-utility",
+ "nostr",
+ "nostr-database",
+ "nostr-relay-pool",
+ "thiserror 1.0.69",
  "tokio",
  "tracing",
 ]
@@ -1170,9 +1456,13 @@ dependencies = [
  "anyhow",
  "async-compression",
  "chrono",
+ "clap",
+ "config",
  "log",
  "nostr-relay-builder",
+ "nostr-sdk",
  "pretty_env_logger",
+ "serde",
  "sled",
  "tokio",
  "warp",
@@ -1208,6 +1498,16 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
+[[package]]
+name = "ordered-multimap"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79"
+dependencies = [
+ "dlv-list",
+ "hashbrown 0.14.5",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1244,6 +1544,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "pathdiff"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
+
 [[package]]
 name = "pbkdf2"
 version = "0.12.2"
@@ -1260,6 +1566,51 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
+[[package]]
+name = "pest"
+version = "2.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc"
+dependencies = [
+ "memchr",
+ "thiserror 2.0.7",
+ "ucd-trie",
+]
+
+[[package]]
+name = "pest_derive"
+version = "2.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e"
+dependencies = [
+ "pest",
+ "pest_generator",
+]
+
+[[package]]
+name = "pest_generator"
+version = "2.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b"
+dependencies = [
+ "pest",
+ "pest_meta",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pest_meta"
+version = "2.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea"
+dependencies = [
+ "once_cell",
+ "pest",
+ "sha2",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.7"
@@ -1382,7 +1733,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -1429,6 +1780,28 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "ron"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
+dependencies = [
+ "base64 0.21.7",
+ "bitflags 2.6.0",
+ "serde",
+ "serde_derive",
+]
+
+[[package]]
+name = "rust-ini"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a"
+dependencies = [
+ "cfg-if",
+ "ordered-multimap",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.24"
@@ -1559,6 +1932,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -1661,6 +2043,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -1704,7 +2092,16 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767"
+dependencies = [
+ "thiserror-impl 2.0.7",
 ]
 
 [[package]]
@@ -1718,6 +2115,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thiserror-impl"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.7.6"
@@ -1781,6 +2198,18 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-socks"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f"
+dependencies = [
+ "either",
+ "futures-util",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-tungstenite"
 version = "0.21.0"
@@ -1822,6 +2251,40 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "tower-service"
 version = "0.3.3"
@@ -1836,9 +2299,21 @@ checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
 dependencies = [
  "log",
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-attributes"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tracing-core"
 version = "0.1.33"
@@ -1868,7 +2343,7 @@ dependencies = [
  "log",
  "rand",
  "sha1",
- "thiserror",
+ "thiserror 1.0.69",
  "url",
  "utf-8",
 ]
@@ -1889,7 +2364,7 @@ dependencies = [
  "rustls",
  "rustls-pki-types",
  "sha1",
- "thiserror",
+ "thiserror 1.0.69",
  "utf-8",
 ]
 
@@ -1899,6 +2374,12 @@ version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
+[[package]]
+name = "ucd-trie"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
+
 [[package]]
 name = "unicase"
 version = "2.8.0"
@@ -1920,6 +2401,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
 [[package]]
 name = "universal-hash"
 version = "0.5.1"
@@ -1966,6 +2453,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -2224,6 +2717,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"
@@ -2236,6 +2738,17 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
 
+[[package]]
+name = "yaml-rust2"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8"
+dependencies = [
+ "arraydeque",
+ "encoding_rs",
+ "hashlink",
+]
+
 [[package]]
 name = "yoke"
 version = "0.7.5"

Cargo.toml

@@ -7,9 +7,13 @@ edition = "2021"
 anyhow = "1.0.94"
 async-compression = { version = "0.4.18", features = ["tokio", "zstd"] }
 chrono = "0.4.39"
+clap = { version = "4.5.23", features = ["derive"] }
+config = { version = "0.14.1", features = ["yaml"] }
 log = "0.4.22"
-nostr-relay-builder = { git = "https://git.v0l.io/nostr/rust-nostr.git", rev = "f21ffbd2de4e9ad87cd8345158039754cee05031" }
+nostr-relay-builder = { git = "https://github.com/rust-nostr/nostr.git", rev = "f21ffbd2de4e9ad87cd8345158039754cee05031", package = "nostr-relay-builder" }
 pretty_env_logger = "0.5.0"
 sled = "0.34.7"
 tokio = { version = "1.42.0", features = ["macros", "fs", "rt", "rt-multi-thread", "signal"] }
 warp = "0.3.7"
+serde = { version = "1.0.216", features = ["derive"] }
+nostr-sdk = { git = "https://github.com/rust-nostr/nostr.git", rev = "f21ffbd2de4e9ad87cd8345158039754cee05031", package = "nostr-sdk" }

config.yaml

@@ -0,0 +1,14 @@
+# Listen address for relay
+listen_relay: "0.0.0.0:8001"
+
+# Relays to connect and stream events from
+relays:
+  - "wss://relay.damus.io"
+  - "wss://nos.lol"
+  - "wss://relay.snort.social"
+  - "wss://nostr.bg"
+  - "wss://relay.primal.net"
+  - "wss://relay.nostr.band"
+
+# Filter event kinds to store in archives
+# kinds: [0,1,3,10002]

src/main.rs

@@ -1,16 +1,21 @@
 use anyhow::Result;
 use async_compression::tokio::write::ZstdEncoder;
 use chrono::{DateTime, Utc};
-use log::{error, info};
+use clap::Parser;
-use nostr_relay_builder::builder::{PolicyResult, QueryPolicy, RateLimit};
+use config::{Config, ConfigBuilder};
+use log::{debug, error, info};
+use nostr_relay_builder::builder::{PolicyResult, QueryPolicy, RateLimit, WritePolicy};
 use nostr_relay_builder::prelude::{
     async_trait, Backend, Coordinate, DatabaseError, DatabaseEventStatus, Event, EventId, Events,
-    JsonUtil, NostrDatabase, RejectedReason, Timestamp,
+    JsonUtil, Kind, NostrDatabase, RejectedReason, Timestamp,
 };
 use nostr_relay_builder::prelude::{
     Filter as RelayFilter, NostrEventsDatabase, RelayUrl, SaveEventStatus,
 };
 use nostr_relay_builder::{LocalRelay, RelayBuilder};
+use nostr_sdk::prelude::StreamExt;
+use nostr_sdk::{Client, FilterOptions};
+use serde::Deserialize;
 use std::collections::HashSet;
 use std::fmt::{Debug, Formatter};
 use std::fs::create_dir_all;
@@ -18,6 +23,7 @@ use std::io::{Error, ErrorKind};
 use std::net::{IpAddr, SocketAddr};
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::fs::{File, OpenOptions};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::sync::Mutex;
@@ -34,6 +40,20 @@ impl QueryPolicy for NoQuery {
     }
 }
 
+#[derive(Debug)]
+struct KindPolicy(HashSet<Kind>);
+
+#[async_trait]
+impl WritePolicy for KindPolicy {
+    async fn admit_event(&self, event: &Event, _addr: &SocketAddr) -> PolicyResult {
+        if self.0.contains(&event.kind) {
+            PolicyResult::Accept
+        } else {
+            PolicyResult::Reject("Kind not accepted".to_string())
+        }
+    }
+}
+
 struct FlatFileWriter {
     pub dir: PathBuf,
     pub current_date: DateTime<Utc>,
@@ -116,6 +136,7 @@ impl FlatFileWriter {
     }
 }
 
+#[derive(Clone)]
 struct FlatFileDatabase {
     database: sled::Db,
     dir: PathBuf,
@@ -160,6 +181,7 @@ impl NostrEventsDatabase for FlatFileDatabase {
                 self.write_event(event).await.map_err(|e| {
                     DatabaseError::Backend(Box::new(Error::new(ErrorKind::Other, e)))
                 })?;
+                debug!("Saved event: {}", event.id);
                 Ok(SaveEventStatus::Success)
             }
             _ => Ok(SaveEventStatus::Rejected(RejectedReason::Duplicate)),
@@ -229,21 +251,85 @@ impl NostrDatabase for FlatFileDatabase {
     }
 }
 
+#[derive(Parser)]
+#[command(version, about)]
+struct Args {
+    /// Define path for config file
+    pub config: Option<PathBuf>,
+}
+
+#[derive(Deserialize)]
+struct Settings {
+    /// Listen address for relay ip:port
+    pub listen_relay: Option<String>,
+
+    /// Nostr relays to ingest events from
+    pub relays: Option<Vec<String>>,
+
+    /// Nostr kinds to accept
+    pub kinds: Option<Vec<u32>>,
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     pretty_env_logger::init();
 
+    let args = Args::parse();
+
+    let config: Settings = Config::builder()
+        .add_source(config::File::from(
+            args.config.unwrap_or(PathBuf::from("config.yaml")),
+        ))
+        .build()?
+        .try_deserialize()?;
+
     let out_dir = PathBuf::from("./data");
-    let b = RelayBuilder::default()
+    let addr: SocketAddr = config
-        .database(FlatFileDatabase::new(out_dir.clone())?)
+        .listen_relay
-        .addr(IpAddr::from([0, 0, 0, 0]))
+        .map(|a| a.parse())
-        .port(8001)
+        .unwrap_or(Ok(SocketAddr::from(([0, 0, 0, 0], 8001))))?;
+
+    let db = FlatFileDatabase::new(out_dir.clone())?;
+    let client = Client::builder().database(db.clone()).build();
+    if let Some(r) = config.relays {
+        for r in r {
+            client.add_relay(r).await?;
+        }
+        client.connect().await;
+        let never = Duration::from_secs(u64::MAX);
+        let mut stream = client
+            .pool()
+            .stream_events(
+                vec![nostr_sdk::Filter::new().limit(100)],
+                never,
+                FilterOptions::WaitDurationAfterEOSE(never),
+            )
+            .await?;
+        let db = db.clone();
+        tokio::spawn(async move {
+            while let Some(event) = stream.next().await {
+                if let Err(e) = db.save_event(&event).await {
+                    error!("Failed to save event: {}", e);
+                }
+            }
+        });
+    }
+    let mut b = RelayBuilder::default()
+        .database(db.clone())
+        .addr(addr.ip())
+        .port(addr.port())
         .query_policy(NoQuery)
         .rate_limit(RateLimit {
             max_reqs: 20,
             notes_per_minute: 100_000,
         });
 
+    if let Some(k) = config.kinds {
+        b = b.write_policy(KindPolicy(
+            k.iter().map(|k| Kind::Custom(*k as u16)).collect(),
+        ));
+    }
+
     let relay = LocalRelay::run(b).await?;
 
     info!("Relay started on: {}", relay.url());