refactor: introduce reth-staged-sync crate (#962)

Cargo.lock

@@ -3970,7 +3970,6 @@ dependencies = [
  "metrics",
  "metrics-exporter-prometheus",
  "metrics-util",
- "reth-cli-utils",
  "reth-consensus",
  "reth-db",
  "reth-discv4",
@@ -3983,6 +3982,7 @@ dependencies = [
  "reth-primitives",
  "reth-provider",
  "reth-rlp",
+ "reth-staged-sync",
  "reth-stages",
  "reth-tracing",
  "reth-transaction-pool",
@@ -3998,19 +3998,6 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "reth-cli-utils"
-version = "0.1.0"
-dependencies = [
- "eyre",
- "reth-db",
- "reth-primitives",
- "serde_json",
- "shellexpand",
- "tracing",
- "walkdir",
-]
-
 [[package]]
 name = "reth-codecs"
 version = "0.1.0"
@@ -4570,6 +4557,25 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "reth-staged-sync"
+version = "0.1.0"
+dependencies = [
+ "confy",
+ "eyre",
+ "reth-db",
+ "reth-discv4",
+ "reth-net-nat",
+ "reth-network",
+ "reth-primitives",
+ "reth-provider",
+ "serde",
+ "serde_json",
+ "shellexpand",
+ "tracing",
+ "walkdir",
+]
+
 [[package]]
 name = "reth-stages"
 version = "0.1.0"
@@ -5251,9 +5257,9 @@ dependencies = [
 
 [[package]]
 name = "shellexpand"
-version = "2.1.2"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4"
+checksum = "dd1c7ddea665294d484c39fd0c0d2b7e35bbfe10035c5fe1854741a57f6880e1"
 dependencies = [
  "dirs",
 ]

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "crates/net/rpc-types",
     "crates/net/downloaders",
     "crates/primitives",
+    "crates/staged-sync",
     "crates/stages",
     "crates/storage/codecs",
     "crates/storage/db",
@@ -33,6 +34,5 @@ members = [
     "crates/transaction-pool",
     "crates/metrics/metrics-derive",
     "crates/metrics/common",
-    "crates/cli/utils",
 ]
 default-members = ["bin/reth"]

bin/reth/Cargo.toml

@@ -12,6 +12,7 @@ reth-primitives = { path = "../../crates/primitives" }
 reth-db = {path = "../../crates/storage/db", features = ["mdbx", "test-utils"] }
 # TODO: Temporary use of the test-utils feature
 reth-provider = { path = "../../crates/storage/provider", features = ["test-utils"] }
+reth-staged-sync = { path = "../../crates/staged-sync" }
 reth-stages = { path = "../../crates/stages"}
 reth-interfaces = { path = "../../crates/interfaces", features = ["test-utils"] }
 reth-transaction-pool = { path = "../../crates/transaction-pool" }
@@ -22,7 +23,6 @@ reth-rlp = { path = "../../crates/common/rlp" }
 reth-network = {path = "../../crates/net/network", features = ["serde"] }
 reth-network-api = {path = "../../crates/net/network-api" }
 reth-downloaders = {path = "../../crates/net/downloaders" }
-reth-cli-utils = { path = "../../crates/cli/utils" }
 reth-tracing = { path = "../../crates/tracing" }
 reth-net-nat = { path = "../../crates/net/nat" }
 reth-discv4 = { path = "../../crates/net/discv4" }
@@ -35,7 +35,7 @@ fdlimit = "0.2.1"
 walkdir = "2.3"
 serde = "1.0"
 serde_json = "1.0"
-shellexpand = "2.1"
+shellexpand = "3.0.0"
 dirs-next = "2.0.0"
 confy = "0.5"
 

bin/reth/src/dirs.rs

@@ -1,5 +1,5 @@
 //! reth data directories.
-use reth_cli_utils::parse_path;
+use reth_staged_sync::utils::parse_path;
 use std::{
     env::VarError,
     fmt::{Debug, Display, Formatter},

bin/reth/src/lib.rs

@@ -7,7 +7,6 @@
 //! Rust Ethereum (reth) binary executable.
 
 pub mod cli;
-pub mod config;
 pub mod db;
 pub mod dirs;
 pub mod node;
@@ -15,7 +14,7 @@ pub mod p2p;
 pub mod prometheus_exporter;
 pub mod stage;
 pub mod test_eth_chain;
-pub use reth_cli_utils as utils;
+pub use reth_staged_sync::utils;
 
 use clap::Args;
 use reth_primitives::NodeRecord;

bin/reth/src/node/mod.rs

@@ -2,7 +2,6 @@
 //!
 //! Starts the client
 use crate::{
-    config::Config,
     dirs::{ConfigPath, DbPath, PlatformPath},
     prometheus_exporter,
     utils::{chainspec::chain_spec_value_parser, init::init_db, parse_socket_address},
@@ -12,7 +11,6 @@ use clap::{crate_version, Parser};
 use eyre::Context;
 use fdlimit::raise_fd_limit;
 use futures::{stream::select as stream_select, Stream, StreamExt};
-use reth_cli_utils::init::init_genesis;
 use reth_consensus::BeaconConsensus;
 use reth_downloaders::{bodies, headers};
 use reth_interfaces::consensus::ForkchoiceState;
@@ -20,6 +18,7 @@ use reth_net_nat::NatResolver;
 use reth_network::NetworkEvent;
 use reth_network_api::NetworkInfo;
 use reth_primitives::{BlockNumber, ChainSpec, H256};
+use reth_staged_sync::{utils::init::init_genesis, Config};
 use reth_stages::{
     metrics::HeaderMetrics,
     stages::{

bin/reth/src/p2p/mod.rs

@@ -1,6 +1,5 @@
 //! P2P Debugging tool
 use crate::{
-    config::Config,
     dirs::{ConfigPath, PlatformPath},
     utils::{chainspec::chain_spec_value_parser, hash_or_num_value_parser},
 };
@@ -14,6 +13,7 @@ use reth_interfaces::p2p::{
 };
 use reth_network::FetchClient;
 use reth_primitives::{BlockHashOrNumber, ChainSpec, NodeRecord, SealedHeader};
+use reth_staged_sync::Config;
 use std::sync::Arc;
 
 /// `reth p2p` command

bin/reth/src/stage/mod.rs

@@ -2,7 +2,6 @@
 //!
 //! Stage debugging tool
 use crate::{
-    config::Config,
     dirs::{ConfigPath, DbPath, PlatformPath},
     prometheus_exporter,
     utils::{chainspec::chain_spec_value_parser, init::init_db},
@@ -13,6 +12,7 @@ use reth_downloaders::bodies::concurrent::ConcurrentDownloader;
 
 use reth_net_nat::NatResolver;
 use reth_primitives::ChainSpec;
+use reth_staged_sync::Config;
 use reth_stages::{
     stages::{bodies::BodyStage, execution::ExecutionStage, sender_recovery::SenderRecoveryStage},
     ExecInput, Stage, StageId, Transaction, UnwindInput,

bin/reth/src/test_eth_chain/mod.rs

@@ -25,7 +25,7 @@ impl Command {
         let mut futs: FuturesUnordered<_> = self
             .path
             .iter()
-            .flat_map(|item| reth_cli_utils::find_all_files_with_postfix(item, ".json"))
+            .flat_map(|item| reth_staged_sync::utils::find_all_files_with_postfix(item, ".json"))
             .map(|file| async { (runner::run_test(file.clone()).await, file) })
             .collect();
 

crates/cli/utils/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "reth-cli-utils"
-version = "0.1.0"
-edition = "2021"
-license = "MIT OR Apache-2.0"
-repository = "https://github.com/paradigmxyz/reth"
-readme = "README.md"
-
-[dependencies]
-# Internal
-reth-primitives = { path = "../../primitives" }
-reth-db = {path = "../../storage/db", features = ["mdbx", "test-utils"] }
-
-# Serialiation
-serde_json = "1.0"
-eyre = "0.6.8"
-shellexpand = "2.1"
-walkdir = "2.3"
-
-# Tracing
-tracing = "0.1"

crates/staged-sync/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "reth-staged-sync"
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/paradigmxyz/reth"
+readme = "README.md"
+description = "Puts together all the Reth stages in a unified abstraction"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# reth
+reth-db = {path = "../../crates/storage/db", features = ["mdbx", "test-utils"] }
+reth-discv4 = { path = "../../crates/net/discv4" }
+reth-network = {path = "../../crates/net/network", features = ["serde"] }
+reth-primitives = { path = "../../crates/primitives" }
+reth-provider = { path = "../../crates/storage/provider", features = ["test-utils"] }
+reth-net-nat = { path = "../../crates/net/nat" }
+
+# io
+serde = "1.0"
+
+#[dev-dependencies]
+confy = "0.5"
+walkdir = "2.3.2"
+serde_json = "1.0.91"
+eyre = "0.6.8"
+shellexpand = "3.0.0"
+tracing = "0.1.37"

crates/staged-sync/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod config;
+pub use config::Config;
+
+pub mod utils;

crates/staged-sync/src/utils/mod.rs

@@ -1,10 +1,3 @@
-#![warn(missing_docs, unreachable_pub, unused_crate_dependencies)]
-#![deny(unused_must_use, rust_2018_idioms)]
-#![doc(test(
-    no_crate_inject,
-    attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
-))]
-
 //! Utility functions.
 use reth_primitives::{BlockHashOrNumber, H256};
 use std::{