diff --git a/Cargo.lock b/Cargo.lock index 4822e4397..56be0b40c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -958,6 +958,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "discv5" version = "0.1.0" @@ -2969,6 +2989,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.8", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.6.0" @@ -3010,13 +3041,21 @@ version = "0.1.0" dependencies = [ "clap 4.0.18", "eyre", + "reth-consensus", + "reth-db", + "reth-interfaces", "reth-primitives", + "reth-rpc", + "reth-stages", + "reth-transaction-pool", "serde", "serde_json", + "shellexpand", "thiserror", "tokio", "tracing", "tracing-futures", + "tracing-subscriber", "walkdir", ] @@ -3902,6 +3941,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "shlex" version = "1.1.0" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 2a00ce0ce..95c41647d 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -7,13 +7,27 @@ repository = "https://github.com/foundry-rs/reth" readme = "README.md" [dependencies] +# reth +reth-primitives = { path = "../../crates/primitives" } +reth-db = {path = "../../crates/db"} +reth-stages = {path = "../../crates/stages"} +reth-interfaces = {path = "../../crates/interfaces"} +reth-transaction-pool = {path = "../../crates/transaction-pool"} +reth-consensus = {path = "../../crates/consensus"} +reth-rpc = {path = "../../crates/net/rpc"} + +# tracing +tracing = "0.1" +tracing-futures = "0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# mics +shellexpand = "2.1" + eyre = "0.6.8" clap = { version = "4.0", features = ["derive"] } thiserror = "1.0" -tracing = "0.1" -tracing-futures = "0.2" tokio = { version = "1.21", features = ["sync", "macros", "rt-multi-thread"] } serde = "1.0" serde_json = "1.0" -reth-primitives = { path = "../../crates/primitives" } walkdir = "2.3" diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index e94b2330c..2643f7190 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -1,12 +1,23 @@ -use clap::{ArgAction, Parser, Subcommand}; +//! CLI definition and entrypoint to executable -use crate::test_eth_chain; +use clap::{ArgAction, Parser, Subcommand}; +use tracing_subscriber::util::SubscriberInitExt; + +use crate::{ + node, test_eth_chain, + util::reth_tracing::{self, TracingMode}, +}; /// main function that parses cli and runs command pub async fn run() -> eyre::Result<()> { let opt = Cli::parse(); + let tracing = if opt.silent { TracingMode::Silent } else { TracingMode::All }; + + reth_tracing::build_subscriber(tracing).init(); + match opt.command { + Commands::Node(command) => command.execute().await, Commands::TestEthChain(command) => command.execute().await, } } @@ -14,6 +25,9 @@ pub async fn run() -> eyre::Result<()> { /// Commands to be executed #[derive(Subcommand)] pub enum Commands { + /// Main node command + #[command(name = "node")] + Node(node::Command), /// Runs Ethereum blockchain tests #[command(name = "test-chain")] TestEthChain(test_eth_chain::Command), diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 085d1bb33..208bba77a 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -4,14 +4,9 @@ no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) ))] - //! Rust Ethereum (reth) binary executable. -/// CLI definition and entrypoint pub mod cli; - -/// Utility functions. -pub mod util; - -/// Command for executing Ethereum blockchain tests +pub mod node; pub mod test_eth_chain; +pub mod util; diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs new file mode 100644 index 000000000..72859a3d9 --- /dev/null +++ b/bin/reth/src/node/mod.rs @@ -0,0 +1,48 @@ +//! Main node command +//! +//! Starts the client + +use clap::Parser; +use std::{path::Path, sync::Arc}; +use tracing::info; + +/// Execute Ethereum blockchain tests by specifying path to json files +#[derive(Debug, Parser)] +pub struct Command { + /// Path to database folder + #[arg(long, default_value = "~/.reth/db")] + db: String, +} + +impl Command { + /// Execute `node` command + pub async fn execute(&self) -> eyre::Result<()> { + info!("Rust Ethereum client"); + + info!("Initialize components:"); + + let path = shellexpand::full(&self.db)?.into_owned(); + let expanded_db_path = Path::new(&path); + std::fs::create_dir_all(expanded_db_path)?; + let db = Arc::new(reth_db::kv::Env::::open( + expanded_db_path, + reth_db::kv::EnvKind::RW, + )?); + info!("DB opened"); + + // let _p2p = (); + // let _consensus = (); + // let _rpc = (); + + let mut pipeline = reth_stages::Pipeline::new(); + + // define all stages here + + // run pipeline + info!("Pipeline started:"); + pipeline.run(db.clone()).await?; + + info!("Finishing"); + Ok(()) + } +} diff --git a/bin/reth/src/test_eth_chain/mod.rs b/bin/reth/src/test_eth_chain/mod.rs index 0f70c211a..1ac81c32c 100644 --- a/bin/reth/src/test_eth_chain/mod.rs +++ b/bin/reth/src/test_eth_chain/mod.rs @@ -1,3 +1,5 @@ +//! Command for running Ethereum chain tests. + use crate::util; use clap::Parser; use std::path::PathBuf; @@ -21,7 +23,7 @@ impl Command { .path .iter() .map(|item| { - util::find_all_json_tests(item).into_iter().map(|file| { + util::find_all_files_with_postfix(item, ".json").into_iter().map(|file| { let tfile = file.clone(); let join = tokio::spawn(async move { runner::run_test(tfile.as_path()).await }); (join, file) diff --git a/bin/reth/src/util/mod.rs b/bin/reth/src/util/mod.rs index 231e29337..5cf285e1e 100644 --- a/bin/reth/src/util/mod.rs +++ b/bin/reth/src/util/mod.rs @@ -1,11 +1,52 @@ +//! Utility functions. + use std::path::{Path, PathBuf}; use walkdir::{DirEntry, WalkDir}; -pub(crate) fn find_all_json_tests(path: &Path) -> Vec { +pub(crate) fn find_all_files_with_postfix(path: &Path, postfix: &str) -> Vec { WalkDir::new(path) .into_iter() .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".json")) + .filter(|e| e.file_name().to_string_lossy().ends_with(postfix)) .map(DirEntry::into_path) .collect::>() } + +/// Tracing utility +pub mod reth_tracing { + use tracing::Subscriber; + use tracing_subscriber::{prelude::*, EnvFilter}; + + /// Tracing modes + pub enum TracingMode { + /// Enable all info traces. + All, + /// Disable tracing + Silent, + } + + impl TracingMode { + fn into_env_filter(self) -> EnvFilter { + match self { + Self::All => EnvFilter::new("reth=info"), + Self::Silent => EnvFilter::new(""), + } + } + } + + /// Build subscriber + pub fn build_subscriber(mods: TracingMode) -> impl Subscriber { + let nocolor = std::env::var("RUST_LOG_STYLE").map(|val| val == "never").unwrap_or(false); + + // Take env over config + let filter = if std::env::var(EnvFilter::DEFAULT_ENV).unwrap_or_default().is_empty() { + mods.into_env_filter() + } else { + EnvFilter::from_default_env() + }; + + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_ansi(!nocolor).with_target(false)) + .with(filter) + } +} diff --git a/crates/db/src/kv/mod.rs b/crates/db/src/kv/mod.rs index b97fd745f..68d3bcf6e 100644 --- a/crates/db/src/kv/mod.rs +++ b/crates/db/src/kv/mod.rs @@ -110,6 +110,7 @@ impl Deref for Env { #[cfg(any(test, feature = "test-utils"))] pub mod test_utils { use super::{Env, EnvKind, EnvironmentKind, Path}; + use std::sync::Arc; /// Error during database creation pub const ERROR_DB_CREATION: &str = "Not able to create the mdbx file."; @@ -119,8 +120,11 @@ pub mod test_utils { pub const ERROR_TEMPDIR: &str = "Not able to create a temporary directory."; /// Create database for testing - pub fn create_test_db(kind: EnvKind) -> Env { - create_test_db_with_path(kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path()) + pub fn create_test_db(kind: EnvKind) -> Arc> { + Arc::new(create_test_db_with_path( + kind, + &tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), + )) } /// Create database for testing with specified path @@ -141,7 +145,7 @@ mod tests { }; use reth_libmdbx::{NoWriteMap, WriteMap}; use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, H256, U256}; - use std::str::FromStr; + use std::{str::FromStr, sync::Arc}; use tempfile::TempDir; const ERROR_DB_CREATION: &str = "Not able to create the mdbx file."; @@ -282,7 +286,7 @@ mod tests { #[test] fn db_sharded_key() { - let db: Env = test_utils::create_test_db(EnvKind::RW); + let db: Arc> = test_utils::create_test_db(EnvKind::RW); let real_key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047").unwrap(); for i in 1..5 { diff --git a/crates/stages/src/pipeline.rs b/crates/stages/src/pipeline.rs index d193554b3..74914d73a 100644 --- a/crates/stages/src/pipeline.rs +++ b/crates/stages/src/pipeline.rs @@ -4,7 +4,10 @@ use crate::{ }; use reth_interfaces::db::{DBContainer, Database, DbTx}; use reth_primitives::BlockNumber; -use std::fmt::{Debug, Formatter}; +use std::{ + fmt::{Debug, Formatter}, + sync::Arc, +}; use tokio::sync::mpsc::Sender; use tracing::*; @@ -139,7 +142,7 @@ impl Pipeline { /// Run the pipeline in an infinite loop. Will terminate early if the user has specified /// a `max_block` in the pipeline. - pub async fn run(&mut self, db: &DB) -> Result<(), PipelineError> { + pub async fn run(&mut self, db: Arc) -> Result<(), PipelineError> { loop { let mut state = PipelineState { events_sender: self.events_sender.clone(), @@ -148,7 +151,7 @@ impl Pipeline { minimum_progress: None, reached_tip: true, }; - let next_action = self.run_loop(&mut state, db).await?; + let next_action = self.run_loop(&mut state, db.as_ref()).await?; // Terminate the loop early if it's reached the maximum user // configured block. @@ -396,7 +399,7 @@ mod tests { false, ) .set_max_block(Some(10)) - .run(&db) + .run(db) .await }); @@ -450,7 +453,7 @@ mod tests { .set_max_block(Some(10)); // Sync first - pipeline.run(&db).await.expect("Could not run pipeline"); + pipeline.run(db.clone()).await.expect("Could not run pipeline"); // Unwind pipeline.set_channel(tx).unwind(&db, 1, None).await.expect("Could not unwind pipeline"); @@ -528,7 +531,7 @@ mod tests { ) .set_max_block(Some(10)) .set_channel(tx) - .run(&db) + .run(db) .await .expect("Could not run pipeline"); }); @@ -620,7 +623,7 @@ mod tests { .set_max_block(Some(10)); // Sync first - pipeline.run(&db).await.expect("Could not run pipeline"); + pipeline.run(db.clone()).await.expect("Could not run pipeline"); // Unwind pipeline.set_channel(tx).unwind(&db, 1, None).await.expect("Could not unwind pipeline"); @@ -690,7 +693,7 @@ mod tests { true, ) .set_max_block(Some(10)) - .run(&db) + .run(db) .await }); diff --git a/crates/stages/src/util.rs b/crates/stages/src/util.rs index c205aabe3..66e0f8415 100644 --- a/crates/stages/src/util.rs +++ b/crates/stages/src/util.rs @@ -145,7 +145,7 @@ pub(crate) mod test_utils { impl Default for StageTestDB { /// Create a new instance of [StageTestDB] fn default() -> Self { - Self { db: Arc::new(create_test_db::(EnvKind::RW)) } + Self { db: create_test_db::(EnvKind::RW) } } }