diff --git a/Cargo.lock b/Cargo.lock index ec85c912c..99cfc38d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6324,6 +6324,7 @@ dependencies = [ "reth-node-ethereum", "reth-node-events", "reth-node-optimism", + "reth-optimism-cli", "reth-optimism-primitives", "reth-payload-builder", "reth-payload-primitives", @@ -6340,6 +6341,7 @@ dependencies = [ "reth-rpc-types", "reth-rpc-types-compat", "reth-stages", + "reth-stages-api", "reth-static-file", "reth-static-file-types", "reth-tasks", @@ -7821,6 +7823,34 @@ dependencies = [ [[package]] name = "reth-optimism-cli" version = "1.0.0" +dependencies = [ + "alloy-primitives", + "clap", + "eyre", + "futures-util", + "reth-cli-commands", + "reth-config", + "reth-consensus", + "reth-db", + "reth-db-api", + "reth-downloaders", + "reth-errors", + "reth-evm-optimism", + "reth-execution-types", + "reth-network-p2p", + "reth-node-core", + "reth-node-events", + "reth-optimism-primitives", + "reth-primitives", + "reth-provider", + "reth-prune", + "reth-stages", + "reth-stages-types", + "reth-static-file", + "reth-static-file-types", + "tokio", + "tracing", +] [[package]] name = "reth-optimism-consensus" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 44edb0d9d..6ea5c346a 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -67,6 +67,8 @@ reth-consensus.workspace = true reth-optimism-primitives.workspace = true reth-engine-util.workspace = true reth-prune.workspace = true +reth-stages-api.workspace = true +reth-optimism-cli = { workspace = true, optional = true } # crypto alloy-rlp.workspace = true @@ -135,6 +137,8 @@ min-debug-logs = ["tracing/release_max_level_debug"] min-trace-logs = ["tracing/release_max_level_trace"] optimism = [ + "dep:reth-optimism-cli", + "reth-optimism-cli?/optimism", "reth-primitives/optimism", "reth-rpc/optimism", "reth-provider/optimism", diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 2d253ea32..a7e8a759a 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -197,11 +197,11 @@ pub enum Commands { /// This syncs RLP encoded OP blocks below Bedrock from a file, without executing. #[cfg(feature = "optimism")] #[command(name = "import-op")] - ImportOp(crate::commands::import_op::ImportOpCommand), + ImportOp(reth_optimism_cli::ImportOpCommand), /// This imports RLP encoded receipts from a file. #[cfg(feature = "optimism")] #[command(name = "import-receipts-op")] - ImportReceiptsOp(crate::commands::import_receipts_op::ImportReceiptsOpCommand), + ImportReceiptsOp(reth_optimism_cli::ImportReceiptsOpCommand), /// Dumps genesis block JSON configuration to stdout. DumpGenesis(dump_genesis::DumpGenesisCommand), /// Database debugging utilities diff --git a/bin/reth/src/commands/mod.rs b/bin/reth/src/commands/mod.rs index 36290922a..a8f5ff3fd 100644 --- a/bin/reth/src/commands/mod.rs +++ b/bin/reth/src/commands/mod.rs @@ -4,8 +4,6 @@ pub mod config_cmd; pub mod debug_cmd; pub mod dump_genesis; pub mod import; -pub mod import_op; -pub mod import_receipts_op; pub mod init_cmd; pub mod init_state; pub mod node; diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 05201b658..b5cd12c33 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -8,3 +8,49 @@ homepage.workspace = true repository.workspace = true [lints] +workspace = true + +[dependencies] +reth-static-file-types = { workspace = true, features = ["clap"] } +clap = { workspace = true, features = ["derive", "env"] } +reth-cli-commands.workspace = true +reth-consensus.workspace = true +reth-db = { workspace = true, features = ["mdbx"] } +reth-db-api.workspace = true +reth-downloaders.workspace = true +reth-optimism-primitives.workspace = true +reth-provider.workspace = true +reth-prune.workspace = true +reth-stages.workspace = true +reth-static-file.workspace = true +reth-execution-types.workspace = true +reth-node-core.workspace = true +reth-primitives.workspace = true + + +reth-stages-types.workspace = true +reth-node-events.workspace = true +reth-network-p2p.workspace = true +reth-errors.workspace = true + +reth-config.workspace = true +alloy-primitives.workspace = true +futures-util.workspace = true +reth-evm-optimism.workspace = true + + + +tokio = { workspace = true, features = [ + "sync", + "macros", + "time", + "rt-multi-thread", +] } +tracing.workspace = true +eyre.workspace = true + +[features] + optimism = [ + "reth-primitives/optimism", + "reth-evm-optimism/optimism", + ] \ No newline at end of file diff --git a/crates/optimism/cli/src/commands/build_pipeline.rs b/crates/optimism/cli/src/commands/build_pipeline.rs new file mode 100644 index 000000000..29761d0f7 --- /dev/null +++ b/crates/optimism/cli/src/commands/build_pipeline.rs @@ -0,0 +1,96 @@ +use alloy_primitives::B256; +use futures_util::{Stream, StreamExt}; +use reth_config::Config; +use reth_consensus::Consensus; +use reth_db_api::database::Database; +use reth_downloaders::{ + bodies::bodies::BodiesDownloaderBuilder, file_client::FileClient, + headers::reverse_headers::ReverseHeadersDownloaderBuilder, +}; +use reth_errors::ProviderError; +use reth_evm_optimism::OpExecutorProvider; +use reth_network_p2p::{ + bodies::downloader::BodyDownloader, + headers::downloader::{HeaderDownloader, SyncTarget}, +}; +use reth_node_events::node::NodeEvent; +use reth_provider::{BlockNumReader, ChainSpecProvider, HeaderProvider, ProviderFactory}; +use reth_prune::PruneModes; +use reth_stages::{sets::DefaultStages, Pipeline, StageSet}; +use reth_stages_types::StageId; +use reth_static_file::StaticFileProducer; +use std::sync::Arc; +use tokio::sync::watch; + +/// Builds import pipeline. +/// +/// If configured to execute, all stages will run. Otherwise, only stages that don't require state +/// will run. +pub async fn build_import_pipeline( + config: &Config, + provider_factory: ProviderFactory, + consensus: &Arc, + file_client: Arc, + static_file_producer: StaticFileProducer, + disable_exec: bool, +) -> eyre::Result<(Pipeline, impl Stream)> +where + DB: Database + Clone + Unpin + 'static, + C: Consensus + 'static, +{ + if !file_client.has_canonical_blocks() { + eyre::bail!("unable to import non canonical blocks"); + } + + // Retrieve latest header found in the database. + let last_block_number = provider_factory.last_block_number()?; + let local_head = provider_factory + .sealed_header(last_block_number)? + .ok_or(ProviderError::HeaderNotFound(last_block_number.into()))?; + + let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers) + .build(file_client.clone(), consensus.clone()) + .into_task(); + // TODO: The pipeline should correctly configure the downloader on its own. + // Find the possibility to remove unnecessary pre-configuration. + header_downloader.update_local_head(local_head); + header_downloader.update_sync_target(SyncTarget::Tip(file_client.tip().unwrap())); + + let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies) + .build(file_client.clone(), consensus.clone(), provider_factory.clone()) + .into_task(); + // TODO: The pipeline should correctly configure the downloader on its own. + // Find the possibility to remove unnecessary pre-configuration. + body_downloader + .set_download_range(file_client.min_block().unwrap()..=file_client.max_block().unwrap()) + .expect("failed to set download range"); + + let (tip_tx, tip_rx) = watch::channel(B256::ZERO); + let executor = OpExecutorProvider::optimism(provider_factory.chain_spec()); + + let max_block = file_client.max_block().unwrap_or(0); + + let pipeline = Pipeline::builder() + .with_tip_sender(tip_tx) + // we want to sync all blocks the file client provides or 0 if empty + .with_max_block(max_block) + .add_stages( + DefaultStages::new( + provider_factory.clone(), + tip_rx, + consensus.clone(), + header_downloader, + body_downloader, + executor, + config.stages.clone(), + PruneModes::default(), + ) + .builder() + .disable_all_if(&StageId::STATE_REQUIRED, || disable_exec), + ) + .build(provider_factory, static_file_producer); + + let events = pipeline.events().map(Into::into); + + Ok((pipeline, events)) +} diff --git a/bin/reth/src/commands/import_op.rs b/crates/optimism/cli/src/commands/import.rs similarity index 98% rename from bin/reth/src/commands/import_op.rs rename to crates/optimism/cli/src/commands/import.rs index 3d308ba0d..d28ec658a 100644 --- a/bin/reth/src/commands/import_op.rs +++ b/crates/optimism/cli/src/commands/import.rs @@ -1,6 +1,5 @@ //! Command that initializes the node by importing OP Mainnet chain segment below Bedrock, from a //! file. -use crate::{commands::import::build_import_pipeline, version::SHORT_VERSION}; use clap::Parser; use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; use reth_consensus::noop::NoopConsensus; @@ -9,6 +8,7 @@ use reth_db_api::transaction::DbTx; use reth_downloaders::file_client::{ ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE, }; +use reth_node_core::version::SHORT_VERSION; use reth_optimism_primitives::bedrock_import::is_dup_tx; use reth_provider::StageCheckpointReader; use reth_prune::PruneModes; @@ -17,6 +17,8 @@ use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; use tracing::{debug, error, info}; +use crate::commands::build_pipeline::build_import_pipeline; + /// Syncs RLP encoded blocks from a file. #[derive(Debug, Parser)] pub struct ImportOpCommand { diff --git a/bin/reth/src/commands/import_receipts_op.rs b/crates/optimism/cli/src/commands/import_receipts.rs similarity index 100% rename from bin/reth/src/commands/import_receipts_op.rs rename to crates/optimism/cli/src/commands/import_receipts.rs diff --git a/crates/optimism/cli/src/commands/mod.rs b/crates/optimism/cli/src/commands/mod.rs new file mode 100644 index 000000000..373e7802c --- /dev/null +++ b/crates/optimism/cli/src/commands/mod.rs @@ -0,0 +1,4 @@ +/// Helper function to build an import pipeline. +pub mod build_pipeline; +pub mod import; +pub mod import_receipts; diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index a5133bea1..67d0ccd61 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -5,5 +5,11 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(all(not(test), feature = "optimism"), warn(unused_crate_dependencies))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +// The `optimism` feature must be enabled to use this crate. +#![cfg(feature = "optimism")] + +/// Optimism CLI commands. +pub mod commands; +pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCommand};