feat: moved optimism commands to create and remove from bin (#9242)

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Cargo.lock

@@ -6324,6 +6324,7 @@ dependencies = [
  "reth-node-ethereum",
  "reth-node-events",
  "reth-node-optimism",
+ "reth-optimism-cli",
  "reth-optimism-primitives",
  "reth-payload-builder",
  "reth-payload-primitives",
@@ -6340,6 +6341,7 @@ dependencies = [
  "reth-rpc-types",
  "reth-rpc-types-compat",
  "reth-stages",
+ "reth-stages-api",
  "reth-static-file",
  "reth-static-file-types",
  "reth-tasks",
@@ -7821,6 +7823,34 @@ dependencies = [
 [[package]]
 name = "reth-optimism-cli"
 version = "1.0.0"
+dependencies = [
+ "alloy-primitives",
+ "clap",
+ "eyre",
+ "futures-util",
+ "reth-cli-commands",
+ "reth-config",
+ "reth-consensus",
+ "reth-db",
+ "reth-db-api",
+ "reth-downloaders",
+ "reth-errors",
+ "reth-evm-optimism",
+ "reth-execution-types",
+ "reth-network-p2p",
+ "reth-node-core",
+ "reth-node-events",
+ "reth-optimism-primitives",
+ "reth-primitives",
+ "reth-provider",
+ "reth-prune",
+ "reth-stages",
+ "reth-stages-types",
+ "reth-static-file",
+ "reth-static-file-types",
+ "tokio",
+ "tracing",
+]
 
 [[package]]
 name = "reth-optimism-consensus"

bin/reth/Cargo.toml

@@ -67,6 +67,8 @@ reth-consensus.workspace = true
 reth-optimism-primitives.workspace = true
 reth-engine-util.workspace = true
 reth-prune.workspace = true
+reth-stages-api.workspace = true
+reth-optimism-cli = { workspace = true, optional = true }
 
 # crypto
 alloy-rlp.workspace = true
@@ -135,6 +137,8 @@ min-debug-logs = ["tracing/release_max_level_debug"]
 min-trace-logs = ["tracing/release_max_level_trace"]
 
 optimism = [
+    "dep:reth-optimism-cli",
+    "reth-optimism-cli?/optimism",
     "reth-primitives/optimism",
     "reth-rpc/optimism",
     "reth-provider/optimism",

bin/reth/src/cli/mod.rs

@@ -197,11 +197,11 @@ pub enum Commands<Ext: clap::Args + fmt::Debug = NoArgs> {
     /// This syncs RLP encoded OP blocks below Bedrock from a file, without executing.
     #[cfg(feature = "optimism")]
     #[command(name = "import-op")]
-    ImportOp(crate::commands::import_op::ImportOpCommand),
+    ImportOp(reth_optimism_cli::ImportOpCommand),
     /// This imports RLP encoded receipts from a file.
     #[cfg(feature = "optimism")]
     #[command(name = "import-receipts-op")]
-    ImportReceiptsOp(crate::commands::import_receipts_op::ImportReceiptsOpCommand),
+    ImportReceiptsOp(reth_optimism_cli::ImportReceiptsOpCommand),
     /// Dumps genesis block JSON configuration to stdout.
     DumpGenesis(dump_genesis::DumpGenesisCommand),
     /// Database debugging utilities

bin/reth/src/commands/mod.rs

@@ -4,8 +4,6 @@ pub mod config_cmd;
 pub mod debug_cmd;
 pub mod dump_genesis;
 pub mod import;
-pub mod import_op;
-pub mod import_receipts_op;
 pub mod init_cmd;
 pub mod init_state;
 pub mod node;

crates/optimism/cli/Cargo.toml

@@ -8,3 +8,49 @@ homepage.workspace = true
 repository.workspace = true
 
 [lints]
+workspace = true
+
+[dependencies]
+reth-static-file-types = { workspace = true, features = ["clap"] }
+clap = { workspace = true, features = ["derive", "env"] }
+reth-cli-commands.workspace = true
+reth-consensus.workspace = true
+reth-db = { workspace = true, features = ["mdbx"] }
+reth-db-api.workspace = true
+reth-downloaders.workspace = true
+reth-optimism-primitives.workspace = true
+reth-provider.workspace = true
+reth-prune.workspace = true
+reth-stages.workspace = true
+reth-static-file.workspace = true
+reth-execution-types.workspace = true
+reth-node-core.workspace = true
+reth-primitives.workspace = true
+
+
+reth-stages-types.workspace = true
+reth-node-events.workspace = true
+reth-network-p2p.workspace = true
+reth-errors.workspace = true
+
+reth-config.workspace = true
+alloy-primitives.workspace = true
+futures-util.workspace = true
+reth-evm-optimism.workspace = true
+
+
+
+tokio = { workspace = true, features = [
+    "sync",
+    "macros",
+    "time",
+    "rt-multi-thread",
+] }
+tracing.workspace = true
+eyre.workspace = true
+
+[features] 
+ optimism = [ 
+    "reth-primitives/optimism",
+    "reth-evm-optimism/optimism",
+ ] 

crates/optimism/cli/src/commands/build_pipeline.rs

@@ -0,0 +1,96 @@
+use alloy_primitives::B256;
+use futures_util::{Stream, StreamExt};
+use reth_config::Config;
+use reth_consensus::Consensus;
+use reth_db_api::database::Database;
+use reth_downloaders::{
+    bodies::bodies::BodiesDownloaderBuilder, file_client::FileClient,
+    headers::reverse_headers::ReverseHeadersDownloaderBuilder,
+};
+use reth_errors::ProviderError;
+use reth_evm_optimism::OpExecutorProvider;
+use reth_network_p2p::{
+    bodies::downloader::BodyDownloader,
+    headers::downloader::{HeaderDownloader, SyncTarget},
+};
+use reth_node_events::node::NodeEvent;
+use reth_provider::{BlockNumReader, ChainSpecProvider, HeaderProvider, ProviderFactory};
+use reth_prune::PruneModes;
+use reth_stages::{sets::DefaultStages, Pipeline, StageSet};
+use reth_stages_types::StageId;
+use reth_static_file::StaticFileProducer;
+use std::sync::Arc;
+use tokio::sync::watch;
+
+/// Builds import pipeline.
+///
+/// If configured to execute, all stages will run. Otherwise, only stages that don't require state
+/// will run.
+pub async fn build_import_pipeline<DB, C>(
+    config: &Config,
+    provider_factory: ProviderFactory<DB>,
+    consensus: &Arc<C>,
+    file_client: Arc<FileClient>,
+    static_file_producer: StaticFileProducer<DB>,
+    disable_exec: bool,
+) -> eyre::Result<(Pipeline<DB>, impl Stream<Item = NodeEvent>)>
+where
+    DB: Database + Clone + Unpin + 'static,
+    C: Consensus + 'static,
+{
+    if !file_client.has_canonical_blocks() {
+        eyre::bail!("unable to import non canonical blocks");
+    }
+
+    // Retrieve latest header found in the database.
+    let last_block_number = provider_factory.last_block_number()?;
+    let local_head = provider_factory
+        .sealed_header(last_block_number)?
+        .ok_or(ProviderError::HeaderNotFound(last_block_number.into()))?;
+
+    let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers)
+        .build(file_client.clone(), consensus.clone())
+        .into_task();
+    // TODO: The pipeline should correctly configure the downloader on its own.
+    // Find the possibility to remove unnecessary pre-configuration.
+    header_downloader.update_local_head(local_head);
+    header_downloader.update_sync_target(SyncTarget::Tip(file_client.tip().unwrap()));
+
+    let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies)
+        .build(file_client.clone(), consensus.clone(), provider_factory.clone())
+        .into_task();
+    // TODO: The pipeline should correctly configure the downloader on its own.
+    // Find the possibility to remove unnecessary pre-configuration.
+    body_downloader
+        .set_download_range(file_client.min_block().unwrap()..=file_client.max_block().unwrap())
+        .expect("failed to set download range");
+
+    let (tip_tx, tip_rx) = watch::channel(B256::ZERO);
+    let executor = OpExecutorProvider::optimism(provider_factory.chain_spec());
+
+    let max_block = file_client.max_block().unwrap_or(0);
+
+    let pipeline = Pipeline::builder()
+        .with_tip_sender(tip_tx)
+        // we want to sync all blocks the file client provides or 0 if empty
+        .with_max_block(max_block)
+        .add_stages(
+            DefaultStages::new(
+                provider_factory.clone(),
+                tip_rx,
+                consensus.clone(),
+                header_downloader,
+                body_downloader,
+                executor,
+                config.stages.clone(),
+                PruneModes::default(),
+            )
+            .builder()
+            .disable_all_if(&StageId::STATE_REQUIRED, || disable_exec),
+        )
+        .build(provider_factory, static_file_producer);
+
+    let events = pipeline.events().map(Into::into);
+
+    Ok((pipeline, events))
+}

crates/optimism/cli/src/commands/import.rs

@@ -1,6 +1,5 @@
 //! Command that initializes the node by importing OP Mainnet chain segment below Bedrock, from a
 //! file.
-use crate::{commands::import::build_import_pipeline, version::SHORT_VERSION};
 use clap::Parser;
 use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs};
 use reth_consensus::noop::NoopConsensus;
@@ -9,6 +8,7 @@ use reth_db_api::transaction::DbTx;
 use reth_downloaders::file_client::{
     ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE,
 };
+use reth_node_core::version::SHORT_VERSION;
 use reth_optimism_primitives::bedrock_import::is_dup_tx;
 use reth_provider::StageCheckpointReader;
 use reth_prune::PruneModes;
@@ -17,6 +17,8 @@ use reth_static_file::StaticFileProducer;
 use std::{path::PathBuf, sync::Arc};
 use tracing::{debug, error, info};
 
+use crate::commands::build_pipeline::build_import_pipeline;
+
 /// Syncs RLP encoded blocks from a file.
 #[derive(Debug, Parser)]
 pub struct ImportOpCommand {

crates/optimism/cli/src/commands/mod.rs

@@ -0,0 +1,4 @@
+/// Helper function to build an import pipeline.
+pub mod build_pipeline;
+pub mod import;
+pub mod import_receipts;

crates/optimism/cli/src/lib.rs

@@ -5,5 +5,11 @@
     html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
     issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
 )]
-#![cfg_attr(not(test), warn(unused_crate_dependencies))]
+#![cfg_attr(all(not(test), feature = "optimism"), warn(unused_crate_dependencies))]
 #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
+// The `optimism` feature must be enabled to use this crate.
+#![cfg(feature = "optimism")]
+
+/// Optimism CLI commands.
+pub mod commands;
+pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCommand};