diff --git a/Cargo.lock b/Cargo.lock index 46a14684c..bd3da5f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6423,6 +6423,7 @@ dependencies = [ "reth-node-ethereum", "reth-node-events", "reth-node-optimism", + "reth-optimism-primitives", "reth-payload-builder", "reth-payload-validator", "reth-primitives", @@ -6645,6 +6646,7 @@ dependencies = [ "mockall", "reth-consensus", "reth-interfaces", + "reth-optimism-primitives", "reth-primitives", "reth-provider", ] @@ -7593,6 +7595,10 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-optimism-primitives" +version = "0.2.0-beta.7" + [[package]] name = "reth-payload-builder" version = "0.2.0-beta.7" diff --git a/Cargo.toml b/Cargo.toml index 4726b63e0..946682343 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "crates/optimism/evm/", "crates/optimism/node/", "crates/optimism/payload/", + "crates/optimism/primitives/", "crates/payload/basic/", "crates/payload/builder/", "crates/payload/ethereum/", @@ -273,6 +274,7 @@ reth-node-events = { path = "crates/node/events" } reth-node-optimism = { path = "crates/optimism/node" } reth-optimism-consensus = { path = "crates/optimism/consensus" } reth-optimism-payload-builder = { path = "crates/optimism/payload" } +reth-optimism-primitives = { path = "crates/optimism/primitives" } reth-payload-builder = { path = "crates/payload/builder" } reth-payload-validator = { path = "crates/payload/validator" } reth-primitives = { path = "crates/primitives" } diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index ab1e9927a..35c5c814a 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -58,6 +58,7 @@ reth-db-common.workspace = true reth-node-builder.workspace = true reth-node-events.workspace = true reth-consensus.workspace = true +reth-optimism-primitives.workspace = true # crypto alloy-rlp.workspace = true diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index deece5b62..4bf413acd 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -2,13 +2,15 @@ #[cfg(feature = "optimism")] use crate::commands::import_op; +#[cfg(feature = "optimism")] +use crate::commands::import_receipts_op; use crate::{ args::{ utils::{chain_help, genesis_value_parser, SUPPORTED_CHAINS}, LogArgs, }, commands::{ - config_cmd, db, debug_cmd, dump_genesis, import, import_receipts, init_cmd, init_state, + config_cmd, db, debug_cmd, dump_genesis, import, init_cmd, init_state, node::{self, NoArgs}, p2p, recover, stage, test_vectors, }, @@ -150,11 +152,12 @@ impl Cli { Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::InitState(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Import(command) => runner.run_blocking_until_ctrl_c(command.execute()), - Commands::ImportReceipts(command) => { - runner.run_blocking_until_ctrl_c(command.execute()) - } #[cfg(feature = "optimism")] Commands::ImportOp(command) => runner.run_blocking_until_ctrl_c(command.execute()), + #[cfg(feature = "optimism")] + Commands::ImportReceiptsOp(command) => { + runner.run_blocking_until_ctrl_c(command.execute()) + } Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Stage(command) => runner.run_command_until_exit(|ctx| command.execute(ctx)), @@ -191,13 +194,14 @@ pub enum Commands { /// This syncs RLP encoded blocks from a file. #[command(name = "import")] Import(import::ImportCommand), - /// This imports RLP encoded receipts from a file. - #[command(name = "import-receipts")] - ImportReceipts(import_receipts::ImportReceiptsCommand), /// This syncs RLP encoded OP blocks below Bedrock from a file, without executing. #[cfg(feature = "optimism")] #[command(name = "import-op")] ImportOp(import_op::ImportOpCommand), + /// This imports RLP encoded receipts from a file. + #[cfg(feature = "optimism")] + #[command(name = "import-receipts-op")] + ImportReceiptsOp(import_receipts_op::ImportReceiptsOpCommand), /// Dumps genesis block JSON configuration to stdout. DumpGenesis(dump_genesis::DumpGenesisCommand), /// Database debugging utilities diff --git a/bin/reth/src/commands/import.rs b/bin/reth/src/commands/import.rs index 70a2c339c..71268fa8e 100644 --- a/bin/reth/src/commands/import.rs +++ b/bin/reth/src/commands/import.rs @@ -71,7 +71,7 @@ pub struct ImportCommand { #[arg(long, verbatim_doc_comment)] no_state: bool, - /// Chunk byte length. + /// Chunk byte length to read from file. #[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)] chunk_len: Option, diff --git a/bin/reth/src/commands/import_op.rs b/bin/reth/src/commands/import_op.rs index a85fc4e3d..a1b23bda8 100644 --- a/bin/reth/src/commands/import_op.rs +++ b/bin/reth/src/commands/import_op.rs @@ -18,7 +18,8 @@ use reth_db_common::init::init_genesis; use reth_downloaders::file_client::{ ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE, }; -use reth_primitives::{op_mainnet::is_dup_tx, stage::StageId, PruneModes}; +use reth_optimism_primitives::bedrock_import::is_dup_tx; +use reth_primitives::{stage::StageId, PruneModes}; use reth_provider::{ProviderFactory, StageCheckpointReader, StaticFileProviderFactory}; use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; @@ -41,7 +42,7 @@ pub struct ImportOpCommand { #[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)] datadir: MaybePlatformPath, - /// Chunk byte length. + /// Chunk byte length to read from file. #[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)] chunk_len: Option, @@ -187,6 +188,9 @@ impl ImportOpCommand { info!(target: "reth::cli", total_imported_blocks, total_imported_txns, + total_decoded_blocks, + total_decoded_txns, + total_filtered_out_dup_txns, "Chain file imported" ); diff --git a/bin/reth/src/commands/import_receipts.rs b/bin/reth/src/commands/import_receipts.rs deleted file mode 100644 index 018ff132b..000000000 --- a/bin/reth/src/commands/import_receipts.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Command that imports receipts from a file. - -use crate::{ - args::{ - utils::{chain_help, genesis_value_parser, SUPPORTED_CHAINS}, - DatabaseArgs, - }, - dirs::{DataDirPath, MaybePlatformPath}, -}; -use clap::Parser; -use reth_db::{database::Database, init_db, transaction::DbTx, DatabaseEnv}; -use reth_downloaders::{ - file_client::{ChunkedFileReader, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE}, - receipt_file_client::ReceiptFileClient, -}; -use reth_node_core::version::SHORT_VERSION; -use reth_primitives::{stage::StageId, ChainSpec, StaticFileSegment}; -use reth_provider::{ - BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader, - StateWriter, StaticFileProviderFactory, StaticFileWriter, -}; -use tracing::{debug, error, info, trace}; - -use std::{path::PathBuf, sync::Arc}; - -/// Initializes the database with the genesis block. -#[derive(Debug, Parser)] -pub struct ImportReceiptsCommand { - /// The path to the data dir for all reth files and subdirectories. - /// - /// Defaults to the OS-specific data directory: - /// - /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - /// - Windows: `{FOLDERID_RoamingAppData}/reth/` - /// - macOS: `$HOME/Library/Application Support/reth/` - #[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)] - datadir: MaybePlatformPath, - - /// The chain this node is running. - /// - /// Possible values are either a built-in chain or the path to a chain specification file. - #[arg( - long, - value_name = "CHAIN_OR_PATH", - long_help = chain_help(), - default_value = SUPPORTED_CHAINS[0], - value_parser = genesis_value_parser - )] - chain: Arc, - - /// Chunk byte length. - #[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)] - chunk_len: Option, - - #[command(flatten)] - db: DatabaseArgs, - - /// The path to a receipts file for import. File must use `HackReceiptCodec` (used for - /// exporting OP chain segment below Bedrock block via testinprod/op-geth). - /// - /// - #[arg(value_name = "IMPORT_PATH", verbatim_doc_comment)] - path: PathBuf, -} - -impl ImportReceiptsCommand { - /// Execute `import` command - pub async fn execute(self) -> eyre::Result<()> { - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); - - debug!(target: "reth::cli", - chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE), - "Chunking receipts import" - ); - - // add network name to data dir - let data_dir = self.datadir.unwrap_or_chain_default(self.chain.chain); - - let db_path = data_dir.db(); - info!(target: "reth::cli", path = ?db_path, "Opening database"); - - let db = Arc::new(init_db(db_path, self.db.database_args())?); - info!(target: "reth::cli", "Database opened"); - let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files())?; - - let provider = provider_factory.provider_rw()?; - let static_file_provider = provider_factory.static_file_provider(); - - for stage in StageId::ALL { - let checkpoint = provider.get_stage_checkpoint(stage)?; - trace!(target: "reth::cli", - ?stage, - ?checkpoint, - "Read stage checkpoints from db" - ); - } - - // prepare the tx for `write_to_storage` - let tx = provider.into_tx(); - let mut total_decoded_receipts = 0; - - // open file - let mut reader = ChunkedFileReader::new(&self.path, self.chunk_len).await?; - - while let Some(file_client) = reader.next_chunk::().await? { - // create a new file client from chunk read from file - let ReceiptFileClient { receipts, first_block, total_receipts: total_receipts_chunk } = - file_client; - - // mark these as decoded - total_decoded_receipts += total_receipts_chunk; - - info!(target: "reth::cli", - first_receipts_block=?first_block, - total_receipts_chunk, - "Importing receipt file chunk" - ); - - // We're reusing receipt writing code internal to - // `BundleStateWithReceipts::write_to_storage`, so we just use a default empty - // `BundleState`. - let bundled_state = - BundleStateWithReceipts::new(Default::default(), receipts, first_block); - - let static_file_producer = - static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?; - - // finally, write the receipts - bundled_state.write_to_storage::<::TXMut>( - &tx, - Some(static_file_producer), - OriginalValuesKnown::Yes, - )?; - } - - tx.commit()?; - // as static files works in file ranges, internally it will be committing when creating the - // next file range already, so we only need to call explicitly at the end. - static_file_provider.commit()?; - - if total_decoded_receipts == 0 { - error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty"); - return Ok(()) - } - - // compare the highest static file block to the number of receipts we decoded - // - // `HeaderNumbers` and `TransactionHashNumbers` tables serve as additional indexes, but - // nothing like this needs to exist for Receipts. So `tx.entries::` would - // return zero here. - let total_imported_receipts = static_file_provider - .get_highest_static_file_block(StaticFileSegment::Receipts) - .expect("static files must exist after ensuring we decoded more than zero"); - - if total_imported_receipts != total_decoded_receipts as u64 { - error!(target: "reth::cli", - total_decoded_receipts, - total_imported_receipts, - "Receipts were partially imported" - ); - } - - info!(target: "reth::cli", total_imported_receipts, "Receipt file imported"); - - Ok(()) - } -} diff --git a/bin/reth/src/commands/import_receipts_op.rs b/bin/reth/src/commands/import_receipts_op.rs new file mode 100644 index 000000000..44c79cd2f --- /dev/null +++ b/bin/reth/src/commands/import_receipts_op.rs @@ -0,0 +1,230 @@ +//! Command that imports OP mainnet receipts from Bedrock datadir, exported via +//! . + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use clap::Parser; +use reth_db::{database::Database, init_db, tables, transaction::DbTx}; +use reth_downloaders::{ + file_client::{ChunkedFileReader, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE}, + receipt_file_client::ReceiptFileClient, +}; +use reth_node_core::version::SHORT_VERSION; +use reth_optimism_primitives::bedrock_import::is_dup_tx; +use reth_primitives::{stage::StageId, Receipts, StaticFileSegment}; +use reth_provider::{ + BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader, + StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader, +}; +use tracing::{debug, error, info, trace}; + +use crate::{ + args::{ + utils::{genesis_value_parser, SUPPORTED_CHAINS}, + DatabaseArgs, + }, + dirs::{DataDirPath, MaybePlatformPath}, +}; + +/// Initializes the database with the genesis block. +#[derive(Debug, Parser)] +pub struct ImportReceiptsOpCommand { + /// The path to the data dir for all reth files and subdirectories. + /// + /// Defaults to the OS-specific data directory: + /// + /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + /// - Windows: `{FOLDERID_RoamingAppData}/reth/` + /// - macOS: `$HOME/Library/Application Support/reth/` + #[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)] + datadir: MaybePlatformPath, + + /// Chunk byte length to read from file. + #[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)] + chunk_len: Option, + + #[command(flatten)] + db: DatabaseArgs, + + /// The path to a receipts file for import. File must use `HackReceiptFileCodec` (used for + /// exporting OP chain segment below Bedrock block via testinprod/op-geth). + /// + /// + #[arg(value_name = "IMPORT_PATH", verbatim_doc_comment)] + path: PathBuf, +} + +impl ImportReceiptsOpCommand { + /// Execute `import` command + pub async fn execute(self) -> eyre::Result<()> { + info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + + debug!(target: "reth::cli", + chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE), + "Chunking receipts import" + ); + + let chain_spec = genesis_value_parser(SUPPORTED_CHAINS[0])?; + + // add network name to data dir + let data_dir = self.datadir.unwrap_or_chain_default(chain_spec.chain); + + let db_path = data_dir.db(); + info!(target: "reth::cli", path = ?db_path, "Opening database"); + + let db = Arc::new(init_db(db_path, self.db.database_args())?); + info!(target: "reth::cli", "Database opened"); + let provider_factory = + ProviderFactory::new(db.clone(), chain_spec.clone(), data_dir.static_files())?; + + import_receipts_from_file( + provider_factory, + self.path, + self.chunk_len, + |first_block, receipts: &mut Receipts| { + let mut total_filtered_out_dup_txns = 0; + for (index, receipts_for_block) in receipts.iter_mut().enumerate() { + if is_dup_tx(first_block + index as u64) { + receipts_for_block.clear(); + total_filtered_out_dup_txns += 1; + } + } + + total_filtered_out_dup_txns + }, + ) + .await + } +} + +/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total +/// number of filtered out receipts. +/// +/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty +/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects +/// indices in the [`Receipts`] list, to map to sequential block numbers. +pub async fn import_receipts_from_file( + provider_factory: ProviderFactory, + path: P, + chunk_len: Option, + mut filter: F, +) -> eyre::Result<()> +where + DB: Database, + P: AsRef, + F: FnMut(u64, &mut Receipts) -> usize, +{ + let provider = provider_factory.provider_rw()?; + let static_file_provider = provider_factory.static_file_provider(); + + let total_imported_txns = static_file_provider + .count_entries::() + .expect("transaction static files must exist before importing receipts"); + let highest_block_transactions = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Transactions) + .expect("transaction static files must exist before importing receipts"); + + for stage in StageId::ALL { + let checkpoint = provider.get_stage_checkpoint(stage)?; + trace!(target: "reth::cli", + ?stage, + ?checkpoint, + "Read stage checkpoints from db" + ); + } + + // prepare the tx for `write_to_storage` + let tx = provider.into_tx(); + let mut total_decoded_receipts = 0; + let mut total_filtered_out_dup_txns = 0; + + // open file + let mut reader = ChunkedFileReader::new(path, chunk_len).await?; + + while let Some(file_client) = reader.next_chunk::().await? { + // create a new file client from chunk read from file + let ReceiptFileClient { mut receipts, first_block, total_receipts: total_receipts_chunk } = + file_client; + + // mark these as decoded + total_decoded_receipts += total_receipts_chunk; + + total_filtered_out_dup_txns += filter(first_block, &mut receipts); + + info!(target: "reth::cli", + first_receipts_block=?first_block, + total_receipts_chunk, + "Importing receipt file chunk" + ); + + // We're reusing receipt writing code internal to + // `BundleStateWithReceipts::write_to_storage`, so we just use a default empty + // `BundleState`. + let bundled_state = BundleStateWithReceipts::new(Default::default(), receipts, first_block); + + let static_file_producer = + static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?; + + // finally, write the receipts + bundled_state.write_to_storage::( + &tx, + Some(static_file_producer), + OriginalValuesKnown::Yes, + )?; + } + + tx.commit()?; + // as static files works in file ranges, internally it will be committing when creating the + // next file range already, so we only need to call explicitly at the end. + static_file_provider.commit()?; + + if total_decoded_receipts == 0 { + error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty"); + return Ok(()) + } + + let total_imported_receipts = static_file_provider + .count_entries::() + .expect("static files must exist after ensuring we decoded more than zero"); + + if total_imported_receipts + total_filtered_out_dup_txns != total_decoded_receipts { + error!(target: "reth::cli", + total_decoded_receipts, + total_imported_receipts, + total_filtered_out_dup_txns, + "Receipts were partially imported" + ); + } + + if total_imported_receipts != total_imported_txns { + error!(target: "reth::cli", + total_imported_receipts, + total_imported_txns, + "Receipts inconsistent with transactions" + ); + } + + let highest_block_receipts = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Receipts) + .expect("static files must exist after ensuring we decoded more than zero"); + + if highest_block_receipts != highest_block_transactions { + error!(target: "reth::cli", + highest_block_receipts, + highest_block_transactions, + "Height of receipts inconsistent with transactions" + ); + } + + info!(target: "reth::cli", + total_imported_receipts, + total_decoded_receipts, + total_filtered_out_dup_txns, + "Receipt file imported" + ); + + Ok(()) +} diff --git a/bin/reth/src/commands/mod.rs b/bin/reth/src/commands/mod.rs index 9e6ff8f84..789724e0d 100644 --- a/bin/reth/src/commands/mod.rs +++ b/bin/reth/src/commands/mod.rs @@ -6,7 +6,7 @@ pub mod debug_cmd; pub mod dump_genesis; pub mod import; pub mod import_op; -pub mod import_receipts; +pub mod import_receipts_op; pub mod init_cmd; pub mod init_state; diff --git a/book/SUMMARY.md b/book/SUMMARY.md index eaa7210cf..fc6deb282 100644 --- a/book/SUMMARY.md +++ b/book/SUMMARY.md @@ -32,7 +32,6 @@ - [`reth init`](./cli/reth/init.md) - [`reth init-state`](./cli/reth/init-state.md) - [`reth import`](./cli/reth/import.md) - - [`reth import-receipts`](./cli/reth/import-receipts.md) - [`reth dump-genesis`](./cli/reth/dump-genesis.md) - [`reth db`](./cli/reth/db.md) - [`reth db stats`](./cli/reth/db/stats.md) diff --git a/book/cli/SUMMARY.md b/book/cli/SUMMARY.md index 8c8ea2f42..ee3d714b2 100644 --- a/book/cli/SUMMARY.md +++ b/book/cli/SUMMARY.md @@ -3,7 +3,6 @@ - [`reth init`](./reth/init.md) - [`reth init-state`](./reth/init-state.md) - [`reth import`](./reth/import.md) - - [`reth import-receipts`](./reth/import-receipts.md) - [`reth dump-genesis`](./reth/dump-genesis.md) - [`reth db`](./reth/db.md) - [`reth db stats`](./reth/db/stats.md) diff --git a/book/cli/op-reth.md b/book/cli/op-reth.md new file mode 100644 index 000000000..2b56fa662 --- /dev/null +++ b/book/cli/op-reth.md @@ -0,0 +1,96 @@ +# op-reth + +Additional op-reth commands. + +```bash +$ op-reth --help +Usage: op-reth [OPTIONS] + +Commands: + import-op Imports the Bedrock datadir blocks from a file + import-receipts-op Imports the Bedrock datadir receipts from a file + +Options: + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/book/cli/reth.md b/book/cli/reth.md index 2e3ebef31..ae1728171 100644 --- a/book/cli/reth.md +++ b/book/cli/reth.md @@ -7,20 +7,19 @@ $ reth --help Usage: reth [OPTIONS] Commands: - node Start the node - init Initialize the database from a genesis file - init-state Initialize the database from a state dump file - import This syncs RLP encoded blocks from a file - import-receipts This imports RLP encoded receipts from a file - dump-genesis Dumps genesis block JSON configuration to stdout - db Database debugging utilities - stage Manipulate individual stages - p2p P2P Debugging utilities - test-vectors Generate Test Vectors - config Write config to stdout - debug Various debug routines - recover Scripts for node recovery - help Print this message or the help of the given subcommand(s) + node Start the node + init Initialize the database from a genesis file + init-state Initialize the database from a state dump file + import This syncs RLP encoded blocks from a file + dump-genesis Dumps genesis block JSON configuration to stdout + db Database debugging utilities + stage Manipulate individual stages + p2p P2P Debugging utilities + test-vectors Generate Test Vectors + config Write config to stdout + debug Various debug routines + recover Scripts for node recovery + help Print this message or the help of the given subcommand(s) Options: --chain diff --git a/book/cli/reth/import-op.md b/book/cli/reth/import-op.md new file mode 100644 index 000000000..d2d81980c --- /dev/null +++ b/book/cli/reth/import-op.md @@ -0,0 +1,134 @@ +# op-reth import + +This syncs RLP encoded blocks from a file. Supports import of OVM blocks +from the Bedrock datadir. Requires blocks, up to same height as receipts +file, to already be imported. + +```bash +$ op-reth import-op --help +Usage: op-reth import-op [OPTIONS] + +Options: + --config + The path to the configuration file to use. + + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --chunk-len + Chunk byte length to read from file. + + [default: 1GB] + + -h, --help + Print help (see a summary with '-h') + +Database: + --db.log-level + Database logging level. Levels higher than "notice" require a debug build + + Possible values: + - fatal: Enables logging for critical conditions, i.e. assertion failures + - error: Enables logging for error conditions + - warn: Enables logging for warning conditions + - notice: Enables logging for normal but significant condition + - verbose: Enables logging for verbose informational + - debug: Enables logging for debug-level messages + - trace: Enables logging for trace debug-level messages + - extra: Enables logging for extra debug-level messages + + --db.exclusive + Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume + + [possible values: true, false] + + + The path to a `.rlp` block file for import. + + The online sync pipeline stages (headers and bodies) are replaced by a file import. Skips block execution since blocks below Bedrock are built on OVM. + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/book/cli/reth/import-receipts.md b/book/cli/reth/import-receipts-op.md similarity index 78% rename from book/cli/reth/import-receipts.md rename to book/cli/reth/import-receipts-op.md index 7cea21d79..a5e1863ce 100644 --- a/book/cli/reth/import-receipts.md +++ b/book/cli/reth/import-receipts-op.md @@ -1,10 +1,13 @@ -# reth import-receipts +# op-reth import-receipts-op -This imports RLP encoded receipts from a file +This imports non-standard RLP encoded receipts from a file. +The supported RLP encoding, is the non-standard encoding used +for receipt export in . +Supports import of OVM receipts from the Bedrock datadir. ```bash -$ reth import-receipts --help -Usage: reth import-receipts [OPTIONS] +$ op-reth import-receipts-op --help +Usage: op-reth import-receipts-op [OPTIONS] Options: --datadir @@ -18,28 +21,10 @@ Options: [default: default] - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, goerli, holesky, dev - - [default: mainnet] - --chunk-len - Chunk byte length. + Chunk byte length to read from file. - --instance - Add a new instance of a node. - - Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. - - Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. - - Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2 - - [default: 1] + [default: 1GB] -h, --help Print help (see a summary with '-h') @@ -64,7 +49,7 @@ Database: [possible values: true, false] - The path to a receipts file for import. File must use `HackReceiptCodec` (used for + The path to a receipts file for import. File must use `HackReceiptFileCodec` (used for exporting OP chain segment below Bedrock block via testinprod/op-geth). diff --git a/book/cli/reth/import.md b/book/cli/reth/import.md index 8493a88f2..9c320d0b6 100644 --- a/book/cli/reth/import.md +++ b/book/cli/reth/import.md @@ -34,7 +34,7 @@ Options: Disables stages that require state. --chunk-len - Chunk byte length. + Chunk byte length to read from file. --instance Add a new instance of a node. diff --git a/book/run/sync-op-mainnet.md b/book/run/sync-op-mainnet.md index b50a32fb4..8c54aa877 100644 --- a/book/run/sync-op-mainnet.md +++ b/book/run/sync-op-mainnet.md @@ -22,7 +22,7 @@ Output from running the command to export state, can also be downloaded from ). +Import of >100 million OVM blocks, from genesis to Bedrock, completes in 6 hours. ```bash ./op-reth import-op @@ -36,14 +36,18 @@ corresponding transactions must already be imported (see [step 1](#1-import-bloc Imports a `.rlp` file of receipts, that has been exported with command specified in (command for exporting receipts uses custom RLP-encoding). +Import of >100 million OVM receipts, from genesis to Bedrock, completes in 30 minutes. + ```bash -./op-reth import-receipts --chain optimism +./op-reth import-receipts-op ``` ### 3. Import State Imports a `.jsonl` state dump. The block at which the state dump is made, must be the latest block in -reth's database. +reth's database. This should be block 105 235 063, the first Bedrock block (see [step 1](#1-import-blocks)). + +Import of >4 million OP mainnet accounts at Bedrock, completes in 10 minutes. ```bash ./op-reth init-state --chain optimism diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index accf2d08e..b18b4dcf6 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth reth-primitives.workspace = true +reth-optimism-primitives.workspace = true reth-consensus.workspace=true [dev-dependencies] diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index ad1e11643..74c515c20 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -1,12 +1,12 @@ //! Collection of methods for block validation. use reth_consensus::ConsensusError; +use reth_optimism_primitives::bedrock_import::is_dup_tx; use reth_primitives::{ constants::{ eip4844::{DATA_GAS_PER_BLOB, MAX_DATA_GAS_PER_BLOCK}, MAXIMUM_EXTRA_DATA_SIZE, }, - op_mainnet::is_dup_tx, ChainSpec, GotExpected, Hardfork, Header, SealedBlock, SealedHeader, }; diff --git a/crates/net/downloaders/src/receipt_file_client.rs b/crates/net/downloaders/src/receipt_file_client.rs index 0eaa4ff1b..b7231889c 100644 --- a/crates/net/downloaders/src/receipt_file_client.rs +++ b/crates/net/downloaders/src/receipt_file_client.rs @@ -75,10 +75,10 @@ impl FromReader for ReceiptFileClient { Err(err) => return Err(err), }; - total_receipts += 1; - match receipt { Some(ReceiptWithBlockNumber { receipt, number }) => { + total_receipts += 1; + if first_block.is_none() { first_block = Some(number); block_number = number; @@ -202,7 +202,8 @@ mod test { let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) = ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap(); - assert_eq!(4, total_receipts); + // 2 non-empty receipt objects + assert_eq!(2, total_receipts); assert_eq!(0, first_block); assert!(receipts[0].is_empty()); assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap()); @@ -229,7 +230,8 @@ mod test { let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) = ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap(); - assert_eq!(4, total_receipts); + // 2 non-empty receipt objects + assert_eq!(2, total_receipts); assert_eq!(0, first_block); assert!(receipts[0].is_empty()); assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap()); @@ -257,7 +259,8 @@ mod test { let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) = ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap(); - assert_eq!(5, total_receipts); + // 4 non-empty receipt objects + assert_eq!(4, total_receipts); assert_eq!(0, first_block); assert!(receipts[0].is_empty()); assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap()); diff --git a/crates/optimism/primitives/Cargo.toml b/crates/optimism/primitives/Cargo.toml new file mode 100644 index 000000000..0acd2f1eb --- /dev/null +++ b/crates/optimism/primitives/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "reth-optimism-primitives" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "OP primitive types" + +[lints] +workspace = true \ No newline at end of file diff --git a/crates/primitives/src/op_mainnet.rs b/crates/optimism/primitives/src/bedrock_import.rs similarity index 94% rename from crates/primitives/src/op_mainnet.rs rename to crates/optimism/primitives/src/bedrock_import.rs index c60504e92..17020f9f2 100644 --- a/crates/primitives/src/op_mainnet.rs +++ b/crates/optimism/primitives/src/bedrock_import.rs @@ -1,4 +1,4 @@ -//! Helpers for working with replayed OP mainnet OVM transactions (in blocks below Bedrock). +//! Replayed OP mainnet OVM transactions (in blocks below Bedrock). /// Transaction 0x9ed8f713b2cc6439657db52dcd2fdb9cc944915428f3c6e2a7703e242b259cb9 in block 985, /// replayed in blocks: diff --git a/crates/optimism/primitives/src/lib.rs b/crates/optimism/primitives/src/lib.rs new file mode 100644 index 000000000..5cdb53def --- /dev/null +++ b/crates/optimism/primitives/src/lib.rs @@ -0,0 +1,10 @@ +//! Standalone crate for Optimism-specific Reth primitive types. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] + +pub mod bedrock_import; diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 7681ddb9b..d3ea34037 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -35,7 +35,6 @@ mod header; mod integer_list; mod log; mod net; -pub mod op_mainnet; pub mod proofs; mod prune; mod receipt;