jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(op): filter out receipts for dup txns (#8400)

Browse Source
This commit is contained in:
Emilia Hane
2024-05-28 18:53:43 +02:00
committed by GitHub
parent d78c5b71fd
commit 3e7f1377ed
24 changed files with 552 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -6423,6 +6423,7 @@ dependencies = [
"reth-node-ethereum",
"reth-node-events",
"reth-node-optimism",
"reth-optimism-primitives",
"reth-payload-builder",
"reth-payload-validator",
"reth-primitives",
@ -6645,6 +6646,7 @@ dependencies = [
"mockall",
"reth-consensus",
"reth-interfaces",
"reth-optimism-primitives",
"reth-primitives",
"reth-provider",
]
@ -7593,6 +7595,10 @@ dependencies = [
"tracing",
]
[[package]]
name = "reth-optimism-primitives"
version = "0.2.0-beta.7"
[[package]]
name = "reth-payload-builder"
version = "0.2.0-beta.7"

2
Cargo.toml
View File

@ -46,6 +46,7 @@ members = [
"crates/optimism/evm/",
"crates/optimism/node/",
"crates/optimism/payload/",
"crates/optimism/primitives/",
"crates/payload/basic/",
"crates/payload/builder/",
"crates/payload/ethereum/",
@ -273,6 +274,7 @@ reth-node-events = { path = "crates/node/events" }
reth-node-optimism = { path = "crates/optimism/node" }
reth-optimism-consensus = { path = "crates/optimism/consensus" }
reth-optimism-payload-builder = { path = "crates/optimism/payload" }
reth-optimism-primitives = { path = "crates/optimism/primitives" }
reth-payload-builder = { path = "crates/payload/builder" }
reth-payload-validator = { path = "crates/payload/validator" }
reth-primitives = { path = "crates/primitives" }

1
bin/reth/Cargo.toml
View File

@ -58,6 +58,7 @@ reth-db-common.workspace = true
reth-node-builder.workspace = true
reth-node-events.workspace = true
reth-consensus.workspace = true
reth-optimism-primitives.workspace = true
# crypto
alloy-rlp.workspace = true

18
bin/reth/src/cli/mod.rs
View File

@ -2,13 +2,15 @@
#[cfg(feature = "optimism")]
use crate::commands::import_op;
#[cfg(feature = "optimism")]
use crate::commands::import_receipts_op;
use crate::{
args::{
utils::{chain_help, genesis_value_parser, SUPPORTED_CHAINS},
LogArgs,
},
commands::{
config_cmd, db, debug_cmd, dump_genesis, import, import_receipts, init_cmd, init_state,
config_cmd, db, debug_cmd, dump_genesis, import, init_cmd, init_state,
node::{self, NoArgs},
p2p, recover, stage, test_vectors,
},
@ -150,11 +152,12 @@ impl<Ext: clap::Args + fmt::Debug> Cli<Ext> {
Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute()),
Commands::InitState(command) => runner.run_blocking_until_ctrl_c(command.execute()),
Commands::Import(command) => runner.run_blocking_until_ctrl_c(command.execute()),
Commands::ImportReceipts(command) => {
runner.run_blocking_until_ctrl_c(command.execute())
}
#[cfg(feature = "optimism")]
Commands::ImportOp(command) => runner.run_blocking_until_ctrl_c(command.execute()),
#[cfg(feature = "optimism")]
Commands::ImportReceiptsOp(command) => {
runner.run_blocking_until_ctrl_c(command.execute())
}
Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()),
Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute()),
Commands::Stage(command) => runner.run_command_until_exit(|ctx| command.execute(ctx)),
@ -191,13 +194,14 @@ pub enum Commands<Ext: clap::Args + fmt::Debug = NoArgs> {
/// This syncs RLP encoded blocks from a file.
#[command(name = "import")]
Import(import::ImportCommand),
/// This imports RLP encoded receipts from a file.
#[command(name = "import-receipts")]
ImportReceipts(import_receipts::ImportReceiptsCommand),
/// This syncs RLP encoded OP blocks below Bedrock from a file, without executing.
#[cfg(feature = "optimism")]
#[command(name = "import-op")]
ImportOp(import_op::ImportOpCommand),
/// This imports RLP encoded receipts from a file.
#[cfg(feature = "optimism")]
#[command(name = "import-receipts-op")]
ImportReceiptsOp(import_receipts_op::ImportReceiptsOpCommand),
/// Dumps genesis block JSON configuration to stdout.
DumpGenesis(dump_genesis::DumpGenesisCommand),
/// Database debugging utilities

2
bin/reth/src/commands/import.rs
View File

@ -71,7 +71,7 @@ pub struct ImportCommand {
#[arg(long, verbatim_doc_comment)]
no_state: bool,
/// Chunk byte length.
/// Chunk byte length to read from file.
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,

8
bin/reth/src/commands/import_op.rs
View File

@ -18,7 +18,8 @@ use reth_db_common::init::init_genesis;
use reth_downloaders::file_client::{
ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE,
};
use reth_primitives::{op_mainnet::is_dup_tx, stage::StageId, PruneModes};
use reth_optimism_primitives::bedrock_import::is_dup_tx;
use reth_primitives::{stage::StageId, PruneModes};
use reth_provider::{ProviderFactory, StageCheckpointReader, StaticFileProviderFactory};
use reth_static_file::StaticFileProducer;
use std::{path::PathBuf, sync::Arc};
@ -41,7 +42,7 @@ pub struct ImportOpCommand {
#[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)]
datadir: MaybePlatformPath<DataDirPath>,
/// Chunk byte length.
/// Chunk byte length to read from file.
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
@ -187,6 +188,9 @@ impl ImportOpCommand {
info!(target: "reth::cli",
total_imported_blocks,
total_imported_txns,
total_decoded_blocks,
total_decoded_txns,
total_filtered_out_dup_txns,
"Chain file imported"
);

168
bin/reth/src/commands/import_receipts.rs
View File

@ -1,168 +0,0 @@
//! Command that imports receipts from a file.
use crate::{
args::{
utils::{chain_help, genesis_value_parser, SUPPORTED_CHAINS},
DatabaseArgs,
},
dirs::{DataDirPath, MaybePlatformPath},
};
use clap::Parser;
use reth_db::{database::Database, init_db, transaction::DbTx, DatabaseEnv};
use reth_downloaders::{
file_client::{ChunkedFileReader, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE},
receipt_file_client::ReceiptFileClient,
};
use reth_node_core::version::SHORT_VERSION;
use reth_primitives::{stage::StageId, ChainSpec, StaticFileSegment};
use reth_provider::{
BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
StateWriter, StaticFileProviderFactory, StaticFileWriter,
};
use tracing::{debug, error, info, trace};
use std::{path::PathBuf, sync::Arc};
/// Initializes the database with the genesis block.
#[derive(Debug, Parser)]
pub struct ImportReceiptsCommand {
/// The path to the data dir for all reth files and subdirectories.
///
/// Defaults to the OS-specific data directory:
///
/// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
/// - Windows: `{FOLDERID_RoamingAppData}/reth/`
/// - macOS: `$HOME/Library/Application Support/reth/`
#[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)]
datadir: MaybePlatformPath<DataDirPath>,
/// The chain this node is running.
///
/// Possible values are either a built-in chain or the path to a chain specification file.
#[arg(
long,
value_name = "CHAIN_OR_PATH",
long_help = chain_help(),
default_value = SUPPORTED_CHAINS[0],
value_parser = genesis_value_parser
)]
chain: Arc<ChainSpec>,
/// Chunk byte length.
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
#[command(flatten)]
db: DatabaseArgs,
/// The path to a receipts file for import. File must use `HackReceiptCodec` (used for
/// exporting OP chain segment below Bedrock block via testinprod/op-geth).
///
/// <https://github.com/testinprod-io/op-geth/pull/1>
#[arg(value_name = "IMPORT_PATH", verbatim_doc_comment)]
path: PathBuf,
}
impl ImportReceiptsCommand {
/// Execute `import` command
pub async fn execute(self) -> eyre::Result<()> {
info!(target: "reth::cli", "reth {} starting", SHORT_VERSION);
debug!(target: "reth::cli",
chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE),
"Chunking receipts import"
);
// add network name to data dir
let data_dir = self.datadir.unwrap_or_chain_default(self.chain.chain);
let db_path = data_dir.db();
info!(target: "reth::cli", path = ?db_path, "Opening database");
let db = Arc::new(init_db(db_path, self.db.database_args())?);
info!(target: "reth::cli", "Database opened");
let provider_factory =
ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files())?;
let provider = provider_factory.provider_rw()?;
let static_file_provider = provider_factory.static_file_provider();
for stage in StageId::ALL {
let checkpoint = provider.get_stage_checkpoint(stage)?;
trace!(target: "reth::cli",
?stage,
?checkpoint,
"Read stage checkpoints from db"
);
}
// prepare the tx for `write_to_storage`
let tx = provider.into_tx();
let mut total_decoded_receipts = 0;
// open file
let mut reader = ChunkedFileReader::new(&self.path, self.chunk_len).await?;
while let Some(file_client) = reader.next_chunk::<ReceiptFileClient>().await? {
// create a new file client from chunk read from file
let ReceiptFileClient { receipts, first_block, total_receipts: total_receipts_chunk } =
file_client;
// mark these as decoded
total_decoded_receipts += total_receipts_chunk;
info!(target: "reth::cli",
first_receipts_block=?first_block,
total_receipts_chunk,
"Importing receipt file chunk"
);
// We're reusing receipt writing code internal to
// `BundleStateWithReceipts::write_to_storage`, so we just use a default empty
// `BundleState`.
let bundled_state =
BundleStateWithReceipts::new(Default::default(), receipts, first_block);
let static_file_producer =
static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
// finally, write the receipts
bundled_state.write_to_storage::<<DatabaseEnv as Database>::TXMut>(
&tx,
Some(static_file_producer),
OriginalValuesKnown::Yes,
)?;
}
tx.commit()?;
// as static files works in file ranges, internally it will be committing when creating the
// next file range already, so we only need to call explicitly at the end.
static_file_provider.commit()?;
if total_decoded_receipts == 0 {
error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty");
return Ok(())
}
// compare the highest static file block to the number of receipts we decoded
//
// `HeaderNumbers` and `TransactionHashNumbers` tables serve as additional indexes, but
// nothing like this needs to exist for Receipts. So `tx.entries::<tables::Receipts>` would
// return zero here.
let total_imported_receipts = static_file_provider
.get_highest_static_file_block(StaticFileSegment::Receipts)
.expect("static files must exist after ensuring we decoded more than zero");
if total_imported_receipts != total_decoded_receipts as u64 {
error!(target: "reth::cli",
total_decoded_receipts,
total_imported_receipts,
"Receipts were partially imported"
);
}
info!(target: "reth::cli", total_imported_receipts, "Receipt file imported");
Ok(())
}
}

230
bin/reth/src/commands/import_receipts_op.rs Normal file
View File

@ -0,0 +1,230 @@
//! Command that imports OP mainnet receipts from Bedrock datadir, exported via
//! <https://github.com/testinprod-io/op-geth/pull/1>.
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use clap::Parser;
use reth_db::{database::Database, init_db, tables, transaction::DbTx};
use reth_downloaders::{
file_client::{ChunkedFileReader, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE},
receipt_file_client::ReceiptFileClient,
};
use reth_node_core::version::SHORT_VERSION;
use reth_optimism_primitives::bedrock_import::is_dup_tx;
use reth_primitives::{stage::StageId, Receipts, StaticFileSegment};
use reth_provider::{
BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader,
};
use tracing::{debug, error, info, trace};
use crate::{
args::{
utils::{genesis_value_parser, SUPPORTED_CHAINS},
DatabaseArgs,
},
dirs::{DataDirPath, MaybePlatformPath},
};
/// Initializes the database with the genesis block.
#[derive(Debug, Parser)]
pub struct ImportReceiptsOpCommand {
/// The path to the data dir for all reth files and subdirectories.
///
/// Defaults to the OS-specific data directory:
///
/// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
/// - Windows: `{FOLDERID_RoamingAppData}/reth/`
/// - macOS: `$HOME/Library/Application Support/reth/`
#[arg(long, value_name = "DATA_DIR", verbatim_doc_comment, default_value_t)]
datadir: MaybePlatformPath<DataDirPath>,
/// Chunk byte length to read from file.
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
#[command(flatten)]
db: DatabaseArgs,
/// The path to a receipts file for import. File must use `HackReceiptFileCodec` (used for
/// exporting OP chain segment below Bedrock block via testinprod/op-geth).
///
/// <https://github.com/testinprod-io/op-geth/pull/1>
#[arg(value_name = "IMPORT_PATH", verbatim_doc_comment)]
path: PathBuf,
}
impl ImportReceiptsOpCommand {
/// Execute `import` command
pub async fn execute(self) -> eyre::Result<()> {
info!(target: "reth::cli", "reth {} starting", SHORT_VERSION);
debug!(target: "reth::cli",
chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE),
"Chunking receipts import"
);
let chain_spec = genesis_value_parser(SUPPORTED_CHAINS[0])?;
// add network name to data dir
let data_dir = self.datadir.unwrap_or_chain_default(chain_spec.chain);
let db_path = data_dir.db();
info!(target: "reth::cli", path = ?db_path, "Opening database");
let db = Arc::new(init_db(db_path, self.db.database_args())?);
info!(target: "reth::cli", "Database opened");
let provider_factory =
ProviderFactory::new(db.clone(), chain_spec.clone(), data_dir.static_files())?;
import_receipts_from_file(
provider_factory,
self.path,
self.chunk_len,
|first_block, receipts: &mut Receipts| {
let mut total_filtered_out_dup_txns = 0;
for (index, receipts_for_block) in receipts.iter_mut().enumerate() {
if is_dup_tx(first_block + index as u64) {
receipts_for_block.clear();
total_filtered_out_dup_txns += 1;
}
}
total_filtered_out_dup_txns
},
)
.await
}
}
/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total
/// number of filtered out receipts.
///
/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty
/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects
/// indices in the [`Receipts`] list, to map to sequential block numbers.
pub async fn import_receipts_from_file<DB, P, F>(
provider_factory: ProviderFactory<DB>,
path: P,
chunk_len: Option<u64>,
mut filter: F,
) -> eyre::Result<()>
where
DB: Database,
P: AsRef<Path>,
F: FnMut(u64, &mut Receipts) -> usize,
{
let provider = provider_factory.provider_rw()?;
let static_file_provider = provider_factory.static_file_provider();
let total_imported_txns = static_file_provider
.count_entries::<tables::Transactions>()
.expect("transaction static files must exist before importing receipts");
let highest_block_transactions = static_file_provider
.get_highest_static_file_block(StaticFileSegment::Transactions)
.expect("transaction static files must exist before importing receipts");
for stage in StageId::ALL {
let checkpoint = provider.get_stage_checkpoint(stage)?;
trace!(target: "reth::cli",
?stage,
?checkpoint,
"Read stage checkpoints from db"
);
}
// prepare the tx for `write_to_storage`
let tx = provider.into_tx();
let mut total_decoded_receipts = 0;
let mut total_filtered_out_dup_txns = 0;
// open file
let mut reader = ChunkedFileReader::new(path, chunk_len).await?;
while let Some(file_client) = reader.next_chunk::<ReceiptFileClient>().await? {
// create a new file client from chunk read from file
let ReceiptFileClient { mut receipts, first_block, total_receipts: total_receipts_chunk } =
file_client;
// mark these as decoded
total_decoded_receipts += total_receipts_chunk;
total_filtered_out_dup_txns += filter(first_block, &mut receipts);
info!(target: "reth::cli",
first_receipts_block=?first_block,
total_receipts_chunk,
"Importing receipt file chunk"
);
// We're reusing receipt writing code internal to
// `BundleStateWithReceipts::write_to_storage`, so we just use a default empty
// `BundleState`.
let bundled_state = BundleStateWithReceipts::new(Default::default(), receipts, first_block);
let static_file_producer =
static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
// finally, write the receipts
bundled_state.write_to_storage::<DB::TXMut>(
&tx,
Some(static_file_producer),
OriginalValuesKnown::Yes,
)?;
}
tx.commit()?;
// as static files works in file ranges, internally it will be committing when creating the
// next file range already, so we only need to call explicitly at the end.
static_file_provider.commit()?;
if total_decoded_receipts == 0 {
error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty");
return Ok(())
}
let total_imported_receipts = static_file_provider
.count_entries::<tables::Receipts>()
.expect("static files must exist after ensuring we decoded more than zero");
if total_imported_receipts + total_filtered_out_dup_txns != total_decoded_receipts {
error!(target: "reth::cli",
total_decoded_receipts,
total_imported_receipts,
total_filtered_out_dup_txns,
"Receipts were partially imported"
);
}
if total_imported_receipts != total_imported_txns {
error!(target: "reth::cli",
total_imported_receipts,
total_imported_txns,
"Receipts inconsistent with transactions"
);
}
let highest_block_receipts = static_file_provider
.get_highest_static_file_block(StaticFileSegment::Receipts)
.expect("static files must exist after ensuring we decoded more than zero");
if highest_block_receipts != highest_block_transactions {
error!(target: "reth::cli",
highest_block_receipts,
highest_block_transactions,
"Height of receipts inconsistent with transactions"
);
}
info!(target: "reth::cli",
total_imported_receipts,
total_decoded_receipts,
total_filtered_out_dup_txns,
"Receipt file imported"
);
Ok(())
}

2
bin/reth/src/commands/mod.rs
View File

@ -6,7 +6,7 @@ pub mod debug_cmd;
pub mod dump_genesis;
pub mod import;
pub mod import_op;
pub mod import_receipts;
pub mod import_receipts_op;
pub mod init_cmd;
pub mod init_state;

1
book/SUMMARY.md
View File

@ -32,7 +32,6 @@
- [`reth init`](./cli/reth/init.md)
- [`reth init-state`](./cli/reth/init-state.md)
- [`reth import`](./cli/reth/import.md)
- [`reth import-receipts`](./cli/reth/import-receipts.md)
- [`reth dump-genesis`](./cli/reth/dump-genesis.md)
- [`reth db`](./cli/reth/db.md)
- [`reth db stats`](./cli/reth/db/stats.md)

1
book/cli/SUMMARY.md vendored
View File

@ -3,7 +3,6 @@
- [`reth init`](./reth/init.md)
- [`reth init-state`](./reth/init-state.md)
- [`reth import`](./reth/import.md)
- [`reth import-receipts`](./reth/import-receipts.md)
- [`reth dump-genesis`](./reth/dump-genesis.md)
- [`reth db`](./reth/db.md)
- [`reth db stats`](./reth/db/stats.md)

96
book/cli/op-reth.md vendored Normal file
View File

@ -0,0 +1,96 @@
# op-reth
Additional op-reth commands.
```bash
$ op-reth --help
Usage: op-reth [OPTIONS] <COMMAND>
Commands:
import-op Imports the Bedrock datadir blocks from a file
import-receipts-op Imports the Bedrock datadir receipts from a file
Options:
-h, --help
Print help (see a summary with '-h')
-V, --version
Print version
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

27
book/cli/reth.md vendored
View File

@ -7,20 +7,19 @@ $ reth --help
Usage: reth [OPTIONS] <COMMAND>
Commands:
node Start the node
init Initialize the database from a genesis file
init-state Initialize the database from a state dump file
import This syncs RLP encoded blocks from a file
import-receipts This imports RLP encoded receipts from a file
dump-genesis Dumps genesis block JSON configuration to stdout
db Database debugging utilities
stage Manipulate individual stages
p2p P2P Debugging utilities
test-vectors Generate Test Vectors
config Write config to stdout
debug Various debug routines
recover Scripts for node recovery
help Print this message or the help of the given subcommand(s)
node Start the node
init Initialize the database from a genesis file
init-state Initialize the database from a state dump file
import This syncs RLP encoded blocks from a file
dump-genesis Dumps genesis block JSON configuration to stdout
db Database debugging utilities
stage Manipulate individual stages
p2p P2P Debugging utilities
test-vectors Generate Test Vectors
config Write config to stdout
debug Various debug routines
recover Scripts for node recovery
help Print this message or the help of the given subcommand(s)
Options:
--chain <CHAIN_OR_PATH>

134
book/cli/reth/import-op.md vendored Normal file
View File

@ -0,0 +1,134 @@
# op-reth import
This syncs RLP encoded blocks from a file. Supports import of OVM blocks
from the Bedrock datadir. Requires blocks, up to same height as receipts
file, to already be imported.
```bash
$ op-reth import-op --help
Usage: op-reth import-op [OPTIONS] <IMPORT_PATH>
Options:
--config <FILE>
The path to the configuration file to use.
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--chunk-len <CHUNK_LEN>
Chunk byte length to read from file.
[default: 1GB]
-h, --help
Print help (see a summary with '-h')
Database:
--db.log-level <LOG_LEVEL>
Database logging level. Levels higher than "notice" require a debug build
Possible values:
- fatal: Enables logging for critical conditions, i.e. assertion failures
- error: Enables logging for error conditions
- warn: Enables logging for warning conditions
- notice: Enables logging for normal but significant condition
- verbose: Enables logging for verbose informational
- debug: Enables logging for debug-level messages
- trace: Enables logging for trace debug-level messages
- extra: Enables logging for extra debug-level messages
--db.exclusive <EXCLUSIVE>
Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume
[possible values: true, false]
<IMPORT_PATH>
The path to a `.rlp` block file for import.
The online sync pipeline stages (headers and bodies) are replaced by a file import. Skips block execution since blocks below Bedrock are built on OVM.
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

35
book/cli/reth/import-receipts.md → book/cli/reth/import-receipts-op.md vendored
View File

@ -1,10 +1,13 @@
# reth import-receipts
# op-reth import-receipts-op
This imports RLP encoded receipts from a file
This imports non-standard RLP encoded receipts from a file.
The supported RLP encoding, is the non-standard encoding used
for receipt export in <https://github.com/testinprod-io/op-geth/pull/1>.
Supports import of OVM receipts from the Bedrock datadir.
```bash
$ reth import-receipts --help
Usage: reth import-receipts [OPTIONS] <IMPORT_PATH>
$ op-reth import-receipts-op --help
Usage: op-reth import-receipts-op [OPTIONS] <IMPORT_PATH>
Options:
--datadir <DATA_DIR>
@ -18,28 +21,10 @@ Options:
[default: default]
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--chunk-len <CHUNK_LEN>
Chunk byte length.
Chunk byte length to read from file.
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
[default: 1GB]
-h, --help
Print help (see a summary with '-h')
@ -64,7 +49,7 @@ Database:
[possible values: true, false]
<IMPORT_PATH>
The path to a receipts file for import. File must use `HackReceiptCodec` (used for
The path to a receipts file for import. File must use `HackReceiptFileCodec` (used for
exporting OP chain segment below Bedrock block via testinprod/op-geth).
<https://github.com/testinprod-io/op-geth/pull/1>

2
book/cli/reth/import.md vendored
View File

@ -34,7 +34,7 @@ Options:
Disables stages that require state.
--chunk-len <CHUNK_LEN>
Chunk byte length.
Chunk byte length to read from file.
--instance <INSTANCE>
Add a new instance of a node.

10
book/run/sync-op-mainnet.md
View File

@ -22,7 +22,7 @@ Output from running the command to export state, can also be downloaded from <ht
Imports a `.rlp` file of blocks.
Note! Requires running in debug mode (TODO: <https://github.com/paradigmxyz/reth/issues/7650>).
Import of >100 million OVM blocks, from genesis to Bedrock, completes in 6 hours.
```bash
./op-reth import-op <exported-blocks>
@ -36,14 +36,18 @@ corresponding transactions must already be imported (see [step 1](#1-import-bloc
Imports a `.rlp` file of receipts, that has been exported with command specified in
<https://github.com/testinprod-io/op-geth/pull/1> (command for exporting receipts uses custom RLP-encoding).
Import of >100 million OVM receipts, from genesis to Bedrock, completes in 30 minutes.
```bash
./op-reth import-receipts --chain optimism <exported-receipts>
./op-reth import-receipts-op <exported-receipts>
```
### 3. Import State
Imports a `.jsonl` state dump. The block at which the state dump is made, must be the latest block in
reth's database.
reth's database. This should be block 105 235 063, the first Bedrock block (see [step 1](#1-import-blocks)).
Import of >4 million OP mainnet accounts at Bedrock, completes in 10 minutes.
```bash
./op-reth init-state --chain optimism <state-dump>

1
crates/consensus/common/Cargo.toml
View File

@ -13,6 +13,7 @@ workspace = true
[dependencies]
# reth
reth-primitives.workspace = true
reth-optimism-primitives.workspace = true
reth-consensus.workspace=true
[dev-dependencies]

2
crates/consensus/common/src/validation.rs
View File

@ -1,12 +1,12 @@
//! Collection of methods for block validation.
use reth_consensus::ConsensusError;
use reth_optimism_primitives::bedrock_import::is_dup_tx;
use reth_primitives::{
constants::{
eip4844::{DATA_GAS_PER_BLOB, MAX_DATA_GAS_PER_BLOCK},
MAXIMUM_EXTRA_DATA_SIZE,
},
op_mainnet::is_dup_tx,
ChainSpec, GotExpected, Hardfork, Header, SealedBlock, SealedHeader,
};

13
crates/net/downloaders/src/receipt_file_client.rs
View File

@ -75,10 +75,10 @@ impl FromReader for ReceiptFileClient {
Err(err) => return Err(err),
};
total_receipts += 1;
match receipt {
Some(ReceiptWithBlockNumber { receipt, number }) => {
total_receipts += 1;
if first_block.is_none() {
first_block = Some(number);
block_number = number;
@ -202,7 +202,8 @@ mod test {
let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) =
ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap();
assert_eq!(4, total_receipts);
// 2 non-empty receipt objects
assert_eq!(2, total_receipts);
assert_eq!(0, first_block);
assert!(receipts[0].is_empty());
assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap());
@ -229,7 +230,8 @@ mod test {
let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) =
ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap();
assert_eq!(4, total_receipts);
// 2 non-empty receipt objects
assert_eq!(2, total_receipts);
assert_eq!(0, first_block);
assert!(receipts[0].is_empty());
assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap());
@ -257,7 +259,8 @@ mod test {
let (ReceiptFileClient { receipts, first_block, total_receipts }, _remaining_bytes) =
ReceiptFileClient::from_reader(reader, encoded_byte_len).await.unwrap();
assert_eq!(5, total_receipts);
// 4 non-empty receipt objects
assert_eq!(4, total_receipts);
assert_eq!(0, first_block);
assert!(receipts[0].is_empty());
assert_eq!(op_mainnet_receipt_block_1().receipt, receipts[1][0].clone().unwrap());

12
crates/optimism/primitives/Cargo.toml Normal file
View File

@ -0,0 +1,12 @@
[package]
name = "reth-optimism-primitives"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
description = "OP primitive types"
[lints]
workspace = true

2
crates/primitives/src/op_mainnet.rs → crates/optimism/primitives/src/bedrock_import.rs
View File

@ -1,4 +1,4 @@
//! Helpers for working with replayed OP mainnet OVM transactions (in blocks below Bedrock).
//! Replayed OP mainnet OVM transactions (in blocks below Bedrock).
/// Transaction 0x9ed8f713b2cc6439657db52dcd2fdb9cc944915428f3c6e2a7703e242b259cb9 in block 985,
/// replayed in blocks:

10
crates/optimism/primitives/src/lib.rs Normal file
View File

@ -0,0 +1,10 @@
//! Standalone crate for Optimism-specific Reth primitive types.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
)]
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
pub mod bedrock_import;

1
crates/primitives/src/lib.rs
View File

@ -35,7 +35,6 @@ mod header;
mod integer_list;
mod log;
mod net;
pub mod op_mainnet;
pub mod proofs;
mod prune;
mod receipt;