From 0a4b717d1bac88f031787ac58a6550fd426f56fb Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 26 Jul 2024 19:07:25 +0200 Subject: [PATCH] fix(op): add empty receipts for genesis if first block is one (#9769) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- Cargo.lock | 2 + .../downloaders/src/receipt_file_client.rs | 18 +- crates/optimism/cli/Cargo.toml | 6 + .../cli/src/commands/import_receipts.rs | 215 +++++++++++++----- .../cli/src/file_codec_ovm_receipt.rs | 2 +- 5 files changed, 172 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85b631902..d61374015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7907,6 +7907,7 @@ dependencies = [ "reth-consensus", "reth-db", "reth-db-api", + "reth-db-common", "reth-downloaders", "reth-errors", "reth-evm-optimism", @@ -7924,6 +7925,7 @@ dependencies = [ "reth-static-file-types", "serde_json", "shellexpand", + "tempfile", "tokio", "tokio-util", "tracing", diff --git a/crates/net/downloaders/src/receipt_file_client.rs b/crates/net/downloaders/src/receipt_file_client.rs index c32a8903e..3889350df 100644 --- a/crates/net/downloaders/src/receipt_file_client.rs +++ b/crates/net/downloaders/src/receipt_file_client.rs @@ -289,11 +289,14 @@ mod test { } } - pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001"); + /// No receipts for genesis block + const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0"); - pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002"); + const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001"); - pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003"); + const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002"); + + const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003"); fn mock_receipt_1() -> MockReceipt { let receipt = receipt_block_1(); @@ -331,7 +334,7 @@ mod test { } } - pub(crate) fn receipt_block_1() -> ReceiptWithBlockNumber { + fn receipt_block_1() -> ReceiptWithBlockNumber { let log_1 = Log { address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")), data: LogData::new( @@ -404,7 +407,7 @@ mod test { ReceiptWithBlockNumber { receipt, number: 1 } } - pub(crate) fn receipt_block_2() -> ReceiptWithBlockNumber { + fn receipt_block_2() -> ReceiptWithBlockNumber { let log_1 = Log { address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")), data: LogData::new( @@ -456,7 +459,7 @@ mod test { ReceiptWithBlockNumber { receipt, number: 2 } } - pub(crate) fn receipt_block_3() -> ReceiptWithBlockNumber { + fn receipt_block_3() -> ReceiptWithBlockNumber { let log_1 = Log { address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")), data: LogData::new( @@ -560,9 +563,6 @@ mod test { assert_eq!(receipt_block_3(), third_decoded_receipt); } - /// No receipts for genesis block - const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0"); - #[tokio::test] async fn receipt_file_client_ovm_codec() { init_test_tracing(); diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 3ca591480..82f9d999d 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -60,8 +60,14 @@ tokio-util = { workspace = true, features = ["codec"] } tracing.workspace = true eyre.workspace = true +[dev-dependencies] +tempfile.workspace = true +reth-stages = { workspace = true, features = ["test-utils"] } +reth-db-common.workspace = true + [features] optimism = [ "reth-primitives/optimism", "reth-evm-optimism/optimism", + "reth-provider/optimism", ] \ No newline at end of file diff --git a/crates/optimism/cli/src/commands/import_receipts.rs b/crates/optimism/cli/src/commands/import_receipts.rs index f6b4a792c..30d2f82c9 100644 --- a/crates/optimism/cli/src/commands/import_receipts.rs +++ b/crates/optimism/cli/src/commands/import_receipts.rs @@ -16,8 +16,8 @@ use reth_node_core::version::SHORT_VERSION; use reth_optimism_primitives::bedrock_import::is_dup_tx; use reth_primitives::Receipts; use reth_provider::{ - writer::StorageWriter, OriginalValuesKnown, ProviderFactory, StageCheckpointReader, - StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader, + writer::StorageWriter, DatabaseProviderFactory, OriginalValuesKnown, ProviderFactory, + StageCheckpointReader, StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader, }; use reth_stages::StageId; use reth_static_file_types::StaticFileSegment; @@ -75,35 +75,29 @@ impl ImportReceiptsOpCommand { } } -/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total -/// number of filtered out receipts. -/// -/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty -/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects -/// indices in the [`Receipts`] list, to map to sequential block numbers. +/// Imports receipts to static files from file in chunks. See [`import_receipts_from_reader`]. pub async fn import_receipts_from_file( provider_factory: ProviderFactory, path: P, chunk_len: Option, - mut filter: F, + filter: F, ) -> eyre::Result<()> where DB: Database, P: AsRef, F: FnMut(u64, &mut Receipts) -> usize, { - let provider = provider_factory.provider_rw()?; - let static_file_provider = provider_factory.static_file_provider(); - - let total_imported_txns = static_file_provider + let total_imported_txns = provider_factory + .static_file_provider() .count_entries::() .expect("transaction static files must exist before importing receipts"); - let highest_block_transactions = static_file_provider + let highest_block_transactions = provider_factory + .static_file_provider() .get_highest_static_file_block(StaticFileSegment::Transactions) .expect("transaction static files must exist before importing receipts"); for stage in StageId::ALL { - let checkpoint = provider.get_stage_checkpoint(stage)?; + let checkpoint = provider_factory.database_provider_ro()?.get_stage_checkpoint(stage)?; trace!(target: "reth::cli", ?stage, ?checkpoint, @@ -111,59 +105,20 @@ where ); } - let mut total_decoded_receipts = 0; - let mut total_filtered_out_dup_txns = 0; - // open file - let mut reader = ChunkedFileReader::new(path, chunk_len).await?; + let reader = ChunkedFileReader::new(&path, chunk_len).await?; - while let Some(file_client) = - reader.next_chunk::>().await? - { - // create a new file client from chunk read from file - let ReceiptFileClient { - mut receipts, - first_block, - total_receipts: total_receipts_chunk, - .. - } = file_client; - - // mark these as decoded - total_decoded_receipts += total_receipts_chunk; - - total_filtered_out_dup_txns += filter(first_block, &mut receipts); - - info!(target: "reth::cli", - first_receipts_block=?first_block, - total_receipts_chunk, - "Importing receipt file chunk" - ); - - // We're reusing receipt writing code internal to - // `StorageWriter::append_receipts_from_blocks`, so we just use a default empty - // `BundleState`. - let execution_outcome = - ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default()); - - let static_file_producer = - static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?; - - // finally, write the receipts - let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer)); - storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?; - } - - provider.commit()?; - // as static files works in file ranges, internally it will be committing when creating the - // next file range already, so we only need to call explicitly at the end. - static_file_provider.commit()?; + // import receipts + let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } = + import_receipts_from_reader(&provider_factory, reader, filter).await?; if total_decoded_receipts == 0 { error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty"); return Ok(()) } - let total_imported_receipts = static_file_provider + let total_imported_receipts = provider_factory + .static_file_provider() .count_entries::() .expect("static files must exist after ensuring we decoded more than zero"); @@ -184,7 +139,8 @@ where ); } - let highest_block_receipts = static_file_provider + let highest_block_receipts = provider_factory + .static_file_provider() .get_highest_static_file_block(StaticFileSegment::Receipts) .expect("static files must exist after ensuring we decoded more than zero"); @@ -205,3 +161,140 @@ where Ok(()) } + +/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total +/// number of filtered out receipts. +/// +/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty +/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects +/// indices in the [`Receipts`] list, to map to sequential block numbers. +pub async fn import_receipts_from_reader( + provider_factory: &ProviderFactory, + mut reader: ChunkedFileReader, + mut filter: F, +) -> eyre::Result +where + DB: Database, + F: FnMut(u64, &mut Receipts) -> usize, +{ + let mut total_decoded_receipts = 0; + let mut total_filtered_out_dup_txns = 0; + + let provider = provider_factory.provider_rw()?; + let static_file_provider = provider_factory.static_file_provider(); + + while let Some(file_client) = + reader.next_chunk::>().await? + { + // create a new file client from chunk read from file + let ReceiptFileClient { + mut receipts, + mut first_block, + total_receipts: total_receipts_chunk, + .. + } = file_client; + + // mark these as decoded + total_decoded_receipts += total_receipts_chunk; + + total_filtered_out_dup_txns += filter(first_block, &mut receipts); + + info!(target: "reth::cli", + first_receipts_block=?first_block, + total_receipts_chunk, + "Importing receipt file chunk" + ); + + // It is possible for the first receipt returned by the file client to be the genesis + // block. In this case, we just prepend empty receipts to the current list of receipts. + // When initially writing to static files, the provider expects the first block to be block + // one. So, if the first block returned by the file client is the genesis block, we remove + // those receipts. + if first_block == 0 { + // remove the first empty receipts + let genesis_receipts = receipts.remove(0); + debug_assert!(genesis_receipts.is_empty()); + // this ensures the execution outcome and static file producer start at block 1 + first_block = 1; + // we don't count this as decoded so the partial import check later does not error if + // this branch is executed + total_decoded_receipts -= 1; // safe because chunk will be `None` if empty + } + + // We're reusing receipt writing code internal to + // `StorageWriter::append_receipts_from_blocks`, so we just use a default empty + // `BundleState`. + let execution_outcome = + ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default()); + + let static_file_producer = + static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?; + + // finally, write the receipts + let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer)); + storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?; + } + + provider.commit()?; + // as static files works in file ranges, internally it will be committing when creating the + // next file range already, so we only need to call explicitly at the end. + static_file_provider.commit()?; + + Ok(ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns }) +} + +/// Result of importing receipts in chunks. +#[derive(Debug)] +pub struct ImportReceiptsResult { + total_decoded_receipts: usize, + total_filtered_out_dup_txns: usize, +} + +#[cfg(test)] +mod test { + use reth_db_common::init::init_genesis; + use reth_primitives::hex; + use reth_stages::test_utils::TestStageDB; + use tempfile::tempfile; + use tokio::{ + fs::File, + io::{AsyncSeekExt, AsyncWriteExt, SeekFrom}, + }; + + use crate::file_codec_ovm_receipt::test::{ + HACK_RECEIPT_ENCODED_BLOCK_1, HACK_RECEIPT_ENCODED_BLOCK_2, HACK_RECEIPT_ENCODED_BLOCK_3, + }; + + use super::*; + + /// No receipts for genesis block + const EMPTY_RECEIPTS_GENESIS_BLOCK: &[u8] = &hex!("c0"); + + #[ignore] + #[tokio::test] + async fn filter_out_genesis_block_receipts() { + let mut f: File = tempfile().unwrap().into(); + f.write_all(EMPTY_RECEIPTS_GENESIS_BLOCK).await.unwrap(); + f.write_all(HACK_RECEIPT_ENCODED_BLOCK_1).await.unwrap(); + f.write_all(HACK_RECEIPT_ENCODED_BLOCK_2).await.unwrap(); + f.write_all(HACK_RECEIPT_ENCODED_BLOCK_3).await.unwrap(); + f.flush().await.unwrap(); + f.seek(SeekFrom::Start(0)).await.unwrap(); + + let reader = + ChunkedFileReader::from_file(f, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE).await.unwrap(); + + let db = TestStageDB::default(); + init_genesis(db.factory.clone()).unwrap(); + + // todo: where does import command init receipts ? probably somewhere in pipeline + + let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } = + import_receipts_from_reader(&TestStageDB::default().factory, reader, |_, _| 0) + .await + .unwrap(); + + assert_eq!(total_decoded_receipts, 3); + assert_eq!(total_filtered_out_dup_txns, 0); + } +} diff --git a/crates/optimism/cli/src/file_codec_ovm_receipt.rs b/crates/optimism/cli/src/file_codec_ovm_receipt.rs index d452efb1c..b2643b840 100644 --- a/crates/optimism/cli/src/file_codec_ovm_receipt.rs +++ b/crates/optimism/cli/src/file_codec_ovm_receipt.rs @@ -93,7 +93,7 @@ impl TryFrom for ReceiptWithBlockNumber { } #[cfg(test)] -pub(super) mod test { +pub(crate) mod test { use reth_primitives::{alloy_primitives::LogData, hex}; use super::*;