fix(op): add empty receipts for genesis if first block is one (#9769)

Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>

Cargo.lock

@@ -7907,6 +7907,7 @@ dependencies = [
  "reth-consensus",
  "reth-db",
  "reth-db-api",
+ "reth-db-common",
  "reth-downloaders",
  "reth-errors",
  "reth-evm-optimism",
@@ -7924,6 +7925,7 @@ dependencies = [
  "reth-static-file-types",
  "serde_json",
  "shellexpand",
+ "tempfile",
  "tokio",
  "tokio-util",
  "tracing",

crates/net/downloaders/src/receipt_file_client.rs

@@ -289,11 +289,14 @@ mod test {
         }
     }
 
-    pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001");
+    /// No receipts for genesis block
+    const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0");
 
-    pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002");
+    const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001");
 
-    pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003");
+    const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002");
+
+    const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003");
 
     fn mock_receipt_1() -> MockReceipt {
         let receipt = receipt_block_1();
@@ -331,7 +334,7 @@ mod test {
         }
     }
 
-    pub(crate) fn receipt_block_1() -> ReceiptWithBlockNumber {
+    fn receipt_block_1() -> ReceiptWithBlockNumber {
         let log_1 = Log {
             address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
             data: LogData::new(
@@ -404,7 +407,7 @@ mod test {
         ReceiptWithBlockNumber { receipt, number: 1 }
     }
 
-    pub(crate) fn receipt_block_2() -> ReceiptWithBlockNumber {
+    fn receipt_block_2() -> ReceiptWithBlockNumber {
         let log_1 = Log {
             address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
             data: LogData::new(
@@ -456,7 +459,7 @@ mod test {
         ReceiptWithBlockNumber { receipt, number: 2 }
     }
 
-    pub(crate) fn receipt_block_3() -> ReceiptWithBlockNumber {
+    fn receipt_block_3() -> ReceiptWithBlockNumber {
         let log_1 = Log {
             address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
             data: LogData::new(
@@ -560,9 +563,6 @@ mod test {
         assert_eq!(receipt_block_3(), third_decoded_receipt);
     }
 
-    /// No receipts for genesis block
-    const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0");
-
     #[tokio::test]
     async fn receipt_file_client_ovm_codec() {
         init_test_tracing();

crates/optimism/cli/Cargo.toml

@@ -60,8 +60,14 @@ tokio-util = { workspace = true, features = ["codec"] }
 tracing.workspace = true
 eyre.workspace = true
 
+[dev-dependencies]
+tempfile.workspace = true
+reth-stages = { workspace = true, features = ["test-utils"] }
+reth-db-common.workspace = true
+
 [features] 
  optimism = [ 
     "reth-primitives/optimism",
     "reth-evm-optimism/optimism",
+    "reth-provider/optimism",
  ] 

crates/optimism/cli/src/commands/import_receipts.rs

@@ -16,8 +16,8 @@ use reth_node_core::version::SHORT_VERSION;
 use reth_optimism_primitives::bedrock_import::is_dup_tx;
 use reth_primitives::Receipts;
 use reth_provider::{
-    writer::StorageWriter, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
+    writer::StorageWriter, DatabaseProviderFactory, OriginalValuesKnown, ProviderFactory,
-    StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader,
+    StageCheckpointReader, StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader,
 };
 use reth_stages::StageId;
 use reth_static_file_types::StaticFileSegment;
@@ -75,35 +75,29 @@ impl ImportReceiptsOpCommand {
     }
 }
 
-/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total
+/// Imports receipts to static files from file in chunks. See [`import_receipts_from_reader`].
-/// number of filtered out receipts.
-///
-/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty
-/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects
-/// indices in the [`Receipts`] list, to map to sequential block numbers.
 pub async fn import_receipts_from_file<DB, P, F>(
     provider_factory: ProviderFactory<DB>,
     path: P,
     chunk_len: Option<u64>,
-    mut filter: F,
+    filter: F,
 ) -> eyre::Result<()>
 where
     DB: Database,
     P: AsRef<Path>,
     F: FnMut(u64, &mut Receipts) -> usize,
 {
-    let provider = provider_factory.provider_rw()?;
+    let total_imported_txns = provider_factory
-    let static_file_provider = provider_factory.static_file_provider();
+        .static_file_provider()
-
-    let total_imported_txns = static_file_provider
         .count_entries::<tables::Transactions>()
         .expect("transaction static files must exist before importing receipts");
-    let highest_block_transactions = static_file_provider
+    let highest_block_transactions = provider_factory
+        .static_file_provider()
         .get_highest_static_file_block(StaticFileSegment::Transactions)
         .expect("transaction static files must exist before importing receipts");
 
     for stage in StageId::ALL {
-        let checkpoint = provider.get_stage_checkpoint(stage)?;
+        let checkpoint = provider_factory.database_provider_ro()?.get_stage_checkpoint(stage)?;
         trace!(target: "reth::cli",
             ?stage,
             ?checkpoint,
@@ -111,59 +105,20 @@ where
         );
     }
 
-    let mut total_decoded_receipts = 0;
-    let mut total_filtered_out_dup_txns = 0;
-
     // open file
-    let mut reader = ChunkedFileReader::new(path, chunk_len).await?;
+    let reader = ChunkedFileReader::new(&path, chunk_len).await?;
 
-    while let Some(file_client) =
+    // import receipts
-        reader.next_chunk::<ReceiptFileClient<HackReceiptFileCodec>>().await?
+    let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } =
-    {
+        import_receipts_from_reader(&provider_factory, reader, filter).await?;
-        // create a new file client from chunk read from file
-        let ReceiptFileClient {
-            mut receipts,
-            first_block,
-            total_receipts: total_receipts_chunk,
-            ..
-        } = file_client;
-
-        // mark these as decoded
-        total_decoded_receipts += total_receipts_chunk;
-
-        total_filtered_out_dup_txns += filter(first_block, &mut receipts);
-
-        info!(target: "reth::cli",
-            first_receipts_block=?first_block,
-            total_receipts_chunk,
-            "Importing receipt file chunk"
-        );
-
-        // We're reusing receipt writing code internal to
-        // `StorageWriter::append_receipts_from_blocks`, so we just use a default empty
-        // `BundleState`.
-        let execution_outcome =
-            ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default());
-
-        let static_file_producer =
-            static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
-
-        // finally, write the receipts
-        let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer));
-        storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?;
-    }
-
-    provider.commit()?;
-    // as static files works in file ranges, internally it will be committing when creating the
-    // next file range already, so we only need to call explicitly at the end.
-    static_file_provider.commit()?;
 
     if total_decoded_receipts == 0 {
         error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty");
         return Ok(())
     }
 
-    let total_imported_receipts = static_file_provider
+    let total_imported_receipts = provider_factory
+        .static_file_provider()
         .count_entries::<tables::Receipts>()
         .expect("static files must exist after ensuring we decoded more than zero");
 
@@ -184,7 +139,8 @@ where
         );
     }
 
-    let highest_block_receipts = static_file_provider
+    let highest_block_receipts = provider_factory
+        .static_file_provider()
         .get_highest_static_file_block(StaticFileSegment::Receipts)
         .expect("static files must exist after ensuring we decoded more than zero");
 
@@ -205,3 +161,140 @@ where
 
     Ok(())
 }
+
+/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total
+/// number of filtered out receipts.
+///
+/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty
+/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects
+/// indices in the [`Receipts`] list, to map to sequential block numbers.
+pub async fn import_receipts_from_reader<DB, F>(
+    provider_factory: &ProviderFactory<DB>,
+    mut reader: ChunkedFileReader,
+    mut filter: F,
+) -> eyre::Result<ImportReceiptsResult>
+where
+    DB: Database,
+    F: FnMut(u64, &mut Receipts) -> usize,
+{
+    let mut total_decoded_receipts = 0;
+    let mut total_filtered_out_dup_txns = 0;
+
+    let provider = provider_factory.provider_rw()?;
+    let static_file_provider = provider_factory.static_file_provider();
+
+    while let Some(file_client) =
+        reader.next_chunk::<ReceiptFileClient<HackReceiptFileCodec>>().await?
+    {
+        // create a new file client from chunk read from file
+        let ReceiptFileClient {
+            mut receipts,
+            mut first_block,
+            total_receipts: total_receipts_chunk,
+            ..
+        } = file_client;
+
+        // mark these as decoded
+        total_decoded_receipts += total_receipts_chunk;
+
+        total_filtered_out_dup_txns += filter(first_block, &mut receipts);
+
+        info!(target: "reth::cli",
+            first_receipts_block=?first_block,
+            total_receipts_chunk,
+            "Importing receipt file chunk"
+        );
+
+        // It is possible for the first receipt returned by the file client to be the genesis
+        // block. In this case, we just prepend empty receipts to the current list of receipts.
+        // When initially writing to static files, the provider expects the first block to be block
+        // one. So, if the first block returned by the file client is the genesis block, we remove
+        // those receipts.
+        if first_block == 0 {
+            // remove the first empty receipts
+            let genesis_receipts = receipts.remove(0);
+            debug_assert!(genesis_receipts.is_empty());
+            // this ensures the execution outcome and static file producer start at block 1
+            first_block = 1;
+            // we don't count this as decoded so the partial import check later does not error if
+            // this branch is executed
+            total_decoded_receipts -= 1; // safe because chunk will be `None` if empty
+        }
+
+        // We're reusing receipt writing code internal to
+        // `StorageWriter::append_receipts_from_blocks`, so we just use a default empty
+        // `BundleState`.
+        let execution_outcome =
+            ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default());
+
+        let static_file_producer =
+            static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
+
+        // finally, write the receipts
+        let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer));
+        storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?;
+    }
+
+    provider.commit()?;
+    // as static files works in file ranges, internally it will be committing when creating the
+    // next file range already, so we only need to call explicitly at the end.
+    static_file_provider.commit()?;
+
+    Ok(ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns })
+}
+
+/// Result of importing receipts in chunks.
+#[derive(Debug)]
+pub struct ImportReceiptsResult {
+    total_decoded_receipts: usize,
+    total_filtered_out_dup_txns: usize,
+}
+
+#[cfg(test)]
+mod test {
+    use reth_db_common::init::init_genesis;
+    use reth_primitives::hex;
+    use reth_stages::test_utils::TestStageDB;
+    use tempfile::tempfile;
+    use tokio::{
+        fs::File,
+        io::{AsyncSeekExt, AsyncWriteExt, SeekFrom},
+    };
+
+    use crate::file_codec_ovm_receipt::test::{
+        HACK_RECEIPT_ENCODED_BLOCK_1, HACK_RECEIPT_ENCODED_BLOCK_2, HACK_RECEIPT_ENCODED_BLOCK_3,
+    };
+
+    use super::*;
+
+    /// No receipts for genesis block
+    const EMPTY_RECEIPTS_GENESIS_BLOCK: &[u8] = &hex!("c0");
+
+    #[ignore]
+    #[tokio::test]
+    async fn filter_out_genesis_block_receipts() {
+        let mut f: File = tempfile().unwrap().into();
+        f.write_all(EMPTY_RECEIPTS_GENESIS_BLOCK).await.unwrap();
+        f.write_all(HACK_RECEIPT_ENCODED_BLOCK_1).await.unwrap();
+        f.write_all(HACK_RECEIPT_ENCODED_BLOCK_2).await.unwrap();
+        f.write_all(HACK_RECEIPT_ENCODED_BLOCK_3).await.unwrap();
+        f.flush().await.unwrap();
+        f.seek(SeekFrom::Start(0)).await.unwrap();
+
+        let reader =
+            ChunkedFileReader::from_file(f, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE).await.unwrap();
+
+        let db = TestStageDB::default();
+        init_genesis(db.factory.clone()).unwrap();
+
+        // todo: where does import command init receipts ? probably somewhere in pipeline
+
+        let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } =
+            import_receipts_from_reader(&TestStageDB::default().factory, reader, |_, _| 0)
+                .await
+                .unwrap();
+
+        assert_eq!(total_decoded_receipts, 3);
+        assert_eq!(total_filtered_out_dup_txns, 0);
+    }
+}

crates/optimism/cli/src/file_codec_ovm_receipt.rs

@@ -93,7 +93,7 @@ impl TryFrom<HackReceipt> for ReceiptWithBlockNumber {
 }
 
 #[cfg(test)]
-pub(super) mod test {
+pub(crate) mod test {
     use reth_primitives::{alloy_primitives::LogData, hex};
 
     use super::*;