feat: add reth test-vectors compact --write|--read (#11954)

.github/workflows/compact.yml

@@ -0,0 +1,42 @@
+# Ensures that `Compact` codec changes are backwards compatible.
+# 
+# 1) checkout `main`
+# 2) randomly generate and serialize to disk many different type vectors with `Compact` (eg. Header, Transaction, etc)
+# 3) checkout `pr`
+# 4) deserialize previously generated test vectors
+
+on:
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+name: compact-codec
+jobs:
+  compact-codec:
+    runs-on:
+      group: Reth
+    strategy:
+      matrix:
+        bin:
+          - cargo run --bin reth --features "dev"
+          - cargo run --bin op-reth --features "optimism dev" --manifest-path crates/optimism/bin/Cargo.toml
+    steps:
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - name: Checkout base
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref || 'main' }}
+      # On `main` branch, generates test vectors and serializes them to disk using `Compact`.
+      - name: Generate compact vectors
+        run: ${{ matrix.bin }} -- test-vectors compact --write
+      - name: Checkout PR
+        uses: actions/checkout@v4
+        with:
+          clean: false
+      # On incoming merge try to read and decode previously generated vectors with `Compact`
+      - name: Read vectors
+        run: ${{ matrix.bin }} -- test-vectors compact --read

Cargo.lock

@@ -6615,6 +6615,7 @@ dependencies = [
  "reth-cli",
  "reth-cli-runner",
  "reth-cli-util",
+ "reth-codecs",
  "reth-config",
  "reth-consensus",
  "reth-db",
@@ -6638,10 +6639,13 @@ dependencies = [
  "reth-primitives",
  "reth-provider",
  "reth-prune",
+ "reth-prune-types",
  "reth-stages",
+ "reth-stages-types",
  "reth-static-file",
  "reth-static-file-types",
  "reth-trie",
+ "reth-trie-common",
  "reth-trie-db",
  "secp256k1",
  "serde",
@@ -6696,6 +6700,7 @@ dependencies = [
  "serde",
  "serde_json",
  "test-fuzz",
+ "visibility",
 ]
 
 [[package]]
@@ -8090,6 +8095,8 @@ dependencies = [
  "clap",
  "eyre",
  "futures-util",
+ "op-alloy-consensus",
+ "proptest",
  "reth-chainspec",
  "reth-cli",
  "reth-cli-commands",
@@ -11233,6 +11240,17 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "visibility"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.80",
+]
+
 [[package]]
 name = "wait-timeout"
 version = "0.2.0"

crates/cli/commands/Cargo.toml

@@ -17,6 +17,7 @@ reth-cli.workspace = true
 reth-ethereum-cli.workspace = true
 reth-cli-runner.workspace = true
 reth-cli-util.workspace = true
+reth-codecs = { workspace = true, optional = true }
 reth-config.workspace = true
 reth-consensus.workspace = true
 reth-db = { workspace = true, features = ["mdbx"] }
@@ -38,11 +39,14 @@ reth-node-metrics.workspace = true
 reth-primitives.workspace = true
 reth-provider.workspace = true
 reth-prune.workspace = true
+reth-prune-types = { workspace = true, optional = true }
 reth-stages.workspace = true
+reth-stages-types = { workspace = true, optional = true }
 reth-static-file-types = { workspace = true, features = ["clap"] }
 reth-static-file.workspace = true
 reth-trie = { workspace = true, features = ["metrics"] }
 reth-trie-db = { workspace = true, features = ["metrics"] }
+reth-trie-common = { workspace = true, optional = true }
 
 # ethereum
 alloy-eips.workspace = true
@@ -89,14 +93,22 @@ reth-discv4.workspace = true
 [features]
 default = []
 arbitrary = [
-    "dep:proptest",
+	"dep:proptest",
-    "dep:arbitrary",
+	"dep:arbitrary",
-    "dep:proptest-arbitrary-interop",
+	"dep:proptest-arbitrary-interop",
-    "reth-primitives/arbitrary",
+	"reth-primitives/arbitrary",
-    "reth-db-api/arbitrary",
+	"reth-db-api/arbitrary",
-    "reth-eth-wire/arbitrary",
+	"reth-eth-wire/arbitrary",
-    "reth-db/arbitrary",
+	"reth-db/arbitrary",
-    "reth-chainspec/arbitrary",
+	"reth-chainspec/arbitrary",
-    "alloy-eips/arbitrary",
+	"alloy-eips/arbitrary",
-    "alloy-primitives/arbitrary",
+	"alloy-primitives/arbitrary",
+	"reth-codecs/test-utils",
+	"reth-prune-types/test-utils",
+	"reth-stages-types/test-utils",
+	"reth-trie-common/test-utils",
+	"reth-codecs?/arbitrary",
+	"reth-prune-types?/arbitrary",
+	"reth-stages-types?/arbitrary",
+	"reth-trie-common?/arbitrary"
 ]

crates/cli/commands/src/test_vectors/compact.rs

@@ -0,0 +1,257 @@
+use alloy_primitives::{hex, private::getrandom::getrandom, TxKind};
+use arbitrary::Arbitrary;
+use eyre::{Context, Result};
+use proptest::{
+    prelude::{ProptestConfig, RngCore},
+    test_runner::{TestRng, TestRunner},
+};
+use reth_codecs::alloy::{
+    authorization_list::Authorization,
+    genesis_account::GenesisAccount,
+    header::{Header, HeaderExt},
+    transaction::{
+        eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702,
+        legacy::TxLegacy,
+    },
+    withdrawal::Withdrawal,
+};
+use reth_db::{
+    models::{AccountBeforeTx, StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals},
+    ClientVersion,
+};
+use reth_fs_util as fs;
+use reth_primitives::{
+    Account, Log, LogData, Receipt, ReceiptWithBloom, StorageEntry, Transaction,
+    TransactionSignedNoHash, TxType, Withdrawals,
+};
+use reth_prune_types::{PruneCheckpoint, PruneMode};
+use reth_stages_types::{
+    AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint,
+    HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, StageUnitCheckpoint,
+    StorageHashingCheckpoint,
+};
+use reth_trie::{hash_builder::HashBuilderValue, TrieMask};
+use reth_trie_common::{hash_builder::HashBuilderState, StoredNibbles, StoredNibblesSubKey};
+use std::{fs::File, io::BufReader};
+
+pub const VECTORS_FOLDER: &str = "testdata/micro/compact";
+pub const VECTOR_SIZE: usize = 100;
+
+#[macro_export]
+macro_rules! compact_types {
+    (regular: [$($regular_ty:ident),*], identifier: [$($id_ty:ident),*]) => {
+        pub const GENERATE_VECTORS: &[fn(&mut TestRunner) -> eyre::Result<()>] = &[
+            $(
+                generate_vector::<$regular_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
+            )*
+            $(
+                generate_vector::<$id_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
+            )*
+        ];
+
+        pub const READ_VECTORS: &[fn() -> eyre::Result<()>] = &[
+            $(
+                read_vector::<$regular_ty> as fn() -> eyre::Result<()>,
+            )*
+            $(
+                read_vector::<$id_ty> as fn() -> eyre::Result<()>,
+            )*
+        ];
+
+        pub static IDENTIFIER_TYPE: std::sync::LazyLock<std::collections::HashSet<String>> = std::sync::LazyLock::new(|| {
+            let mut map = std::collections::HashSet::new();
+            $(
+                map.insert(type_name::<$id_ty>());
+            )*
+            map
+        });
+    };
+}
+
+// The type that **actually** implements `Compact` should go here. If it's an alloy type, import the
+// auxiliary type from reth_codecs::alloy instead.
+compact_types!(
+    regular: [
+        // reth-primitives
+        Account,
+        Receipt,
+        Withdrawals,
+        ReceiptWithBloom,
+        // reth_codecs::alloy
+        Authorization,
+        GenesisAccount,
+        Header,
+        HeaderExt,
+        Withdrawal,
+        TxEip2930,
+        TxEip1559,
+        TxEip4844,
+        TxEip7702,
+        TxLegacy,
+        HashBuilderValue,
+        LogData,
+        Log,
+        // BranchNodeCompact, // todo requires arbitrary
+        TrieMask,
+        // TxDeposit, TODO(joshie): optimism
+        // reth_prune_types
+        PruneCheckpoint,
+        PruneMode,
+        // reth_stages_types
+        AccountHashingCheckpoint,
+        StorageHashingCheckpoint,
+        ExecutionCheckpoint,
+        HeadersCheckpoint,
+        IndexHistoryCheckpoint,
+        EntitiesCheckpoint,
+        CheckpointBlockRange,
+        StageCheckpoint,
+        StageUnitCheckpoint,
+        // reth_db_api
+        StoredBlockOmmers,
+        StoredBlockBodyIndices,
+        StoredBlockWithdrawals,
+        // Manual implementations
+        TransactionSignedNoHash,
+        // Bytecode, // todo revm arbitrary
+        StorageEntry,
+        // MerkleCheckpoint, // todo storedsubnode -> branchnodecompact arbitrary
+        AccountBeforeTx,
+        ClientVersion,
+        StoredNibbles,
+        StoredNibblesSubKey,
+        // StorageTrieEntry, // todo branchnodecompact arbitrary
+        // StoredSubNode, // todo branchnodecompact arbitrary
+        HashBuilderState
+    ],
+    // These types require an extra identifier which is usually stored elsewhere (eg. parent type).
+    identifier: [
+        // Signature todo we for v we only store parity(true || false), while v can take more values
+        Transaction,
+        TxType,
+        TxKind
+    ]
+);
+
+/// Generates a vector of type `T` to a file.
+pub fn generate_vectors() -> Result<()> {
+    generate_vectors_with(GENERATE_VECTORS)
+}
+
+pub fn read_vectors() -> Result<()> {
+    read_vectors_with(READ_VECTORS)
+}
+
+/// Generates a vector of type `T` to a file.
+pub fn generate_vectors_with(gen: &[fn(&mut TestRunner) -> eyre::Result<()>]) -> Result<()> {
+    // Prepare random seed for test (same method as used by proptest)
+    let mut seed = [0u8; 32];
+    getrandom(&mut seed)?;
+    println!("Seed for compact test vectors: {:?}", hex::encode_prefixed(seed));
+
+    // Start the runner with the seed
+    let config = ProptestConfig::default();
+    let rng = TestRng::from_seed(config.rng_algorithm, &seed);
+    let mut runner = TestRunner::new_with_rng(config, rng);
+
+    fs::create_dir_all(VECTORS_FOLDER)?;
+
+    for generate_fn in gen {
+        generate_fn(&mut runner)?;
+    }
+
+    Ok(())
+}
+
+/// Reads multiple vectors of different types ensuring their correctness by decoding and
+/// re-encoding.
+pub fn read_vectors_with(read: &[fn() -> eyre::Result<()>]) -> Result<()> {
+    fs::create_dir_all(VECTORS_FOLDER)?;
+
+    for read_fn in read {
+        read_fn()?;
+    }
+
+    Ok(())
+}
+
+/// Generates test vectors for a specific type `T`.
+pub fn generate_vector<T>(runner: &mut TestRunner) -> Result<()>
+where
+    T: for<'a> Arbitrary<'a> + reth_codecs::Compact,
+{
+    let type_name = type_name::<T>();
+    print!("{}", &type_name);
+
+    let mut bytes = std::iter::repeat(0u8).take(256).collect::<Vec<u8>>();
+    let mut compact_buffer = vec![];
+
+    let mut values = Vec::with_capacity(VECTOR_SIZE);
+    for _ in 0..VECTOR_SIZE {
+        runner.rng().fill_bytes(&mut bytes);
+        compact_buffer.clear();
+
+        let obj = T::arbitrary(&mut arbitrary::Unstructured::new(&bytes))?;
+        let res = obj.to_compact(&mut compact_buffer);
+
+        if IDENTIFIER_TYPE.contains(&type_name) {
+            compact_buffer.push(res as u8);
+        }
+
+        values.push(hex::encode(&compact_buffer));
+    }
+
+    serde_json::to_writer(
+        std::io::BufWriter::new(
+            std::fs::File::create(format!("{VECTORS_FOLDER}/{}.json", &type_name)).unwrap(),
+        ),
+        &values,
+    )?;
+
+    println!(" ✅");
+
+    Ok(())
+}
+
+/// Reads a vector of type `T` from a file and compares each item with its reconstructed version
+/// using `T::from_compact`.
+pub fn read_vector<T>() -> Result<()>
+where
+    T: reth_codecs::Compact,
+{
+    let type_name = type_name::<T>();
+    print!("{}", &type_name);
+
+    // Read the file where the vectors are stored
+    let file_path = format!("{VECTORS_FOLDER}/{}.json", &type_name);
+    let file = File::open(&file_path).wrap_err_with(|| {
+        "Failed to open vector. Make sure to run `reth test-vectors compact --write` first."
+    })?;
+    let reader = BufReader::new(file);
+
+    let stored_values: Vec<String> = serde_json::from_reader(reader)?;
+    let mut buffer = vec![];
+
+    for hex_str in stored_values {
+        let mut compact_bytes = hex::decode(hex_str)?;
+        let mut identifier = None;
+        buffer.clear();
+
+        if IDENTIFIER_TYPE.contains(&type_name) {
+            identifier = compact_bytes.pop().map(|b| b as usize);
+        }
+        let len_or_identifier = identifier.unwrap_or(compact_bytes.len());
+
+        let (reconstructed, _) = T::from_compact(&compact_bytes, len_or_identifier);
+        reconstructed.to_compact(&mut buffer);
+        assert_eq!(buffer, compact_bytes);
+    }
+
+    println!(" ✅");
+
+    Ok(())
+}
+
+pub fn type_name<T>() -> String {
+    std::any::type_name::<T>().replace("::", "__")
+}

crates/cli/commands/src/test_vectors/mod.rs

@@ -2,7 +2,8 @@
 
 use clap::{Parser, Subcommand};
 
-mod tables;
+pub mod compact;
+pub mod tables;
 
 /// Generate test-vectors for different data types.
 #[derive(Debug, Parser)]
@@ -19,6 +20,22 @@ pub enum Subcommands {
         /// List of table names. Case-sensitive.
         names: Vec<String>,
     },
+    /// Randomly generate test vectors for each `Compact` type using the `--write` flag.
+    ///
+    /// The generated vectors are serialized in both `json` and `Compact` formats and saved to a
+    /// file.
+    ///
+    /// Use the `--read` flag to read and validate the previously generated vectors from file.
+    #[group(multiple = false, required = true)]
+    Compact {
+        /// Write test vectors to a file.
+        #[arg(long)]
+        write: bool,
+
+        /// Read test vectors from a file.
+        #[arg(long)]
+        read: bool,
+    },
 }
 
 impl Command {
@@ -28,6 +45,13 @@ impl Command {
             Subcommands::Tables { names } => {
                 tables::generate_vectors(names)?;
             }
+            Subcommands::Compact { write, .. } => {
+                if write {
+                    compact::generate_vectors()?;
+                } else {
+                    compact::read_vectors()?;
+                }
+            }
         }
         Ok(())
     }

crates/cli/commands/src/test_vectors/tables.rs

@@ -1,4 +1,4 @@
-use alloy_primitives::private::getrandom::getrandom;
+use alloy_primitives::{hex, private::getrandom::getrandom};
 use arbitrary::Arbitrary;
 use eyre::Result;
 use proptest::{
@@ -17,11 +17,11 @@ const VECTORS_FOLDER: &str = "testdata/micro/db";
 const PER_TABLE: usize = 1000;
 
 /// Generates test vectors for specified `tables`. If list is empty, then generate for all tables.
-pub(crate) fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
+pub fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
     // Prepare random seed for test (same method as used by proptest)
     let mut seed = [0u8; 32];
     getrandom(&mut seed)?;
-    println!("Seed for test vectors: {:?}", seed);
+    println!("Seed for table test vectors: {:?}", hex::encode_prefixed(seed));
 
     // Start the runner with the seed
     let config = ProptestConfig::default();

crates/engine/tree/Cargo.toml

@@ -95,5 +95,6 @@ test-utils = [
 	"reth-revm/test-utils",
 	"reth-stages-api/test-utils",
 	"reth-provider/test-utils",
-	"reth-trie/test-utils"
+	"reth-trie/test-utils",
+	"reth-prune-types?/test-utils"
 ]

crates/evm/Cargo.toml

@@ -57,11 +57,12 @@ std = [
     "revm/std",
 ]
 test-utils = [
-    "dep:parking_lot",
+	"dep:parking_lot",
-    "reth-chainspec/test-utils",
+	"reth-chainspec/test-utils",
-    "reth-consensus/test-utils",
+	"reth-consensus/test-utils",
-    "reth-primitives/test-utils",
+	"reth-primitives/test-utils",
-    "reth-primitives-traits/test-utils",
+	"reth-primitives-traits/test-utils",
-    "reth-revm/test-utils",
+	"reth-revm/test-utils",
-    "revm/test-utils",
+	"revm/test-utils",
+	"reth-prune-types/test-utils"
 ]

crates/net/eth-wire/Cargo.toml

@@ -71,7 +71,8 @@ arbitrary = [
 	"dep:arbitrary",
 	"reth-chainspec/arbitrary",
 	"alloy-eips/arbitrary",
-	"alloy-primitives/arbitrary"
+	"alloy-primitives/arbitrary",
+	"reth-codecs/arbitrary"
 ]
 serde = [
 	"dep:serde",
@@ -80,7 +81,8 @@ serde = [
 	"alloy-primitives/serde",
 	"bytes/serde",
 	"rand/serde",
-	"secp256k1/serde"
+	"secp256k1/serde",
+	"reth-codecs/serde"
 ]
 
 [[test]]

crates/optimism/bin/Cargo.toml

@@ -47,6 +47,10 @@ optimism = [
 	"reth-provider/optimism"
 ]
 
+dev = [
+	"reth-optimism-cli/dev"
+]
+
 min-error-logs = ["tracing/release_max_level_error"]
 min-warn-logs = ["tracing/release_max_level_warn"]
 min-info-logs = ["tracing/release_max_level_info"]

crates/optimism/cli/Cargo.toml

@@ -65,6 +65,13 @@ tokio-util = { workspace = true, features = ["codec"] }
 tracing.workspace = true
 eyre.workspace = true
 
+# reth test-vectors
+proptest = { workspace = true, optional = true }
+op-alloy-consensus = { workspace = true, features = [
+    "arbitrary",
+], optional = true }
+
+
 [dev-dependencies]
 tempfile.workspace = true
 reth-stages = { workspace = true, features = ["test-utils"] }
@@ -94,3 +101,9 @@ jemalloc = [
     "reth-node-core/jemalloc",
     "reth-node-metrics/jemalloc"
 ]
+
+dev = [
+    "dep:proptest",
+    "reth-cli-commands/arbitrary",
+    "op-alloy-consensus"
+]

crates/optimism/cli/src/commands/mod.rs

@@ -16,6 +16,9 @@ pub mod import;
 pub mod import_receipts;
 pub mod init_state;
 
+#[cfg(feature = "dev")]
+pub mod test_vectors;
+
 /// Commands to be executed
 #[derive(Debug, Subcommand)]
 pub enum Commands<Spec: ChainSpecParser = OpChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs>
@@ -55,4 +58,8 @@ pub enum Commands<Spec: ChainSpecParser = OpChainSpecParser, Ext: clap::Args + f
     /// Prune according to the configuration without any limits
     #[command(name = "prune")]
     Prune(prune::PruneCommand<Spec>),
+    /// Generate Test Vectors
+    #[cfg(feature = "dev")]
+    #[command(name = "test-vectors")]
+    TestVectors(test_vectors::Command),
 }

crates/optimism/cli/src/commands/test_vectors.rs

@@ -0,0 +1,72 @@
+//! Command for generating test vectors.
+
+use clap::{Parser, Subcommand};
+use op_alloy_consensus::TxDeposit;
+use proptest::test_runner::TestRunner;
+use reth_cli_commands::{
+    compact_types,
+    test_vectors::{
+        compact,
+        compact::{
+            generate_vector, read_vector, GENERATE_VECTORS as ETH_GENERATE_VECTORS,
+            READ_VECTORS as ETH_READ_VECTORS,
+        },
+        tables,
+    },
+};
+
+/// Generate test-vectors for different data types.
+#[derive(Debug, Parser)]
+pub struct Command {
+    #[command(subcommand)]
+    command: Subcommands,
+}
+
+#[derive(Subcommand, Debug)]
+/// `reth test-vectors` subcommands
+pub enum Subcommands {
+    /// Generates test vectors for specified tables. If no table is specified, generate for all.
+    Tables {
+        /// List of table names. Case-sensitive.
+        names: Vec<String>,
+    },
+    /// Generates test vectors for `Compact` types with `--write`. Reads and checks generated
+    /// vectors with `--read`.
+    #[group(multiple = false, required = true)]
+    Compact {
+        /// Write test vectors to a file.
+        #[arg(long)]
+        write: bool,
+
+        /// Read test vectors from a file.
+        #[arg(long)]
+        read: bool,
+    },
+}
+
+impl Command {
+    /// Execute the command
+    pub async fn execute(self) -> eyre::Result<()> {
+        match self.command {
+            Subcommands::Tables { names } => {
+                tables::generate_vectors(names)?;
+            }
+            Subcommands::Compact { write, .. } => {
+                compact_types!(
+                    regular: [
+                        TxDeposit
+                    ], identifier: []
+                );
+
+                if write {
+                    compact::generate_vectors_with(ETH_GENERATE_VECTORS)?;
+                    compact::generate_vectors_with(GENERATE_VECTORS)?;
+                } else {
+                    compact::read_vectors_with(ETH_READ_VECTORS)?;
+                    compact::read_vectors_with(READ_VECTORS)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}

crates/optimism/cli/src/lib.rs

@@ -169,6 +169,8 @@ where
                 runner.run_command_until_exit(|ctx| command.execute::<OptimismNode>(ctx))
             }
             Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<OptimismNode>()),
+            #[cfg(feature = "dev")]
+            Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()),
         }
     }
 

crates/primitives-traits/Cargo.toml

@@ -74,7 +74,8 @@ arbitrary = [
 	"dep:proptest",
 	"dep:proptest-arbitrary-interop",
 	"alloy-eips/arbitrary",
-	"revm-primitives/arbitrary"
+	"revm-primitives/arbitrary",
+	"reth-codecs/arbitrary"
 ]
 serde-bincode-compat = [
 	"serde_with",

crates/primitives/Cargo.toml

@@ -62,7 +62,7 @@ arbitrary = { workspace = true, features = ["derive"], optional = true }
 [dev-dependencies]
 # eth
 reth-chainspec.workspace = true
-reth-codecs.workspace = true
+reth-codecs = { workspace = true, features = ["test-utils"] }
 reth-primitives-traits = { workspace = true, features = ["arbitrary"] }
 reth-testing-utils.workspace = true
 revm-primitives = { workspace = true, features = ["arbitrary"] }
@@ -120,6 +120,7 @@ arbitrary = [
 	"alloy-serde?/arbitrary",
 	"op-alloy-consensus?/arbitrary",
 	"op-alloy-rpc-types?/arbitrary",
+	"reth-codecs?/arbitrary"
 ]
 secp256k1 = ["dep:secp256k1"]
 c-kzg = [

crates/primitives/src/receipt.rs

@@ -503,6 +503,18 @@ mod tests {
     use super::*;
     use alloy_primitives::{address, b256, bytes, hex_literal::hex};
 
+    #[test]
+    fn test_decode_receipt() {
+        #[cfg(not(feature = "optimism"))]
+        reth_codecs::test_utils::test_decode::<Receipt>(&hex!(
+            "c428b52ffd23fc42696156b10200f034792b6a94c3850215c2fef7aea361a0c31b79d9a32652eefc0d4e2e730036061cff7344b6fc6132b50cda0ed810a991ae58ef013150c12b2522533cb3b3a8b19b7786a8b5ff1d3cdc84225e22b02def168c8858df"
+        ));
+        #[cfg(feature = "optimism")]
+        reth_codecs::test_utils::test_decode::<Receipt>(&hex!(
+            "c30328b52ffd23fc426961a00105007eb0042307705a97e503562eacf2b95060cce9de6de68386b6c155b73a9650021a49e2f8baad17f30faff5899d785c4c0873e45bc268bcf07560106424570d11f9a59e8f3db1efa4ceec680123712275f10d92c3411e1caaa11c7c5d591bc11487168e09934a9986848136da1b583babf3a7188e3aed007a1520f1cf4c1ca7d3482c6c28d37c298613c70a76940008816c4c95644579fd08471dc34732fd0f24"
+        ));
+    }
+
     // Test vector from: https://eips.ethereum.org/EIPS/eip-2481
     #[test]
     fn encode_legacy_receipt() {

crates/primitives/src/transaction/mod.rs

@@ -649,10 +649,12 @@ impl reth_codecs::Compact for Transaction {
                         let (tx, buf) = TxDeposit::from_compact(buf, buf.len());
                         (Self::Deposit(tx), buf)
                     }
-                    _ => unreachable!("Junk data in database: unknown Transaction variant"),
+                    _ => unreachable!(
+                        "Junk data in database: unknown Transaction variant: {identifier}"
+                    ),
                 }
             }
-            _ => unreachable!("Junk data in database: unknown Transaction variant"),
+            _ => unreachable!("Junk data in database: unknown Transaction variant: {identifier}"),
         }
     }
 }

crates/prune/types/Cargo.toml

@@ -20,6 +20,7 @@ derive_more.workspace = true
 modular-bitfield.workspace = true
 serde.workspace = true
 thiserror.workspace = true
+arbitrary = { workspace = true, features = ["derive"], optional = true }
 
 [dev-dependencies]
 arbitrary = { workspace = true, features = ["derive"] }
@@ -29,3 +30,13 @@ proptest-arbitrary-interop.workspace = true
 serde_json.workspace = true
 test-fuzz.workspace = true
 toml.workspace = true
+
+[features]
+test-utils = [
+	"dep:arbitrary",
+	"reth-codecs/test-utils"
+]
+arbitrary = [
+	"alloy-primitives/arbitrary",
+	"reth-codecs/arbitrary"
+]

crates/prune/types/src/checkpoint.rs

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 
 /// Saves the pruning progress of a stage.
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(Default, arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(Default, arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct PruneCheckpoint {
     /// Highest pruned block number. If it's [None], the pruning for block `0` is not finished yet.

crates/prune/types/src/mode.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 /// Prune mode.
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)]
 #[serde(rename_all = "lowercase")]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub enum PruneMode {
     /// Prune all blocks.
@@ -17,6 +17,13 @@ pub enum PruneMode {
     Before(BlockNumber),
 }
 
+#[cfg(any(test, feature = "test-utils"))]
+impl Default for PruneMode {
+    fn default() -> Self {
+        Self::Full
+    }
+}
+
 impl PruneMode {
     /// Prune blocks up to the specified block number. The specified block number is also pruned.
     ///
@@ -69,13 +76,6 @@ impl PruneMode {
     }
 }
 
-#[cfg(test)]
-impl Default for PruneMode {
-    fn default() -> Self {
-        Self::Full
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use crate::{

crates/revm/Cargo.toml

@@ -43,10 +43,11 @@ std = [
     "alloy-consensus/std",
 ]
 test-utils = [
-    "dep:reth-trie",
+	"dep:reth-trie",
-    "reth-primitives/test-utils",
+	"reth-primitives/test-utils",
-    "reth-trie?/test-utils",
+	"reth-trie?/test-utils",
-    "revm/test-utils",
+	"revm/test-utils",
+	"reth-prune-types/test-utils"
 ]
 serde = [
     "revm/serde",

crates/stages/api/Cargo.toml

@@ -50,5 +50,6 @@ test-utils = [
 	"reth-consensus/test-utils",
 	"reth-network-p2p/test-utils",
 	"reth-primitives-traits/test-utils",
-	"reth-provider/test-utils"
+	"reth-provider/test-utils",
+	"reth-stages-types/test-utils"
 ]

crates/stages/stages/Cargo.toml

@@ -110,7 +110,8 @@ test-utils = [
 	"reth-codecs/test-utils",
 	"reth-db-api/test-utils",
 	"reth-trie-db/test-utils",
-	"reth-trie/test-utils"
+	"reth-trie/test-utils",
+	"reth-prune-types/test-utils"
 ]
 
 [[bench]]

crates/stages/types/Cargo.toml

@@ -19,6 +19,7 @@ alloy-primitives.workspace = true
 modular-bitfield.workspace = true
 bytes.workspace = true
 serde.workspace = true
+arbitrary = { workspace = true, features = ["derive"], optional = true }
 
 [dev-dependencies]
 arbitrary = { workspace = true, features = ["derive"] }
@@ -26,3 +27,15 @@ proptest.workspace = true
 proptest-arbitrary-interop.workspace = true
 test-fuzz.workspace = true
 rand.workspace = true
+
+[features]
+test-utils = [
+	"dep:arbitrary",
+	"reth-codecs/test-utils",
+	"reth-trie-common/test-utils"
+]
+arbitrary = [
+	"alloy-primitives/arbitrary",
+	"reth-codecs/arbitrary",
+	"reth-trie-common/arbitrary"
+]

crates/stages/types/src/checkpoints.rs

@@ -76,7 +76,7 @@ impl Compact for MerkleCheckpoint {
 
 /// Saves the progress of AccountHashing stage.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct AccountHashingCheckpoint {
     /// The next account to start hashing from.
@@ -89,7 +89,7 @@ pub struct AccountHashingCheckpoint {
 
 /// Saves the progress of StorageHashing stage.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct StorageHashingCheckpoint {
     /// The next account to start hashing from.
@@ -104,7 +104,7 @@ pub struct StorageHashingCheckpoint {
 
 /// Saves the progress of Execution stage.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct ExecutionCheckpoint {
     /// Block range which this checkpoint is valid for.
@@ -115,7 +115,7 @@ pub struct ExecutionCheckpoint {
 
 /// Saves the progress of Headers stage.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct HeadersCheckpoint {
     /// Block range which this checkpoint is valid for.
@@ -126,7 +126,7 @@ pub struct HeadersCheckpoint {
 
 /// Saves the progress of Index History stages.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct IndexHistoryCheckpoint {
     /// Block range which this checkpoint is valid for.
@@ -137,7 +137,7 @@ pub struct IndexHistoryCheckpoint {
 
 /// Saves the progress of abstract stage iterating over or downloading entities.
 #[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct EntitiesCheckpoint {
     /// Number of entities already processed.
@@ -166,7 +166,7 @@ impl EntitiesCheckpoint {
 /// Saves the block range. Usually, it's used to check the validity of some stage checkpoint across
 /// multiple executions.
 #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct CheckpointBlockRange {
     /// The first block of the range, inclusive.
@@ -189,7 +189,7 @@ impl From<&RangeInclusive<BlockNumber>> for CheckpointBlockRange {
 
 /// Saves the progress of a stage.
 #[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub struct StageCheckpoint {
     /// The maximum block processed by the stage.
@@ -256,7 +256,7 @@ impl StageCheckpoint {
 //  is not a Copy type.
 /// Stage-specific checkpoint metrics.
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
 #[add_arbitrary_tests(compact)]
 pub enum StageUnitCheckpoint {
     /// Saves the progress of AccountHashing stage.

crates/storage/codecs/Cargo.toml

@@ -27,6 +27,9 @@ op-alloy-consensus = { workspace = true, optional = true }
 # misc
 bytes.workspace = true
 modular-bitfield = { workspace = true, optional = true }
+visibility = { version = "0.1.1", optional = true}
+serde.workspace = true
+arbitrary = { workspace = true, features = ["derive"], optional = true }
 
 [dev-dependencies]
 alloy-eips = { workspace = true, default-features = false, features = [
@@ -45,7 +48,6 @@ serde_json.workspace = true
 arbitrary = { workspace = true, features = ["derive"] }
 proptest.workspace = true
 proptest-arbitrary-interop.workspace = true
-serde.workspace = true
 
 [features]
 default = ["std", "alloy"]
@@ -66,4 +68,25 @@ alloy = [
     "dep:alloy-trie",
 ]
 optimism = ["alloy", "dep:op-alloy-consensus"]
-test-utils = []
+test-utils = [
+	"std",
+    "alloy",
+	"arbitrary",
+    "dep:visibility",
+    "dep:arbitrary"
+]
+serde = [
+	"alloy-consensus?/serde",
+	"alloy-eips?/serde",
+	"alloy-primitives/serde",
+	"alloy-trie?/serde",
+	"bytes/serde",
+	"op-alloy-consensus?/serde"
+]
+arbitrary = [
+	"alloy-consensus?/arbitrary",
+	"alloy-eips?/arbitrary",
+	"alloy-primitives/arbitrary",
+	"alloy-trie?/arbitrary",
+	"op-alloy-consensus?/arbitrary"
+]

crates/storage/codecs/src/alloy/access_list.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AccessList`]
+
 use crate::Compact;
 use alloc::vec::Vec;
 use alloy_eips::eip2930::{AccessList, AccessListItem};

crates/storage/codecs/src/alloy/authorization_list.rs

@@ -1,16 +1,21 @@
-use core::ops::Deref;
+//! Compact implementation for [`AlloyAuthorization`]
 
 use crate::Compact;
 use alloy_eips::eip7702::{Authorization as AlloyAuthorization, SignedAuthorization};
 use alloy_primitives::{Address, U256};
 use bytes::Buf;
+use core::ops::Deref;
 use reth_codecs_derive::add_arbitrary_tests;
 
 /// Authorization acts as bridge which simplifies Compact implementation for AlloyAuthorization.
 ///
 /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip7702::Authorization`
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct Authorization {
     chain_id: u64,

crates/storage/codecs/src/alloy/genesis_account.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyGenesisAccount`]
+
 use crate::Compact;
 use alloc::vec::Vec;
 use alloy_genesis::GenesisAccount as AlloyGenesisAccount;
@@ -22,8 +24,14 @@ pub(crate) struct GenesisAccountRef<'a> {
     private_key: Option<&'a B256>,
 }
 
+/// Acts as bridge which simplifies Compact implementation for
+/// `AlloyGenesisAccount`.
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct GenesisAccount {
     /// The nonce of the account at genesis.
@@ -39,14 +47,20 @@ pub(crate) struct GenesisAccount {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct StorageEntries {
     entries: Vec<StorageEntry>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct StorageEntry {
     key: B256,

crates/storage/codecs/src/alloy/header.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyHeader`]
+
 use crate::Compact;
 use alloy_consensus::Header as AlloyHeader;
 use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256};
@@ -10,7 +12,11 @@ use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256};
 /// will automatically apply to this type.
 ///
 /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::Header`]
-#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(serde::Serialize, serde::Deserialize, arbitrary::Arbitrary)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
 pub(crate) struct Header {
     parent_hash: B256,
@@ -42,7 +48,11 @@ pub(crate) struct Header {
 /// used as a field of [`Header`] for backwards compatibility.
 ///
 /// More information: <https://github.com/paradigmxyz/reth/issues/7820> & [`reth_codecs_derive::Compact`].
-#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(serde::Serialize, serde::Deserialize, arbitrary::Arbitrary)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
 pub(crate) struct HeaderExt {
     requests_hash: Option<B256>,

crates/storage/codecs/src/alloy/mod.rs

@@ -1,13 +1,29 @@
-mod access_list;
+//! Implements Compact for alloy types.
-mod authorization_list;
+
-mod genesis_account;
+/// Will make it a pub mod if test-utils is enabled
-mod header;
+macro_rules! cond_mod {
-mod log;
+    ($($mod_name:ident),*) => {
-mod signature;
+        $(
-mod transaction;
+            #[cfg(feature = "test-utils")]
-mod trie;
+            pub mod $mod_name;
-mod txkind;
+            #[cfg(not(feature = "test-utils"))]
-mod withdrawal;
+            mod $mod_name;
+        )*
+    };
+}
+
+cond_mod!(
+    access_list,
+    authorization_list,
+    genesis_account,
+    header,
+    log,
+    signature,
+    transaction,
+    trie,
+    txkind,
+    withdrawal
+);
 
 #[cfg(test)]
 mod tests {

crates/storage/codecs/src/alloy/signature.rs

@@ -1,6 +1,7 @@
-use alloy_primitives::{Parity, Signature, U256};
+//! Compact implementation for [`Signature`]
 
 use crate::Compact;
+use alloy_primitives::{Parity, Signature, U256};
 
 impl Compact for Signature {
     fn to_compact<B>(&self, buf: &mut B) -> usize

crates/storage/codecs/src/alloy/transaction/eip1559.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyTxEip1559`]
+
 use crate::Compact;
 use alloy_consensus::TxEip1559 as AlloyTxEip1559;
 use alloy_eips::eip2930::AccessList;
@@ -11,8 +13,12 @@ use alloy_primitives::{Bytes, ChainId, TxKind, U256};
 ///
 /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip1559`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Compact, Default)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
-#[cfg_attr(test, crate::add_arbitrary_tests(compact))]
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(any(test, feature = "test-utils"), crate::add_arbitrary_tests(compact))]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 pub(crate) struct TxEip1559 {
     chain_id: ChainId,
     nonce: u64,

crates/storage/codecs/src/alloy/transaction/eip2930.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyTxEip2930`]
+
 use crate::Compact;
 use alloy_consensus::TxEip2930 as AlloyTxEip2930;
 use alloy_eips::eip2930::AccessList;
@@ -13,7 +15,11 @@ use reth_codecs_derive::add_arbitrary_tests;
 ///
 /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip2930`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct TxEip2930 {
     chain_id: ChainId,

crates/storage/codecs/src/alloy/transaction/eip4844.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyTxEip4844`]
+
 use crate::{Compact, CompactPlaceholder};
 use alloc::vec::Vec;
 use alloy_consensus::TxEip4844 as AlloyTxEip4844;
@@ -14,7 +16,8 @@ use reth_codecs_derive::add_arbitrary_tests;
 ///
 /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip4844`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(any(test, feature = "test-utils"), derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct TxEip4844 {
     chain_id: ChainId,
@@ -25,6 +28,13 @@ pub(crate) struct TxEip4844 {
     /// TODO(debt): this should be removed if we break the DB.
     /// Makes sure that the Compact bitflag struct has one bit after the above field:
     /// <https://github.com/paradigmxyz/reth/pull/8291#issuecomment-2117545016>
+    #[cfg_attr(
+        feature = "test-utils",
+        serde(
+            serialize_with = "serialize_placeholder",
+            deserialize_with = "deserialize_placeholder"
+        )
+    )]
     placeholder: Option<CompactPlaceholder>,
     to: Address,
     value: U256,
@@ -75,6 +85,54 @@ impl Compact for AlloyTxEip4844 {
     }
 }
 
+#[cfg(any(test, feature = "test-utils"))]
+impl<'a> arbitrary::Arbitrary<'a> for TxEip4844 {
+    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
+        Ok(Self {
+            chain_id: ChainId::arbitrary(u)?,
+            nonce: u64::arbitrary(u)?,
+            gas_limit: u64::arbitrary(u)?,
+            max_fee_per_gas: u128::arbitrary(u)?,
+            max_priority_fee_per_gas: u128::arbitrary(u)?,
+            // Should always be Some for TxEip4844
+            placeholder: Some(()),
+            to: Address::arbitrary(u)?,
+            value: U256::arbitrary(u)?,
+            access_list: AccessList::arbitrary(u)?,
+            blob_versioned_hashes: Vec::<B256>::arbitrary(u)?,
+            max_fee_per_blob_gas: u128::arbitrary(u)?,
+            input: Bytes::arbitrary(u)?,
+        })
+    }
+}
+
+#[cfg(any(test, feature = "test-utils"))]
+fn serialize_placeholder<S>(value: &Option<()>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    // Required otherwise `serde_json` will serialize it as null and would be `None` when decoding
+    // it again.
+    match value {
+        Some(()) => serializer.serialize_str("placeholder"), // Custom serialization
+        None => serializer.serialize_none(),
+    }
+}
+
+#[cfg(any(test, feature = "test-utils"))]
+fn deserialize_placeholder<'de, D>(deserializer: D) -> Result<Option<()>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    use serde::de::Deserialize;
+    let s: Option<String> = Option::deserialize(deserializer)?;
+    match s.as_deref() {
+        Some("placeholder") => Ok(Some(())),
+        None => Ok(None),
+        _ => Err(serde::de::Error::custom("unexpected value")),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/storage/codecs/src/alloy/transaction/eip7702.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyTxEip7702`]
+
 use crate::Compact;
 use alloc::vec::Vec;
 use alloy_consensus::TxEip7702 as AlloyTxEip7702;
@@ -14,7 +16,11 @@ use reth_codecs_derive::add_arbitrary_tests;
 ///
 /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip7702`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct TxEip7702 {
     chain_id: ChainId,

crates/storage/codecs/src/alloy/transaction/legacy.rs

@@ -1,11 +1,17 @@
+//! Compact implementation for [`AlloyTxLegacy`]
+
 use crate::Compact;
 use alloy_consensus::TxLegacy as AlloyTxLegacy;
 use alloy_primitives::{Bytes, ChainId, TxKind, U256};
 
 /// Legacy transaction.
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
-#[cfg_attr(test, crate::add_arbitrary_tests(compact))]
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize),
+    crate::add_arbitrary_tests(compact)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 pub(crate) struct TxLegacy {
     /// Added as EIP-155: Simple replay attack protection
     chain_id: Option<ChainId>,

crates/storage/codecs/src/alloy/transaction/mod.rs

@@ -1,10 +1,18 @@
-pub(crate) mod eip1559;
+//! Compact implementation for transaction types
-pub(crate) mod eip2930;
+
-pub(crate) mod eip4844;
+cond_mod!(
-pub(crate) mod eip7702;
+    eip1559,
-pub(crate) mod legacy;
+    eip2930,
-#[cfg(feature = "optimism")]
+    eip4844,
-pub(crate) mod optimism;
+    eip7702,
+    legacy
+);
+
+
+#[cfg(all(feature = "test-utils", feature = "optimism"))]
+pub mod optimism;
+#[cfg(all(not(feature = "test-utils"), feature = "optimism"))]
+mod optimism;
 
 #[cfg(test)]
 mod tests {
@@ -15,9 +23,13 @@ mod tests {
     // this check is to ensure we do not inadvertently add too many fields to a struct which would
     // expand the flags field and break backwards compatibility
 
-    use crate::alloy::transaction::{
+    use alloy_primitives::hex;
-        eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702,
+    use crate::{
-        legacy::TxLegacy,
+        alloy::{header::Header, transaction::{
+            eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702,
+            legacy::TxLegacy,
+        }},
+        test_utils::test_decode,
     };
 
     #[test]
@@ -34,4 +46,54 @@ mod tests {
     fn test_ensure_backwards_compatibility_optimism() {
         assert_eq!(crate::alloy::transaction::optimism::TxDeposit::bitflag_encoded_bytes(), 2);
     }
+
+    #[test]
+    fn test_decode_header() {
+        test_decode::<Header>(&hex!(
+            "01000000fbbb564baeafd064b979c2ac032df5cd987098066a8c6969514dfb8ecfbf043e667fa19efcc00d1dd197c309a3cc42dec820cd627af8f7f38f3274f842406891b22624431d0ea858422db8415b1181f8d19befbd21287debaf98a94e84b3ec20be846f35abfbf743ee3eda4fdda6a6f9124d295da97e26eaa1cedd09936f0a3c560b6bc10316dba5e82abd21afcf519a985feb09a6ce7fba2e8163b10f06c99828b8049c29b993d88d1d112dca60a03ebd8ebc6d69a7e1f301ca6d67c21fe0949d67bca251edf36c96a2cf7c84d98fc60a53988ac95820f434eb35280d98c8ba4d7484e7ee8fefd63591ad4c937ccaaea23871d05c77bac754c5759b34cf9b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+        ));
+    }
+
+    #[test]
+    fn test_decode_eip1559() {
+        test_decode::<TxEip1559>(&hex!(
+            "88086110b81b05bc5bb59ec3e4cd44e895a9dcb2656d5003e2f64ecb2e15443898cc1cc19af19ca96fc2b4eafc4abc26e4bbd70a3ddb10b7530b65eea128f4095c97164f712c04239902c1b08acf3949d4687123cdd72d5c73df113d2dc6ed7e519f410ace5553ca805975240a208b57013532de78c5cb407423ea11921ab11b13e93ef35d4d01c9a23166c4d627987545fe4675528d0ab111b0a1dc83fba0a4e1cd5c826a94db3f"
+        ));
+    }
+
+    #[test]
+    fn test_decode_eip2930() {
+        test_decode::<TxEip2930>(&hex!(
+            "7810833fce14e3e2921e94fd3727eb71e91551d2c1e029697a654bfab510f3963aa57074015e152065d1c807f8830079fb0aeadc251d248eaec7147e78580ed638c4e667827775e24270edd5aad475776533ece65373afa71722bfeba3c900"
+        ));
+    }
+
+    #[test]
+    fn test_decode_eip4844() {
+        test_decode::<TxEip4844>(&hex!(
+            "88086110025c359180ea680b5007c856f9e1ad4d1be7a5019feb42133f4fc4bdf74da1b457ab787462385a28a1bf8edb401adabf3ff21ac18f695e30180348ea67246fc4dc25e88add12b7c317651a0ce08946d98dbbe5b38883aa758a0f247e23b0fe3ac1bcc43d7212c984d6ccc770d70135890c9a07d715cacb9032c90d539d0b3d209a8d600178bcfb416fd489e5d5dd56d9cfc6addae810ae70bdaee65672b871dc2b3f35ec00dbaa0d872f78cb58b3199984c608c8ba"
+        ));
+    }
+
+    #[test]
+    fn test_decode_eip7702() {
+        test_decode::<TxEip7702>(&hex!(
+            "8808210881415c034feba383d7a6efd3f2601309b33a6d682ad47168cac0f7a5c5136a33370e5e7ca7f570d5530d7a0d18bf5eac33583fdc27b6580f61e8cbd34d6de596f925c1f353188feb2c1e9e20de82a80b57f0be425d8c5896280d4f5f66cdcfba256d0c9ac8abd833859a62ec019501b4585fa176f048de4f88b93bdefecfcaf4d8f0dd04767bc683a4569c893632e44ba9d53f90d758125c9b24c0192a649166520cd5eecbc110b53eda400cf184b8ef9932c81d0deb2ea27dfa863392a87bfd53af3ec67379f20992501e76e387cbe3933861beead1b49649383cf8b2a2d5c6d04b7edc376981ed9b12cf7199fe7fabf5198659e001bed40922969b82a6cd000000000000"
+        ));
+    }
+
+    #[test]
+    fn test_decode_legacy() {
+        test_decode::<TxLegacy>(&hex!(
+            "112210080a8ba06a8d108540bb3140e9f71a0812c46226f9ea77ae880d98d19fe27e5911801175c3b32620b2e887af0296af343526e439b775ee3b1c06750058e9e5fc4cd5965c3010f86184"
+        ));
+    }
+
+    #[cfg(feature = "optimism")]
+    #[test]
+    fn test_decode_deposit() {
+        test_decode::<op_alloy_consensus::TxDeposit>(&hex!(
+            "8108ac8f15983d59b6ae4911a00ff7bfcd2e53d2950926f8c82c12afad02861c46fcb293e776204052725e1c08ff2e9ff602ca916357601fa972a14094891fe3598b718758f22c46f163c18bcaa6296ce87e5267ef3fd932112842fbbf79011548cdf067d93ce6098dfc0aaf5a94531e439f30d6dfd0c6"
+        )); 
+    }
 }

crates/storage/codecs/src/alloy/transaction/optimism.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyTxDeposit`]
+
 use crate::Compact;
 use alloy_primitives::{Address, Bytes, TxKind, B256, U256};
 use op_alloy_consensus::TxDeposit as AlloyTxDeposit;
@@ -12,7 +14,11 @@ use reth_codecs_derive::add_arbitrary_tests;
 ///
 /// Notice: Make sure this struct is 1:1 with [`op_alloy_consensus::TxDeposit`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct TxDeposit {
     source_hash: B256,

crates/storage/codecs/src/alloy/withdrawal.rs

@@ -1,3 +1,5 @@
+//! Compact implementation for [`AlloyWithdrawal`]
+
 use crate::Compact;
 use alloy_eips::eip4895::Withdrawal as AlloyWithdrawal;
 use alloy_primitives::Address;
@@ -7,7 +9,11 @@ use reth_codecs_derive::add_arbitrary_tests;
 ///
 /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip4895::Withdrawal`
 #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)]
-#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))]
+#[cfg_attr(
+    any(test, feature = "test-utils"),
+    derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize)
+)]
+#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))]
 #[add_arbitrary_tests(compact)]
 pub(crate) struct Withdrawal {
     /// Monotonically increasing identifier issued by consensus layer.

crates/storage/codecs/src/lib.rs

@@ -18,6 +18,7 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 
 pub use reth_codecs_derive::*;
+use serde as _;
 
 use alloy_primitives::{Address, Bloom, Bytes, FixedBytes, U256};
 use bytes::{Buf, BufMut};
@@ -25,6 +26,10 @@ use bytes::{Buf, BufMut};
 extern crate alloc;
 use alloc::vec::Vec;
 
+#[cfg(feature = "test-utils")]
+pub mod alloy;
+
+#[cfg(not(feature = "test-utils"))]
 #[cfg(any(test, feature = "alloy"))]
 mod alloy;
 

crates/storage/codecs/src/test_utils.rs

@@ -79,3 +79,12 @@ impl UnusedBits {
         matches!(self, Self::NotZero)
     }
 }
+
+/// Tests decoding and re-encoding to ensure correctness.
+pub fn test_decode<T: crate::Compact>(buf: &[u8]) {
+    let (decoded, _) = T::from_compact(buf, buf.len());
+    let mut encoded = Vec::with_capacity(buf.len());
+
+    decoded.to_compact(&mut encoded);
+    assert_eq!(buf, &encoded[..]);
+}

crates/storage/db-api/Cargo.toml

@@ -62,7 +62,9 @@ test-utils = [
 	"reth-primitives-traits/test-utils",
 	"reth-codecs/test-utils",
 	"reth-db-models/test-utils",
-	"reth-trie-common/test-utils"
+	"reth-trie-common/test-utils",
+	"reth-prune-types/test-utils",
+	"reth-stages-types/test-utils"
 ]
 arbitrary = [
 	"reth-primitives/arbitrary",
@@ -72,7 +74,10 @@ arbitrary = [
 	"reth-primitives-traits/arbitrary",
 	"reth-trie-common/arbitrary",
 	"alloy-primitives/arbitrary",
-	"parity-scale-codec/arbitrary"
+	"parity-scale-codec/arbitrary",
+	"reth-codecs/arbitrary",
+	"reth-prune-types/arbitrary",
+	"reth-stages-types/arbitrary"
 ]
 optimism = [
 	"reth-primitives/optimism",

crates/storage/db-models/Cargo.toml

@@ -48,5 +48,6 @@ arbitrary = [
 	"reth-primitives/arbitrary",
 	"dep:arbitrary",
 	"dep:proptest",
-	"alloy-primitives/arbitrary"
+	"alloy-primitives/arbitrary",
+	"reth-codecs/arbitrary"
 ]

crates/storage/db-models/src/accounts.rs

@@ -8,7 +8,7 @@ use reth_primitives::Account;
 ///
 /// [`Address`] is the subkey.
 #[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)]
-#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
+#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary, serde::Deserialize))]
 #[add_arbitrary_tests(compact)]
 pub struct AccountBeforeTx {
     /// Address for the account. Acts as `DupSort::SubKey`.

crates/storage/db/Cargo.toml

@@ -97,7 +97,9 @@ test-utils = [
 	"reth-primitives-traits/test-utils",
 	"reth-db-api/test-utils",
 	"reth-nippy-jar/test-utils",
-	"reth-trie-common/test-utils"
+	"reth-trie-common/test-utils",
+	"reth-prune-types/test-utils",
+	"reth-stages-types/test-utils"
 ]
 bench = []
 arbitrary = [
@@ -105,7 +107,9 @@ arbitrary = [
 	"reth-db-api/arbitrary",
 	"reth-primitives-traits/arbitrary",
 	"reth-trie-common/arbitrary",
-	"alloy-primitives/arbitrary"
+	"alloy-primitives/arbitrary",
+	"reth-prune-types/arbitrary",
+	"reth-stages-types/arbitrary"
 ]
 optimism = [
 	"reth-primitives/optimism",

crates/storage/provider/Cargo.toml

@@ -108,7 +108,8 @@ serde = [
 	"notify/serde",
 	"parking_lot/serde",
 	"rand/serde",
-	"revm/serde"
+	"revm/serde",
+	"reth-codecs/serde"
 ]
 test-utils = [
 	"reth-db/test-utils",
@@ -124,5 +125,7 @@ test-utils = [
 	"reth-codecs/test-utils",
 	"reth-db-api/test-utils",
 	"reth-trie-db/test-utils",
-	"revm/test-utils"
+	"revm/test-utils",
+	"reth-prune-types/test-utils",
+	"reth-stages-types/test-utils"
 ]

crates/trie/common/Cargo.toml

@@ -55,5 +55,6 @@ arbitrary = [
 	"alloy-consensus/arbitrary",
 	"alloy-primitives/arbitrary",
 	"nybbles/arbitrary",
-	"revm-primitives/arbitrary"
+	"revm-primitives/arbitrary",
+	"reth-codecs/arbitrary"
 ]

crates/trie/common/src/nibbles.rs

@@ -19,6 +19,7 @@ pub use nybbles::Nibbles;
     Deserialize,
     derive_more::Index,
 )]
+#[cfg_attr(feature = "test-utils", derive(arbitrary::Arbitrary))]
 pub struct StoredNibbles(pub Nibbles);
 
 impl From<Nibbles> for StoredNibbles {
@@ -74,6 +75,7 @@ impl Compact for StoredNibbles {
 
 /// The representation of nibbles of the merkle trie stored in the database.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord, Hash, Deref)]
+#[cfg_attr(feature = "test-utils", derive(arbitrary::Arbitrary))]
 pub struct StoredNibblesSubKey(pub Nibbles);
 
 impl From<Nibbles> for StoredNibblesSubKey {

crates/trie/trie/Cargo.toml

@@ -79,7 +79,8 @@ test-utils = [
 	"triehash",
 	"reth-trie-common/test-utils",
 	"reth-primitives/test-utils",
-	"revm/test-utils"
+	"revm/test-utils",
+	"reth-stages-types/test-utils"
 ]
 
 [[bench]]