feat: blob e2e test (#7823)

crates/e2e-test-utils/Cargo.toml

@@ -33,5 +33,5 @@ alloy-signer.workspace = true
 alloy-signer-wallet = { workspace = true, features = ["mnemonic"] }
 alloy-rpc-types.workspace = true
 alloy-network.workspace = true
-alloy-consensus.workspace = true
+alloy-consensus = { workspace = true, features = ["kzg"] }
 tracing.workspace = true

crates/e2e-test-utils/src/engine_api.rs

@@ -13,13 +13,13 @@ use reth_primitives::B256;
 use std::marker::PhantomData;
 
 /// Helper for engine api operations
-pub struct EngineApiHelper<E> {
+pub struct EngineApiTestContext<E> {
     pub canonical_stream: CanonStateNotificationStream,
     pub engine_api_client: HttpClient,
     pub _marker: PhantomData<E>,
 }
 
-impl<E: EngineTypes + 'static> EngineApiHelper<E> {
+impl<E: EngineTypes + 'static> EngineApiTestContext<E> {
     /// Retrieves a v3 payload from the engine api
     pub async fn get_payload_v3(
         &self,
@@ -34,6 +34,7 @@ impl<E: EngineTypes + 'static> EngineApiHelper<E> {
         payload: E::BuiltPayload,
         payload_builder_attributes: E::PayloadBuilderAttributes,
         expected_status: PayloadStatusEnum,
+        versioned_hashes: Vec<B256>,
     ) -> eyre::Result<B256>
     where
         E::ExecutionPayloadV3: From<E::BuiltPayload> + PayloadEnvelopeExt,
@@ -45,7 +46,7 @@ impl<E: EngineTypes + 'static> EngineApiHelper<E> {
         let submission = EngineApiClient::<E>::new_payload_v3(
             &self.engine_api_client,
             envelope_v3.execution_payload(),
-            vec![],
+            versioned_hashes,
             payload_builder_attributes.parent_beacon_block_root().unwrap(),
         )
         .await?;
@@ -56,18 +57,17 @@ impl<E: EngineTypes + 'static> EngineApiHelper<E> {
     }
 
     /// Sends forkchoice update to the engine api
-    pub async fn update_forkchoice(&self, hash: B256) -> eyre::Result<()> {
+    pub async fn update_forkchoice(&self, current_head: B256, new_head: B256) -> eyre::Result<()> {
         EngineApiClient::<E>::fork_choice_updated_v2(
             &self.engine_api_client,
             ForkchoiceState {
-                head_block_hash: hash,
-                safe_block_hash: hash,
-                finalized_block_hash: hash,
+                head_block_hash: new_head,
+                safe_block_hash: current_head,
+                finalized_block_hash: current_head,
             },
             None,
         )
         .await?;
-
         Ok(())
     }
 

crates/e2e-test-utils/src/lib.rs

@@ -1,4 +1,4 @@
-use node::NodeHelper;
+use node::NodeTestContext;
 use reth::{
     args::{DiscoveryArgs, NetworkArgs, RpcServerArgs},
     builder::{NodeBuilder, NodeConfig, NodeHandle},
@@ -18,6 +18,9 @@ use wallet::Wallet;
 /// Wrapper type to create test nodes
 pub mod node;
 
+/// Helper for transaction operations
+pub mod transaction;
+
 /// Helper type to yield accounts from mnemonic
 pub mod wallet;
 
@@ -29,6 +32,8 @@ mod network;
 
 /// Helper for engine api operations
 mod engine_api;
+/// Helper for rpc operations
+mod rpc;
 
 /// Helper traits
 mod traits;
@@ -75,7 +80,7 @@ where
             .launch()
             .await?;
 
-        let mut node = NodeHelper::new(node).await?;
+        let mut node = NodeTestContext::new(node).await?;
 
         // Connect each node in a chain.
         if let Some(previous_node) = nodes.last_mut() {
@@ -104,4 +109,5 @@ type TmpPool<N> = <<N as reth_node_builder::Node<TmpNodeAdapter<N>>>::PoolBuilde
 type TmpNodeAdapter<N> = FullNodeTypesAdapter<N, TmpDB, BlockchainProvider<TmpDB>>;
 
 /// Type alias for a type of NodeHelper
-pub type NodeHelperType<N> = NodeHelper<FullNodeComponentsAdapter<TmpNodeAdapter<N>, TmpPool<N>>>;
+pub type NodeHelperType<N> =
+    NodeTestContext<FullNodeComponentsAdapter<TmpNodeAdapter<N>, TmpPool<N>>>;

crates/e2e-test-utils/src/network.rs

@@ -5,12 +5,12 @@ use reth_tracing::tracing::info;
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
 /// Helper for network operations
-pub struct NetworkHelper {
+pub struct NetworkTestContext {
     network_events: UnboundedReceiverStream<NetworkEvent>,
     network: NetworkHandle,
 }
 
-impl NetworkHelper {
+impl NetworkTestContext {
     /// Creates a new network helper
     pub fn new(network: NetworkHandle) -> Self {
         let network_events = network.event_listener();

crates/e2e-test-utils/src/node.rs

@@ -1,35 +1,36 @@
 use crate::{
-    engine_api::EngineApiHelper, network::NetworkHelper, payload::PayloadHelper,
-    traits::PayloadEnvelopeExt,
+    engine_api::EngineApiTestContext, network::NetworkTestContext, payload::PayloadTestContext,
+    rpc::RpcTestContext, traits::PayloadEnvelopeExt,
 };
+
 use alloy_rpc_types::BlockNumberOrTag;
 use eyre::Ok;
+
 use futures_util::Future;
 use reth::{
     api::{BuiltPayload, EngineTypes, FullNodeComponents, PayloadBuilderAttributes},
     builder::FullNode,
     providers::{BlockReader, BlockReaderIdExt, CanonStateSubscriptions, StageCheckpointReader},
-    rpc::{
-        eth::{error::EthResult, EthTransactions},
-        types::engine::PayloadStatusEnum,
-    },
+    rpc::types::engine::PayloadStatusEnum,
 };
+use reth_node_builder::NodeTypes;
 use reth_primitives::{stage::StageId, BlockHash, BlockNumber, Bytes, B256};
 use std::{marker::PhantomData, pin::Pin};
 use tokio_stream::StreamExt;
 
 /// An helper struct to handle node actions
-pub struct NodeHelper<Node>
+pub struct NodeTestContext<Node>
 where
     Node: FullNodeComponents,
 {
     pub inner: FullNode<Node>,
-    pub payload: PayloadHelper<Node::Engine>,
-    pub network: NetworkHelper,
-    pub engine_api: EngineApiHelper<Node::Engine>,
+    pub payload: PayloadTestContext<Node::Engine>,
+    pub network: NetworkTestContext,
+    pub engine_api: EngineApiTestContext<Node::Engine>,
+    pub rpc: RpcTestContext<Node>,
 }
 
-impl<Node> NodeHelper<Node>
+impl<Node> NodeTestContext<Node>
 where
     Node: FullNodeComponents,
 {
@@ -39,17 +40,18 @@ where
 
         Ok(Self {
             inner: node.clone(),
-            network: NetworkHelper::new(node.network.clone()),
-            payload: PayloadHelper::new(builder).await?,
-            engine_api: EngineApiHelper {
+            payload: PayloadTestContext::new(builder).await?,
+            network: NetworkTestContext::new(node.network.clone()),
+            engine_api: EngineApiTestContext {
                 engine_api_client: node.auth_server_handle().http_client(),
                 canonical_stream: node.provider.canonical_state_stream(),
                 _marker: PhantomData::<Node::Engine>,
             },
+            rpc: RpcTestContext { inner: node.rpc_registry },
         })
     }
 
-    pub async fn connect(&mut self, node: &mut NodeHelper<Node>) {
+    pub async fn connect(&mut self, node: &mut NodeTestContext<Node>) {
         self.network.add_peer(node.network.record()).await;
         node.network.add_peer(self.network.record()).await;
         node.network.expect_session().await;
@@ -62,7 +64,7 @@ where
     pub async fn advance(
         &mut self,
         length: u64,
-        tx_generator: impl Fn() -> Pin<Box<dyn Future<Output = Bytes>>>,
+        tx_generator: impl Fn(u64) -> Pin<Box<dyn Future<Output = Bytes>>>,
         attributes_generator: impl Fn(u64) -> <Node::Engine as EngineTypes>::PayloadBuilderAttributes
             + Copy,
     ) -> eyre::Result<
@@ -76,60 +78,74 @@ where
             From<<Node::Engine as EngineTypes>::BuiltPayload> + PayloadEnvelopeExt,
     {
         let mut chain = Vec::with_capacity(length as usize);
-        for _ in 0..length {
-            let (payload, _) =
-                self.advance_block(tx_generator().await, attributes_generator).await?;
-            chain.push(payload);
+        for i in 0..length {
+            let raw_tx = tx_generator(i).await;
+            let tx_hash = self.rpc.inject_tx(raw_tx).await?;
+            let (payload, eth_attr) = self.advance_block(vec![], attributes_generator).await?;
+            let block_hash = payload.block().hash();
+            let block_number = payload.block().number;
+            self.assert_new_block(tx_hash, block_hash, block_number).await?;
+            chain.push((payload, eth_attr));
         }
         Ok(chain)
     }
 
-    /// Advances the node forward one block
-    pub async fn advance_block(
+    /// Creates a new payload from given attributes generator
+    /// expects a payload attribute event and waits until the payload is built.
+    ///
+    /// It triggers the resolve payload via engine api and expects the built payload event.
+    pub async fn new_payload(
         &mut self,
-        raw_tx: Bytes,
         attributes_generator: impl Fn(u64) -> <Node::Engine as EngineTypes>::PayloadBuilderAttributes,
     ) -> eyre::Result<(
-        (
-            <Node::Engine as EngineTypes>::BuiltPayload,
-            <Node::Engine as EngineTypes>::PayloadBuilderAttributes,
-        ),
-        B256,
+        <<Node as NodeTypes>::Engine as EngineTypes>::BuiltPayload,
+        <<Node as NodeTypes>::Engine as EngineTypes>::PayloadBuilderAttributes,
     )>
     where
         <Node::Engine as EngineTypes>::ExecutionPayloadV3:
             From<<Node::Engine as EngineTypes>::BuiltPayload> + PayloadEnvelopeExt,
     {
-        // push tx into pool via RPC server
-        let tx_hash = self.inject_tx(raw_tx).await?;
-
         // trigger new payload building draining the pool
         let eth_attr = self.payload.new_payload(attributes_generator).await.unwrap();
-
         // first event is the payload attributes
         self.payload.expect_attr_event(eth_attr.clone()).await?;
-
         // wait for the payload builder to have finished building
         self.payload.wait_for_built_payload(eth_attr.payload_id()).await;
-
         // trigger resolve payload via engine api
         self.engine_api.get_payload_v3(eth_attr.payload_id()).await?;
-
         // ensure we're also receiving the built payload as event
-        let payload = self.payload.expect_built_payload().await?;
+        Ok((self.payload.expect_built_payload().await?, eth_attr))
+    }
+
+    /// Advances the node forward one block
+    pub async fn advance_block(
+        &mut self,
+        versioned_hashes: Vec<B256>,
+        attributes_generator: impl Fn(u64) -> <Node::Engine as EngineTypes>::PayloadBuilderAttributes,
+    ) -> eyre::Result<(
+        <Node::Engine as EngineTypes>::BuiltPayload,
+        <<Node as NodeTypes>::Engine as EngineTypes>::PayloadBuilderAttributes,
+    )>
+    where
+        <Node::Engine as EngineTypes>::ExecutionPayloadV3:
+            From<<Node::Engine as EngineTypes>::BuiltPayload> + PayloadEnvelopeExt,
+    {
+        let (payload, eth_attr) = self.new_payload(attributes_generator).await?;
 
-        // submit payload via engine api
         let block_hash = self
             .engine_api
-            .submit_payload(payload.clone(), eth_attr.clone(), PayloadStatusEnum::Valid)
+            .submit_payload(
+                payload.clone(),
+                eth_attr.clone(),
+                PayloadStatusEnum::Valid,
+                versioned_hashes,
+            )
             .await?;
 
         // trigger forkchoice update via engine api to commit the block to the blockchain
-        self.engine_api.update_forkchoice(block_hash).await?;
+        self.engine_api.update_forkchoice(block_hash, block_hash).await?;
 
-        // assert the block has been committed to the blockchain
-        self.assert_new_block(tx_hash, block_hash, payload.block().number).await?;
-        Ok(((payload, eth_attr), tx_hash))
+        Ok((payload, eth_attr))
     }
 
     /// Waits for block to be available on node.
@@ -169,12 +185,6 @@ where
         Ok(())
     }
 
-    /// Injects a raw transaction into the node tx pool via RPC server
-    async fn inject_tx(&mut self, raw_tx: Bytes) -> EthResult<B256> {
-        let eth_api = self.inner.rpc_registry.eth_api();
-        eth_api.send_raw_transaction(raw_tx).await
-    }
-
     /// Asserts that a new block has been added to the blockchain
     /// and the tx has been included in the block
     pub async fn assert_new_block(

crates/e2e-test-utils/src/payload.rs

@@ -4,13 +4,13 @@ use reth_payload_builder::{Events, PayloadBuilderHandle, PayloadId};
 use tokio_stream::wrappers::BroadcastStream;
 
 /// Helper for payload operations
-pub struct PayloadHelper<E: EngineTypes + 'static> {
+pub struct PayloadTestContext<E: EngineTypes + 'static> {
     pub payload_event_stream: BroadcastStream<Events<E>>,
     payload_builder: PayloadBuilderHandle<E>,
     pub timestamp: u64,
 }
 
-impl<E: EngineTypes + 'static> PayloadHelper<E> {
+impl<E: EngineTypes + 'static> PayloadTestContext<E> {
     /// Creates a new payload helper
     pub async fn new(payload_builder: PayloadBuilderHandle<E>) -> eyre::Result<Self> {
         let payload_events = payload_builder.subscribe().await?;

crates/e2e-test-utils/src/rpc.rs

@@ -0,0 +1,24 @@
+use alloy_consensus::TxEnvelope;
+use alloy_network::eip2718::Decodable2718;
+use reth::{api::FullNodeComponents, builder::rpc::RpcRegistry, rpc::api::DebugApiServer};
+use reth_primitives::{Bytes, B256};
+use reth_rpc::eth::{error::EthResult, EthTransactions};
+
+pub struct RpcTestContext<Node: FullNodeComponents> {
+    pub inner: RpcRegistry<Node>,
+}
+
+impl<Node: FullNodeComponents> RpcTestContext<Node> {
+    /// Injects a raw transaction into the node tx pool via RPC server
+    pub async fn inject_tx(&mut self, raw_tx: Bytes) -> EthResult<B256> {
+        let eth_api = self.inner.eth_api();
+        eth_api.send_raw_transaction(raw_tx).await
+    }
+
+    /// Retrieves a transaction envelope by its hash
+    pub async fn envelope_by_hash(&mut self, hash: B256) -> eyre::Result<TxEnvelope> {
+        let tx = self.inner.debug_api().raw_transaction(hash).await?.unwrap();
+        let tx = tx.to_vec();
+        Ok(TxEnvelope::decode_2718(&mut tx.as_ref()).unwrap())
+    }
+}

crates/e2e-test-utils/src/transaction.rs

@@ -0,0 +1,80 @@
+use alloy_consensus::{
+    BlobTransactionSidecar, SidecarBuilder, SimpleCoder, TxEip4844Variant, TxEnvelope,
+};
+use alloy_network::{eip2718::Encodable2718, EthereumSigner, TransactionBuilder};
+use alloy_rpc_types::{TransactionInput, TransactionRequest};
+use alloy_signer_wallet::LocalWallet;
+use eyre::Ok;
+use reth_primitives::{hex, Address, Bytes, U256};
+
+use reth_primitives::{constants::eip4844::MAINNET_KZG_TRUSTED_SETUP, B256};
+
+pub struct TransactionTestContext;
+
+impl TransactionTestContext {
+    /// Creates a static transfer and signs it
+    pub async fn transfer_tx(chain_id: u64, wallet: LocalWallet) -> Bytes {
+        let tx = tx(chain_id, None, 0);
+        let signer = EthereumSigner::from(wallet);
+        tx.build(&signer).await.unwrap().encoded_2718().into()
+    }
+
+    /// Creates a tx with blob sidecar and sign it
+    pub async fn tx_with_blobs(chain_id: u64, wallet: LocalWallet) -> eyre::Result<Bytes> {
+        let mut tx = tx(chain_id, None, 0);
+
+        let mut builder = SidecarBuilder::<SimpleCoder>::new();
+        builder.ingest(b"dummy blob");
+        let sidecar: BlobTransactionSidecar = builder.build()?;
+
+        tx.set_blob_sidecar(sidecar);
+        tx.set_max_fee_per_blob_gas(15e9 as u128);
+
+        let signer = EthereumSigner::from(wallet);
+        let signed = tx.clone().build(&signer).await.unwrap();
+
+        Ok(signed.encoded_2718().into())
+    }
+
+    pub async fn optimism_l1_block_info_tx(
+        chain_id: u64,
+        wallet: LocalWallet,
+        nonce: u64,
+    ) -> Bytes {
+        let l1_block_info = Bytes::from_static(&hex!("7ef9015aa044bae9d41b8380d781187b426c6fe43df5fb2fb57bd4466ef6a701e1f01e015694deaddeaddeaddeaddeaddeaddeaddeaddead000194420000000000000000000000000000000000001580808408f0d18001b90104015d8eb900000000000000000000000000000000000000000000000000000000008057650000000000000000000000000000000000000000000000000000000063d96d10000000000000000000000000000000000000000000000000000000000009f35273d89754a1e0387b89520d989d3be9c37c1f32495a88faf1ea05c61121ab0d1900000000000000000000000000000000000000000000000000000000000000010000000000000000000000002d679b567db6187c0c8323fa982cfb88b74dbcc7000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240"));
+        let tx = tx(chain_id, Some(l1_block_info), nonce);
+        let signer = EthereumSigner::from(wallet);
+        tx.build(&signer).await.unwrap().encoded_2718().into()
+    }
+
+    /// Validates the sidecar of a given tx envelope and returns the versioned hashes
+    pub fn validate_sidecar(tx: TxEnvelope) -> Vec<B256> {
+        let proof_setting = MAINNET_KZG_TRUSTED_SETUP.clone();
+
+        match tx {
+            TxEnvelope::Eip4844(signed) => match signed.tx() {
+                TxEip4844Variant::TxEip4844WithSidecar(tx) => {
+                    tx.validate_blob(&proof_setting).unwrap();
+                    tx.sidecar.versioned_hashes().collect()
+                }
+                _ => panic!("Expected Eip4844 transaction with sidecar"),
+            },
+            _ => panic!("Expected Eip4844 transaction"),
+        }
+    }
+}
+
+/// Creates a type 2 transaction
+fn tx(chain_id: u64, data: Option<Bytes>, nonce: u64) -> TransactionRequest {
+    TransactionRequest {
+        nonce: Some(nonce),
+        value: Some(U256::from(100)),
+        to: Some(Address::random()),
+        gas: Some(210000),
+        max_fee_per_gas: Some(20e9 as u128),
+        max_priority_fee_per_gas: Some(20e9 as u128),
+        chain_id: Some(chain_id),
+        input: TransactionInput { input: None, data },
+        ..Default::default()
+    }
+}

crates/e2e-test-utils/src/wallet.rs

@@ -1,19 +1,19 @@
-use alloy_network::{eip2718::Encodable2718, EthereumSigner, TransactionBuilder};
-use alloy_rpc_types::{TransactionInput, TransactionRequest};
+use alloy_signer::Signer;
 use alloy_signer_wallet::{coins_bip39::English, LocalWallet, MnemonicBuilder};
-use reth_primitives::{hex, Address, Bytes, U256};
+
 /// One of the accounts of the genesis allocations.
 pub struct Wallet {
-    inner: LocalWallet,
-    pub nonce: u64,
+    pub inner: LocalWallet,
     chain_id: u64,
+    amount: usize,
+    derivation_path: Option<String>,
 }
 
 impl Wallet {
     /// Creates a new account from one of the secret/pubkeys of the genesis allocations (test.json)
-    pub(crate) fn new(phrase: &str) -> Self {
-        let inner = MnemonicBuilder::<English>::default().phrase(phrase).build().unwrap();
-        Self { inner, chain_id: 1, nonce: 0 }
+    pub fn new(amount: usize) -> Self {
+        let inner = MnemonicBuilder::<English>::default().phrase(TEST_MNEMONIC).build().unwrap();
+        Self { inner, chain_id: 1, amount, derivation_path: None }
     }
 
     /// Sets chain id
@@ -22,31 +22,24 @@ impl Wallet {
         self
     }
 
-    /// Creates a static transfer and signs it
-    pub async fn transfer_tx(&mut self) -> Bytes {
-        self.tx(None).await
+    fn get_derivation_path(&self) -> &str {
+        self.derivation_path.as_deref().unwrap_or("m/44'/60'/0'/0/")
     }
 
-    pub async fn optimism_l1_block_info_tx(&mut self) -> Bytes {
-        let l1_block_info = Bytes::from_static(&hex!("7ef9015aa044bae9d41b8380d781187b426c6fe43df5fb2fb57bd4466ef6a701e1f01e015694deaddeaddeaddeaddeaddeaddeaddeaddead000194420000000000000000000000000000000000001580808408f0d18001b90104015d8eb900000000000000000000000000000000000000000000000000000000008057650000000000000000000000000000000000000000000000000000000063d96d10000000000000000000000000000000000000000000000000000000000009f35273d89754a1e0387b89520d989d3be9c37c1f32495a88faf1ea05c61121ab0d1900000000000000000000000000000000000000000000000000000000000000010000000000000000000000002d679b567db6187c0c8323fa982cfb88b74dbcc7000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240"));
-        self.tx(Some(l1_block_info)).await
-    }
+    pub fn gen(&self) -> Vec<LocalWallet> {
+        let builder = MnemonicBuilder::<English>::default().phrase(TEST_MNEMONIC);
 
-    /// Creates a transaction with data and signs it
-    pub async fn tx(&mut self, data: Option<Bytes>) -> Bytes {
-        let tx = TransactionRequest {
-            nonce: Some(self.nonce),
-            value: Some(U256::from(100)),
-            to: Some(Address::random()),
-            gas_price: Some(20e9 as u128),
-            gas: Some(210000),
-            chain_id: Some(self.chain_id),
-            input: TransactionInput { input: None, data },
-            ..Default::default()
-        };
-        self.nonce += 1;
-        let signer = EthereumSigner::from(self.inner.clone());
-        tx.build(&signer).await.unwrap().encoded_2718().into()
+        // use the derivation path
+        let derivation_path = self.get_derivation_path();
+
+        let mut wallets = Vec::with_capacity(self.amount);
+        for idx in 0..self.amount {
+            let builder =
+                builder.clone().derivation_path(&format!("{derivation_path}{idx}")).unwrap();
+            let wallet = builder.build().unwrap().with_chain_id(Some(self.chain_id));
+            wallets.push(wallet)
+        }
+        wallets
     }
 }
 
@@ -54,6 +47,6 @@ const TEST_MNEMONIC: &str = "test test test test test test test test test test t
 
 impl Default for Wallet {
     fn default() -> Self {
-        Wallet::new(TEST_MNEMONIC)
+        Wallet::new(1)
     }
 }