feat: integrate blobs into the payload builder (#4305)

Cargo.lock

@@ -5751,6 +5751,7 @@ dependencies = [
  "reth-revm-primitives",
  "reth-rlp",
  "reth-rpc-types",
+ "reth-transaction-pool",
  "revm-primitives",
  "sha2",
  "thiserror",

crates/payload/basic/src/lib.rs

@@ -22,7 +22,9 @@ use reth_payload_builder::{
 };
 use reth_primitives::{
     bytes::{Bytes, BytesMut},
+    calculate_excess_blob_gas,
     constants::{
+        eip4844::{DATA_GAS_PER_BLOB, MAX_DATA_GAS_PER_BLOCK},
         BEACON_NONCE, EMPTY_RECEIPTS, EMPTY_TRANSACTIONS, EMPTY_WITHDRAWALS,
         ETHEREUM_BLOCK_GAS_LIMIT, RETH_CLIENT_VERSION, SLOT_DURATION,
     },
@@ -651,6 +653,7 @@ where
     let mut post_state = PostState::default();
 
     let mut cumulative_gas_used = 0;
+    let mut sum_blob_gas_used = 0;
     let block_gas_limit: u64 = initialized_block_env.gas_limit.try_into().unwrap_or(u64::MAX);
     let base_fee = initialized_block_env.basefee.to::<u64>();
 
@@ -679,6 +682,20 @@ where
         // convert tx to a signed transaction
         let tx = pool_tx.to_recovered_transaction();
 
+        if let Some(blob_tx) = tx.transaction.as_eip4844() {
+            let tx_blob_gas = blob_tx.blob_versioned_hashes.len() as u64 * DATA_GAS_PER_BLOB;
+            if sum_blob_gas_used + tx_blob_gas > MAX_DATA_GAS_PER_BLOCK {
+                // we can't fit this _blob_ transaction into the block, so we mark it as invalid,
+                // which removes its dependent transactions from the iterator. This is similar to
+                // the gas limit condition for regular transactions above.
+                best_txs.mark_invalid(&pool_tx);
+                continue
+            } else {
+                // add to the data gas if we're going to execute the transaction
+                sum_blob_gas_used += tx_blob_gas;
+            }
+        }
+
         // Configure the environment for the block.
         let env = Env {
             cfg: initialized_cfg.clone(),
@@ -765,6 +782,34 @@ where
     // create the block header
     let transactions_root = proofs::calculate_transaction_root(&executed_txs);
 
+    // initialize empty blob sidecars at first. If cancun is active then this will
+    let mut blob_sidecars = Vec::new();
+    let mut excess_blob_gas = None;
+    let mut blob_gas_used = None;
+
+    // only determine cancun fields when active
+    if chain_spec.is_cancun_activated_at_timestamp(attributes.timestamp) {
+        // grab the blob sidecars from the executed txs
+        let blobs = pool.get_all_blobs(
+            executed_txs.iter().filter(|tx| tx.is_eip4844()).map(|tx| tx.hash).collect(),
+        )?;
+
+        // map to just the sidecars
+        blob_sidecars = blobs.into_iter().map(|(_, sidecars)| sidecars).collect();
+
+        excess_blob_gas = if chain_spec.is_cancun_activated_at_timestamp(parent_block.timestamp) {
+            let parent_excess_blob_gas = parent_block.excess_blob_gas.unwrap_or_default();
+            let parent_blob_gas_used = parent_block.blob_gas_used.unwrap_or_default();
+            Some(calculate_excess_blob_gas(parent_excess_blob_gas, parent_blob_gas_used))
+        } else {
+            // for the first post-fork block, both parent.blob_gas_used and parent.excess_blob_gas
+            // are evaluated as 0
+            Some(calculate_excess_blob_gas(0, 0))
+        };
+
+        blob_gas_used = Some(sum_blob_gas_used);
+    }
+
     let header = Header {
         parent_hash: parent_block.hash,
         ommers_hash: EMPTY_OMMER_ROOT,
@@ -783,19 +828,23 @@ where
         difficulty: U256::ZERO,
         gas_used: cumulative_gas_used,
         extra_data: extra_data.into(),
-        blob_gas_used: None,
-        excess_blob_gas: None,
         parent_beacon_block_root: None,
+        blob_gas_used,
+        excess_blob_gas,
     };
 
     // seal the block
     let block = Block { header, body: executed_txs, ommers: vec![], withdrawals };
 
     let sealed_block = block.seal_slow();
-    Ok(BuildOutcome::Better {
+    let mut payload = BuiltPayload::new(attributes.id, sealed_block, total_fees);
-        payload: BuiltPayload::new(attributes.id, sealed_block, total_fees),
+
-        cached_reads,
+    if !blob_sidecars.is_empty() {
-    })
+        // extend the payload with the blob sidecars from the executed txs
+        payload.extend_sidecars(blob_sidecars);
+    }
+
+    Ok(BuildOutcome::Better { payload, cached_reads })
 }
 
 /// Builds an empty payload without any transactions.

crates/payload/builder/Cargo.toml

@@ -13,6 +13,7 @@ description = "reth payload builder"
 reth-primitives.workspace = true
 reth-rpc-types.workspace = true
 reth-rlp.workspace = true
+reth-transaction-pool.workspace = true
 reth-interfaces.workspace = true
 reth-revm-primitives = { path = "../../revm/revm-primitives" }
 

crates/payload/builder/src/error.rs

@@ -1,6 +1,7 @@
 //! Error types emitted by types or implementations of this crate.
 
 use reth_primitives::H256;
+use reth_transaction_pool::BlobStoreError;
 use revm_primitives::EVMError;
 use tokio::sync::oneshot;
 
@@ -13,6 +14,9 @@ pub enum PayloadBuilderError {
     /// An oneshot channels has been closed.
     #[error("sender has been dropped")]
     ChannelClosed,
+    /// Error occurring in the blob store.
+    #[error(transparent)]
+    BlobStore(#[from] BlobStoreError),
     /// Other internal error
     #[error(transparent)]
     Internal(#[from] reth_interfaces::Error),

crates/payload/builder/src/payload.rs

@@ -1,6 +1,8 @@
 //! Contains types required for building a payload.
 
-use reth_primitives::{Address, ChainSpec, Header, SealedBlock, Withdrawal, H256, U256};
+use reth_primitives::{
+    Address, BlobTransactionSidecar, ChainSpec, Header, SealedBlock, Withdrawal, H256, U256,
+};
 use reth_revm_primitives::config::revm_spec_by_timestamp_after_merge;
 use reth_rlp::Encodable;
 use reth_rpc_types::engine::{
@@ -21,6 +23,9 @@ pub struct BuiltPayload {
     pub(crate) block: SealedBlock,
     /// The fees of the block
     pub(crate) fees: U256,
+    /// The blobs, proofs, and commitments in the block. If the block is pre-cancun, this will be
+    /// empty.
+    pub(crate) sidecars: Vec<BlobTransactionSidecar>,
 }
 
 // === impl BuiltPayload ===
@@ -28,7 +33,7 @@ pub struct BuiltPayload {
 impl BuiltPayload {
     /// Initializes the payload with the given initial block.
     pub fn new(id: PayloadId, block: SealedBlock, fees: U256) -> Self {
-        Self { id, block, fees }
+        Self { id, block, fees, sidecars: Vec::new() }
     }
 
     /// Returns the identifier of the payload.
@@ -46,6 +51,11 @@ impl BuiltPayload {
         self.fees
     }
 
+    /// Adds sidecars to the payload.
+    pub fn extend_sidecars(&mut self, sidecars: Vec<BlobTransactionSidecar>) {
+        self.sidecars.extend(sidecars)
+    }
+
     /// Converts the type into the response expected by `engine_getPayloadV1`
     pub fn into_v1_payload(self) -> ExecutionPayload {
         self.into()
@@ -53,6 +63,14 @@ impl BuiltPayload {
 
     /// Converts the type into the response expected by `engine_getPayloadV2`
     pub fn into_v2_payload(self) -> ExecutionPayloadEnvelope {
+        let mut envelope: ExecutionPayloadEnvelope = self.into();
+        envelope.blobs_bundle = None;
+        envelope.should_override_builder = None;
+        envelope
+    }
+
+    /// Converts the type into the response expected by `engine_getPayloadV2`
+    pub fn into_v3_payload(self) -> ExecutionPayloadEnvelope {
         self.into()
     }
 }
@@ -65,14 +83,27 @@ impl From<BuiltPayload> for ExecutionPayload {
 }
 
 // V2 engine_getPayloadV2 response
+// TODO(rjected): we could improve this by wrapping envelope / payload types by version, so we can
+// have explicitly versioned return types for getPayload. Then BuiltPayload could essentially be a
+// builder for those types, and it would not be possible to e.g. return cancun fields for a
+// pre-cancun endpoint.
 impl From<BuiltPayload> for ExecutionPayloadEnvelope {
     fn from(value: BuiltPayload) -> Self {
-        let BuiltPayload { block, fees, .. } = value;
+        let BuiltPayload { block, fees, sidecars, .. } = value;
 
         ExecutionPayloadEnvelope {
             block_value: fees,
             payload: block.into(),
-            should_override_builder: None,
+            // From the engine API spec:
+            //
+            // > Client software **MAY** use any heuristics to decide whether to set
+            // `shouldOverrideBuilder` flag or not. If client software does not implement any
+            // heuristic this flag **SHOULD** be set to `false`.
+            //
+            // Spec:
+            // <https://github.com/ethereum/execution-apis/blob/fe8e13c288c592ec154ce25c534e26cb7ce0530d/src/engine/cancun.md#specification-2>
+            should_override_builder: Some(false),
+            blobs_bundle: Some(sidecars.into()),
         }
     }
 }

crates/rpc/rpc-engine-api/src/engine_api.rs

@@ -180,7 +180,7 @@ where
     ///
     /// Note:
     /// > Provider software MAY stop the corresponding build process after serving this call.
-    async fn get_payload_v2(
+    pub async fn get_payload_v2(
         &self,
         payload_id: PayloadId,
     ) -> EngineApiResult<ExecutionPayloadEnvelope> {
@@ -193,6 +193,26 @@ where
             .map(|payload| (*payload).clone().into_v2_payload())?)
     }
 
+    /// Returns the most recent version of the payload that is available in the corresponding
+    /// payload build process at the time of receiving this call.
+    ///
+    /// See also <https://github.com/ethereum/execution-apis/blob/fe8e13c288c592ec154ce25c534e26cb7ce0530d/src/engine/cancun.md#engine_getpayloadv3>
+    ///
+    /// Note:
+    /// > Provider software MAY stop the corresponding build process after serving this call.
+    pub async fn get_payload_v3(
+        &self,
+        payload_id: PayloadId,
+    ) -> EngineApiResult<ExecutionPayloadEnvelope> {
+        Ok(self
+            .inner
+            .payload_store
+            .resolve(payload_id)
+            .await
+            .ok_or(EngineApiError::UnknownPayload)?
+            .map(|payload| (*payload).clone().into_v3_payload())?)
+    }
+
     /// Returns the execution payload bodies by the range starting at `start`, containing `count`
     /// blocks.
     ///

crates/rpc/rpc-types/src/eth/engine/payload.rs

@@ -1,8 +1,9 @@
 use reth_primitives::{
     constants::{MAXIMUM_EXTRA_DATA_SIZE, MIN_PROTOCOL_BASE_FEE_U256},
+    kzg::{Blob, Bytes48},
     proofs::{self, EMPTY_LIST_HASH},
-    Address, Block, Bloom, Bytes, Header, SealedBlock, TransactionSigned, UintTryTo, Withdrawal,
+    Address, BlobTransactionSidecar, Block, Bloom, Bytes, Header, SealedBlock, TransactionSigned,
-    H256, H64, U256, U64,
+    UintTryTo, Withdrawal, H256, H64, U256, U64,
 };
 use reth_rlp::Decodable;
 use serde::{ser::SerializeMap, Deserialize, Serialize, Serializer};
@@ -49,10 +50,9 @@ pub struct ExecutionPayloadEnvelope {
     /// The expected value to be received by the feeRecipient in wei
     #[serde(rename = "blockValue")]
     pub block_value: U256,
-    //
+    /// The blobs, commitments, and proofs associated with the executed payload.
-    // // TODO(mattsse): for V3
+    #[serde(rename = "blobsBundle", skip_serializing_if = "Option::is_none")]
-    // #[serde(rename = "blobsBundle", skip_serializing_if = "Option::is_none")]
+    pub blobs_bundle: Option<BlobsBundleV1>,
-    // pub blobs_bundle: Option<BlobsBundleV1>,
     /// Introduced in V3, this represents a suggestion from the execution layer if the payload
     /// should be used instead of an externally provided one.
     #[serde(rename = "shouldOverrideBuilder", skip_serializing_if = "Option::is_none")]
@@ -214,9 +214,24 @@ impl ExecutionPayload {
 /// This includes all bundled blob related data of an executed payload.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct BlobsBundleV1 {
-    pub commitments: Vec<Bytes>,
+    pub commitments: Vec<Bytes48>,
-    pub proofs: Vec<Bytes>,
+    pub proofs: Vec<Bytes48>,
-    pub blobs: Vec<Bytes>,
+    pub blobs: Vec<Blob>,
+}
+
+impl From<Vec<BlobTransactionSidecar>> for BlobsBundleV1 {
+    fn from(sidecars: Vec<BlobTransactionSidecar>) -> Self {
+        let (commitments, proofs, blobs) = sidecars.into_iter().fold(
+            (Vec::new(), Vec::new(), Vec::new()),
+            |(mut commitments, mut proofs, mut blobs), sidecar| {
+                commitments.extend(sidecar.commitments);
+                proofs.extend(sidecar.proofs);
+                blobs.extend(sidecar.blobs);
+                (commitments, proofs, blobs)
+            },
+        );
+        Self { commitments, proofs, blobs }
+    }
 }
 
 /// Error that can occur when handling payloads.

crates/transaction-pool/src/lib.rs

@@ -165,8 +165,8 @@ use std::{
 use tokio::sync::mpsc::Receiver;
 use tracing::{instrument, trace};
 
-use crate::blobstore::{BlobStore, BlobStoreError};
 pub use crate::{
+    blobstore::{BlobStore, BlobStoreError},
     config::{
         PoolConfig, PriceBumpConfig, SubPoolLimit, DEFAULT_PRICE_BUMP, REPLACE_BLOB_PRICE_BUMP,
         TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER, TXPOOL_SUBPOOL_MAX_SIZE_MB_DEFAULT,