feat(engine-api): Shanghai support (#1445)

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

crates/rpc/rpc-api/src/engine.rs

@@ -1,8 +1,8 @@
 use jsonrpsee::{core::RpcResult as Result, proc_macros::rpc};
 use reth_primitives::{BlockHash, BlockNumber, H64};
 use reth_rpc_types::engine::{
-    ExecutionPayload, ExecutionPayloadBody, ForkchoiceState, ForkchoiceUpdated, PayloadAttributes,
-    PayloadStatus, TransitionConfiguration,
+    ExecutionPayload, ExecutionPayloadBodies, ForkchoiceState, ForkchoiceUpdated,
+    PayloadAttributes, PayloadStatus, TransitionConfiguration,
 };
 
 #[cfg_attr(not(feature = "client"), rpc(server))]
@@ -50,7 +50,7 @@ pub trait EngineApi {
     async fn get_payload_bodies_by_hash_v1(
         &self,
         block_hashes: Vec<BlockHash>,
-    ) -> Result<Vec<ExecutionPayloadBody>>;
+    ) -> Result<ExecutionPayloadBodies>;
 
     /// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyrangev1>
     #[method(name = "engine_getPayloadBodiesByRangeV1")]
@@ -58,7 +58,7 @@ pub trait EngineApi {
         &self,
         start: BlockNumber,
         count: u64,
-    ) -> Result<Vec<ExecutionPayloadBody>>;
+    ) -> Result<ExecutionPayloadBodies>;
 
     /// See also <https://github.com/ethereum/execution-apis/blob/6709c2a795b707202e93c4f2867fa0bf2640a84f/src/engine/paris.md#engine_exchangetransitionconfigurationv1>
     #[method(name = "engine_exchangeTransitionConfigurationV1")]

crates/rpc/rpc-engine-api/src/error.rs

@@ -4,8 +4,13 @@ use thiserror::Error;
 /// The Engine API result type
 pub type EngineApiResult<Ok> = Result<Ok, EngineApiError>;
 
+/// Payload unknown error code.
+pub const UNKNOWN_PAYLOAD_CODE: i32 = -38001;
+/// Request too large error code.
+pub const REQUEST_TOO_LARGE_CODE: i32 = -38004;
+
 /// Error returned by [`EngineApi`][crate::EngineApi]
-#[derive(Error, Debug)]
+#[derive(Error, PartialEq, Debug)]
 pub enum EngineApiError {
     /// Invalid payload extra data.
     #[error("Invalid payload extra data: {0}")]
@@ -41,6 +46,15 @@ pub enum EngineApiError {
     /// Unknown payload requested.
     #[error("Unknown payload")]
     PayloadUnknown,
+    /// The payload body request length is too large.
+    #[error("Payload request too large: {len}")]
+    PayloadRequestTooLarge {
+        /// The length that was requested.
+        len: u64,
+    },
+    /// The params are invalid.
+    #[error("Invalid params")]
+    InvalidParams,
     /// Terminal total difficulty mismatch during transition configuration exchange.
     #[error(
         "Invalid transition terminal total difficulty. Execution: {execution}. Consensus: {consensus}"

crates/rpc/rpc-engine-api/src/lib.rs

@@ -9,14 +9,14 @@
 //! [Read more](https://github.com/ethereum/execution-apis/tree/main/src/engine).
 
 /// The Engine API implementation.
-pub mod engine_api;
+mod engine_api;
 
 /// The Engine API message type.
-pub mod message;
+mod message;
 
 /// Engine API error.
-pub mod error;
+mod error;
 
 pub use engine_api::{EngineApi, EngineApiSender};
-pub use error::{EngineApiError, EngineApiResult};
-pub use message::EngineApiMessage;
+pub use error::*;
+pub use message::{EngineApiMessage, EngineApiMessageVersion};

crates/rpc/rpc-engine-api/src/message.rs

@@ -1,19 +1,25 @@
 use crate::EngineApiSender;
 use reth_interfaces::consensus::ForkchoiceState;
-use reth_primitives::H64;
+use reth_primitives::{BlockHash, BlockNumber, H64};
 use reth_rpc_types::engine::{
-    ExecutionPayload, ForkchoiceUpdated, PayloadAttributes, PayloadStatus, TransitionConfiguration,
+    ExecutionPayload, ExecutionPayloadBodies, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
+    TransitionConfiguration,
 };
 
 /// Message type for communicating with [`EngineApi`][crate::EngineApi].
 #[derive(Debug)]
 pub enum EngineApiMessage {
     /// New payload message
-    NewPayload(ExecutionPayload, EngineApiSender<PayloadStatus>),
+    NewPayload(EngineApiMessageVersion, ExecutionPayload, EngineApiSender<PayloadStatus>),
     /// Get payload message
     GetPayload(H64, EngineApiSender<ExecutionPayload>),
+    /// Get payload bodies by range message
+    GetPayloadBodiesByRange(BlockNumber, u64, EngineApiSender<ExecutionPayloadBodies>),
+    /// Get payload bodies by hash message
+    GetPayloadBodiesByHash(Vec<BlockHash>, EngineApiSender<ExecutionPayloadBodies>),
     /// Forkchoice updated message
     ForkchoiceUpdated(
+        EngineApiMessageVersion,
         ForkchoiceState,
         Option<PayloadAttributes>,
         EngineApiSender<ForkchoiceUpdated>,
@@ -24,3 +30,12 @@ pub enum EngineApiMessage {
         EngineApiSender<TransitionConfiguration>,
     ),
 }
+
+/// The version of Engine API message.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EngineApiMessageVersion {
+    /// Version 1
+    V1,
+    /// Version 2
+    V2,
+}

crates/rpc/rpc-types/Cargo.toml

@@ -25,3 +25,6 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 jsonrpsee-types = { version = "0.16" }
 lru = "0.9"
+
+[dev-dependencies]
+reth-interfaces = { path = "../../interfaces", features = ["test-utils"] }

crates/rpc/rpc-types/src/eth/engine.rs

@@ -3,7 +3,7 @@
 #![allow(missing_docs)]
 
 use reth_primitives::{
-    bytes::BytesMut, Address, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64,
+    Address, Block, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64,
 };
 use reth_rlp::Encodable;
 use serde::{Deserialize, Serialize};
@@ -40,9 +40,9 @@ impl From<SealedBlock> for ExecutionPayload {
             .body
             .iter()
             .map(|tx| {
-                let mut encoded = BytesMut::new();
+                let mut encoded = Vec::new();
                 tx.encode(&mut encoded);
-                encoded.freeze().into()
+                encoded.into()
             })
             .collect();
         ExecutionPayload {
@@ -70,10 +70,27 @@ impl From<SealedBlock> for ExecutionPayload {
 /// See also: <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#executionpayloadbodyv1>
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ExecutionPayloadBody {
-    transactions: Vec<Bytes>,
-    withdrawals: Vec<Withdrawal>,
+    pub transactions: Vec<Bytes>,
+    pub withdrawals: Vec<Withdrawal>,
 }
 
+impl From<Block> for ExecutionPayloadBody {
+    fn from(value: Block) -> Self {
+        let transactions = value.body.into_iter().map(|tx| {
+            let mut out = Vec::new();
+            tx.encode(&mut out);
+            out.into()
+        });
+        ExecutionPayloadBody {
+            transactions: transactions.collect(),
+            withdrawals: value.withdrawals.unwrap_or_default(),
+        }
+    }
+}
+
+/// The execution payload body response that allows for `null` values.
+pub type ExecutionPayloadBodies = Vec<Option<ExecutionPayloadBody>>;
+
 /// This structure encapsulates the fork choice state
 #[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -171,3 +188,30 @@ impl ForkchoiceUpdated {
         self
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use reth_interfaces::test_utils::generators::random_block_range;
+    use reth_primitives::{TransactionSigned, H256};
+    use reth_rlp::Decodable;
+
+    #[test]
+    fn payload_body_roundtrip() {
+        for block in random_block_range(0..100, H256::default(), 0..2) {
+            let unsealed = block.clone().unseal();
+            let payload_body: ExecutionPayloadBody = unsealed.into();
+
+            assert_eq!(
+                Ok(block.body),
+                payload_body
+                    .transactions
+                    .iter()
+                    .map(|x| TransactionSigned::decode(&mut &x[..]))
+                    .collect::<Result<Vec<_>, _>>(),
+            );
+
+            assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals);
+        }
+    }
+}

crates/rpc/rpc/src/engine/mod.rs

@@ -1,12 +1,18 @@
-use crate::result::{internal_rpc_err, rpc_err};
+use crate::result::rpc_err;
 use async_trait::async_trait;
-use jsonrpsee::core::{Error, RpcResult as Result};
+use jsonrpsee::{
+    core::{Error, RpcResult as Result},
+    types::error::INVALID_PARAMS_CODE,
+};
 use reth_interfaces::consensus::ForkchoiceState;
 use reth_primitives::{BlockHash, BlockNumber, H64};
 use reth_rpc_api::EngineApiServer;
-use reth_rpc_engine_api::{EngineApiError, EngineApiMessage, EngineApiResult};
+use reth_rpc_engine_api::{
+    EngineApiError, EngineApiMessage, EngineApiMessageVersion, EngineApiResult,
+    REQUEST_TOO_LARGE_CODE, UNKNOWN_PAYLOAD_CODE,
+};
 use reth_rpc_types::engine::{
-    ExecutionPayload, ExecutionPayloadBody, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
+    ExecutionPayload, ExecutionPayloadBodies, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
     TransitionConfiguration,
 };
 use tokio::sync::{
@@ -35,7 +41,9 @@ impl EngineApi {
         let _ = self.engine_tx.send(msg);
         rx.await.map_err(|err| Error::Custom(err.to_string()))?.map_err(|err| {
             let code = match err {
-                EngineApiError::PayloadUnknown => -38001,
+                EngineApiError::InvalidParams => INVALID_PARAMS_CODE,
+                EngineApiError::PayloadUnknown => UNKNOWN_PAYLOAD_CODE,
+                EngineApiError::PayloadRequestTooLarge { .. } => REQUEST_TOO_LARGE_CODE,
                 // Any other server error
                 _ => jsonrpsee::types::error::INTERNAL_ERROR_CODE,
             };
@@ -50,13 +58,21 @@ impl EngineApiServer for EngineApi {
     /// Caution: This should not accept the `withdrawals` field
     async fn new_payload_v1(&self, payload: ExecutionPayload) -> Result<PayloadStatus> {
         let (tx, rx) = oneshot::channel();
-        self.delegate_request(EngineApiMessage::NewPayload(payload, tx), rx).await
+        self.delegate_request(
+            EngineApiMessage::NewPayload(EngineApiMessageVersion::V1, payload, tx),
+            rx,
+        )
+        .await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/8db51dcd2f4bdfbd9ad6e4a7560aac97010ad063/src/engine/specification.md#engine_newpayloadv1>
-    async fn new_payload_v2(&self, _payload: ExecutionPayload) -> Result<PayloadStatus> {
-        // TODO:
-        Err(internal_rpc_err("unimplemented"))
+    async fn new_payload_v2(&self, payload: ExecutionPayload) -> Result<PayloadStatus> {
+        let (tx, rx) = oneshot::channel();
+        self.delegate_request(
+            EngineApiMessage::NewPayload(EngineApiMessageVersion::V2, payload, tx),
+            rx,
+        )
+        .await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/8db51dcd2f4bdfbd9ad6e4a7560aac97010ad063/src/engine/specification.md#engine_forkchoiceUpdatedV1>
@@ -69,7 +85,12 @@ impl EngineApiServer for EngineApi {
     ) -> Result<ForkchoiceUpdated> {
         let (tx, rx) = oneshot::channel();
         self.delegate_request(
-            EngineApiMessage::ForkchoiceUpdated(fork_choice_state, payload_attributes, tx),
+            EngineApiMessage::ForkchoiceUpdated(
+                EngineApiMessageVersion::V1,
+                fork_choice_state,
+                payload_attributes,
+                tx,
+            ),
             rx,
         )
         .await
@@ -78,11 +99,20 @@ impl EngineApiServer for EngineApi {
     /// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_forkchoiceupdatedv2>
     async fn fork_choice_updated_v2(
         &self,
-        _fork_choice_state: ForkchoiceState,
-        _payload_attributes: Option<PayloadAttributes>,
+        fork_choice_state: ForkchoiceState,
+        payload_attributes: Option<PayloadAttributes>,
     ) -> Result<ForkchoiceUpdated> {
-        // TODO:
-        Err(internal_rpc_err("unimplemented"))
+        let (tx, rx) = oneshot::channel();
+        self.delegate_request(
+            EngineApiMessage::ForkchoiceUpdated(
+                EngineApiMessageVersion::V2,
+                fork_choice_state,
+                payload_attributes,
+                tx,
+            ),
+            rx,
+        )
+        .await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/8db51dcd2f4bdfbd9ad6e4a7560aac97010ad063/src/engine/specification.md#engine_getPayloadV1>
@@ -94,28 +124,28 @@ impl EngineApiServer for EngineApi {
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_getpayloadv2>
-    async fn get_payload_v2(&self, _payload_id: H64) -> Result<ExecutionPayload> {
-        // TODO:
-        Err(internal_rpc_err("unimplemented"))
+    async fn get_payload_v2(&self, payload_id: H64) -> Result<ExecutionPayload> {
+        let (tx, rx) = oneshot::channel();
+        self.delegate_request(EngineApiMessage::GetPayload(payload_id, tx), rx).await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1>
     async fn get_payload_bodies_by_hash_v1(
         &self,
-        _block_hashes: Vec<BlockHash>,
-    ) -> Result<Vec<ExecutionPayloadBody>> {
-        // TODO:
-        Err(internal_rpc_err("unimplemented"))
+        block_hashes: Vec<BlockHash>,
+    ) -> Result<ExecutionPayloadBodies> {
+        let (tx, rx) = oneshot::channel();
+        self.delegate_request(EngineApiMessage::GetPayloadBodiesByHash(block_hashes, tx), rx).await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyrangev1>
     async fn get_payload_bodies_by_range_v1(
         &self,
-        _start: BlockNumber,
-        _count: u64,
-    ) -> Result<Vec<ExecutionPayloadBody>> {
-        // TODO:
-        Err(internal_rpc_err("unimplemented"))
+        start: BlockNumber,
+        count: u64,
+    ) -> Result<ExecutionPayloadBodies> {
+        let (tx, rx) = oneshot::channel();
+        self.delegate_request(EngineApiMessage::GetPayloadBodiesByRange(start, count, tx), rx).await
     }
 
     /// See also <https://github.com/ethereum/execution-apis/blob/8db51dcd2f4bdfbd9ad6e4a7560aac97010ad063/src/engine/specification.md#engine_exchangeTransitionConfigurationV1>