Merge pull request #105 from hl-archive-node/feat/cache-spot-meta

feat: cache spot metadata in database to reduce API calls

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "reth_hl"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 build = "build.rs"
 
 [lib]
@@ -26,67 +26,67 @@ lto = "fat"
 codegen-units = 1
 
 [dependencies]
-reth = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-cli = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-cli-commands = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-basic-payload-builder = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-db = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-db-api = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-chainspec = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-cli-util = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-discv4 = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-engine-primitives = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-ethereum-forks = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-ethereum-payload-builder = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-ethereum-primitives = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-eth-wire = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-eth-wire-types = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-evm = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-evm-ethereum = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-node-core = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-revm = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-network = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-network-p2p = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-network-api = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-node-ethereum = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-network-peers = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-payload-primitives = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-primitives = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-primitives-traits = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-provider = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505", features = ["test-utils"] }
-reth-rpc = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-rpc-eth-api = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-rpc-engine-api = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-tracing = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-trie-common = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-trie-db = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-codecs = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-transaction-pool = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-stages-types = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-storage-api = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-errors = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-rpc-convert = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-rpc-eth-types = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-rpc-server-types = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-reth-metrics = { git = "https://github.com/sprites0/reth", rev = "d26fd2e25b57d695aa453c93f15a8cd158a1f505" }
-revm = { version = "29.0.0", default-features = false }
+reth = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-cli = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-cli-commands = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-basic-payload-builder = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-db = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-db-api = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-chainspec = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-cli-util = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-discv4 = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-engine-primitives = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-ethereum-forks = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-ethereum-payload-builder = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-ethereum-primitives = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-eth-wire = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-eth-wire-types = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-evm = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-evm-ethereum = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-node-core = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-revm = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-network = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-network-p2p = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-network-api = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-node-ethereum = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-network-peers = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-payload-primitives = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-primitives = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-primitives-traits = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-provider = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a", features = ["test-utils"] }
+reth-rpc = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-rpc-eth-api = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-rpc-engine-api = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-tracing = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-trie-common = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-trie-db = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-codecs = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-transaction-pool = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-stages-types = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-storage-api = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-errors = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-rpc-convert = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-rpc-eth-types = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-rpc-server-types = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+reth-metrics = { git = "https://github.com/hl-archive-node/reth", rev = "416c2e26756f1c8ee86e6b8e4081f434952b3a1a" }
+revm = { version = "29.0.1", default-features = false }
 
 # alloy dependencies
-alloy-genesis = { version = "1.0.30", default-features = false }
-alloy-consensus = { version = "1.0.30", default-features = false }
+alloy-genesis = { version = "1.0.37", default-features = false }
+alloy-consensus = { version = "1.0.37", default-features = false }
 alloy-chains = { version = "0.2.5", default-features = false }
-alloy-eips = { version = "1.0.30", default-features = false }
-alloy-evm = { version = "0.20.1", default-features = false }
+alloy-eips = { version = "1.0.37", default-features = false }
+alloy-evm = { version = "0.21.0", default-features = false }
 alloy-json-abi = { version = "1.3.1", default-features = false }
-alloy-json-rpc = { version = "1.0.30", default-features = false }
+alloy-json-rpc = { version = "1.0.37", default-features = false }
 alloy-dyn-abi = "1.3.1"
-alloy-network = { version = "1.0.30", default-features = false }
+alloy-network = { version = "1.0.37", default-features = false }
 alloy-primitives = { version = "1.3.1", default-features = false, features = ["map-foldhash"] }
 alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
-alloy-rpc-types = { version = "1.0.30", features = ["eth"], default-features = false }
-alloy-rpc-types-eth = { version = "1.0.30", default-features = false }
-alloy-rpc-types-engine = { version = "1.0.30", default-features = false }
-alloy-signer = { version = "1.0.30", default-features = false }
+alloy-rpc-types = { version = "1.0.37", features = ["eth"], default-features = false }
+alloy-rpc-types-eth = { version = "1.0.37", default-features = false }
+alloy-rpc-types-engine = { version = "1.0.37", default-features = false }
+alloy-signer = { version = "1.0.37", default-features = false }
 alloy-sol-macro = "1.3.1"
 alloy-sol-types = { version = "1.3.1", default-features = false }
 

src/addons/call_forwarder.rs

@@ -2,18 +2,18 @@ use alloy_eips::BlockId;
 use alloy_json_rpc::RpcObject;
 use alloy_primitives::{Bytes, U256};
 use alloy_rpc_types_eth::{
-    state::{EvmOverrides, StateOverride},
     BlockOverrides,
+    state::{EvmOverrides, StateOverride},
 };
 use jsonrpsee::{
     http_client::{HttpClient, HttpClientBuilder},
     proc_macros::rpc,
     rpc_params,
-    types::{error::INTERNAL_ERROR_CODE, ErrorObject},
+    types::{ErrorObject, error::INTERNAL_ERROR_CODE},
 };
-use jsonrpsee_core::{async_trait, client::ClientT, ClientError, RpcResult};
+use jsonrpsee_core::{ClientError, RpcResult, async_trait, client::ClientT};
 use reth_rpc::eth::EthApiTypes;
-use reth_rpc_eth_api::{helpers::EthCall, RpcTxReq};
+use reth_rpc_eth_api::{RpcTxReq, helpers::EthCall};
 
 #[rpc(server, namespace = "eth")]
 pub(crate) trait CallForwarderApi<TxReq: RpcObject> {

src/addons/hl_node_compliance.rs

@@ -7,70 +7,35 @@
 //! For non-system transactions, we can just return the log as is, and the client will
 //! adjust the transaction index accordingly.
 
-use alloy_consensus::{transaction::TransactionMeta, BlockHeader, TxReceipt};
+use alloy_consensus::{
+    BlockHeader, TxReceipt,
+    transaction::{TransactionMeta, TxHashRef},
+};
 use alloy_eips::{BlockId, BlockNumberOrTag};
 use alloy_json_rpc::RpcObject;
 use alloy_primitives::{B256, U256};
 use alloy_rpc_types::{
-    pubsub::{Params, SubscriptionKind},
     BlockTransactions, Filter, FilterChanges, FilterId, Log, PendingTransactionFilterKind,
     TransactionInfo,
+    pubsub::{Params, SubscriptionKind},
 };
-use jsonrpsee::{proc_macros::rpc, PendingSubscriptionSink, SubscriptionMessage, SubscriptionSink};
-use jsonrpsee_core::{async_trait, RpcResult};
-use jsonrpsee_types::{error::INTERNAL_ERROR_CODE, ErrorObject};
+use jsonrpsee::{PendingSubscriptionSink, proc_macros::rpc};
+use jsonrpsee_core::{RpcResult, async_trait};
+use jsonrpsee_types::{ErrorObject, error::INTERNAL_ERROR_CODE};
 use reth::{api::FullNodeComponents, builder::rpc::RpcContext, tasks::TaskSpawner};
-use reth_primitives_traits::{BlockBody as _, SignedTransaction};
+use reth_primitives_traits::SignedTransaction;
 use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, ReceiptProvider};
-use reth_rpc::{eth::pubsub::SubscriptionSerializeError, EthFilter, EthPubSub, RpcTypes};
+use reth_rpc::{EthFilter, EthPubSub};
 use reth_rpc_eth_api::{
-    helpers::{EthBlocks, EthTransactions, LoadReceipt},
-    transaction::ConvertReceiptInput,
-    EthApiServer, EthApiTypes, EthFilterApiServer, EthPubSubApiServer, FullEthApiTypes, RpcBlock,
-    RpcConvert, RpcHeader, RpcNodeCoreExt, RpcReceipt, RpcTransaction, RpcTxReq,
+    EthApiTypes, EthFilterApiServer, EthPubSubApiServer, RpcBlock, RpcConvert, RpcReceipt,
+    RpcTransaction, helpers::EthBlocks, transaction::ConvertReceiptInput,
 };
-use serde::Serialize;
-use std::{borrow::Cow, marker::PhantomData, sync::Arc};
-use tokio_stream::{Stream, StreamExt};
-use tracing::{trace, Instrument};
+use reth_rpc_eth_types::EthApiError;
+use std::{marker::PhantomData, sync::Arc};
+use tokio_stream::StreamExt;
+use tracing::{Instrument, trace};
 
-use crate::{node::primitives::HlPrimitives, HlBlock};
-
-pub trait EthWrapper:
-    EthApiServer<
-        RpcTxReq<Self::NetworkTypes>,
-        RpcTransaction<Self::NetworkTypes>,
-        RpcBlock<Self::NetworkTypes>,
-        RpcReceipt<Self::NetworkTypes>,
-        RpcHeader<Self::NetworkTypes>,
-    > + FullEthApiTypes<
-        Primitives = HlPrimitives,
-        NetworkTypes: RpcTypes<TransactionResponse = alloy_rpc_types_eth::Transaction>,
-    > + RpcNodeCoreExt<Provider: BlockReader<Block = HlBlock>>
-    + EthBlocks
-    + EthTransactions
-    + LoadReceipt
-    + 'static
-{
-}
-
-impl<T> EthWrapper for T where
-    T: EthApiServer<
-            RpcTxReq<Self::NetworkTypes>,
-            RpcTransaction<Self::NetworkTypes>,
-            RpcBlock<Self::NetworkTypes>,
-            RpcReceipt<Self::NetworkTypes>,
-            RpcHeader<Self::NetworkTypes>,
-        > + FullEthApiTypes<
-            Primitives = HlPrimitives,
-            NetworkTypes: RpcTypes<TransactionResponse = alloy_rpc_types_eth::Transaction>,
-        > + RpcNodeCoreExt<Provider: BlockReader<Block = HlBlock>>
-        + EthBlocks
-        + EthTransactions
-        + LoadReceipt
-        + 'static
-{
-}
+use crate::addons::utils::{EthWrapper, new_headers_stream, pipe_from_stream};
 
 #[rpc(server, namespace = "eth")]
 #[async_trait]
@@ -182,7 +147,7 @@ impl<Eth: EthWrapper> HlSystemTransactionExt<Eth> {
                 };
 
                 let input = ConvertReceiptInput {
-                    receipt: Cow::Borrowed(receipt),
+                    receipt: receipt.clone(),
                     tx,
                     gas_used: receipt.cumulative_gas_used() - gas_used,
                     next_log_index,
@@ -382,6 +347,8 @@ where
                     pubsub.log_stream(filter).filter_map(|log| adjust_log::<Eth>(log, &provider)),
                 )
                 .await;
+            } else if kind == SubscriptionKind::NewHeads {
+                let _ = pipe_from_stream(sink, new_headers_stream::<Eth>(&provider)).await;
             } else {
                 let _ = pubsub.handle_accepted(sink, kind, params).await;
             }
@@ -408,23 +375,6 @@ fn adjust_log<Eth: EthWrapper>(mut log: Log, provider: &Eth::Provider) -> Option
     Some(log)
 }
 
-async fn pipe_from_stream<T: Serialize, St: Stream<Item = T> + Unpin>(
-    sink: SubscriptionSink,
-    mut stream: St,
-) -> Result<(), ErrorObject<'static>> {
-    loop {
-        tokio::select! {
-            _ = sink.closed() => break Ok(()),
-            maybe_item = stream.next() => {
-                let Some(item) = maybe_item else { break Ok(()) };
-                let msg = SubscriptionMessage::new(sink.method_name(), sink.subscription_id(), &item)
-                    .map_err(SubscriptionSerializeError::from)?;
-                if sink.send(msg).await.is_err() { break Ok(()); }
-            }
-        }
-    }
-}
-
 pub struct HlNodeBlockFilterHttp<Eth: EthWrapper> {
     eth_api: Arc<Eth>,
     _marker: PhantomData<Eth>,
@@ -530,7 +480,7 @@ async fn adjust_block_receipts<Eth: EthWrapper>(
                 };
 
                 let input = ConvertReceiptInput {
-                    receipt: Cow::Borrowed(receipt),
+                    receipt: receipt.clone(),
                     tx,
                     gas_used: receipt.cumulative_gas_used() - gas_used,
                     next_log_index,
@@ -575,10 +525,9 @@ async fn adjust_transaction_receipt<Eth: EthWrapper>(
 // This function assumes that `block_id` is already validated by the caller.
 fn system_tx_count_for_block<Eth: EthWrapper>(eth_api: &Eth, block_id: BlockId) -> usize {
     let provider = eth_api.provider();
-    let block = provider.block_by_id(block_id).unwrap().unwrap();
-    let system_tx_count =
-        block.body.transactions().iter().filter(|tx| tx.is_system_transaction()).count();
-    system_tx_count
+    let header = provider.header_by_id(block_id).unwrap().unwrap();
+
+    header.extras.system_tx_count.try_into().unwrap()
 }
 
 #[async_trait]
@@ -652,6 +601,9 @@ where
         block_id: BlockId,
     ) -> RpcResult<Option<Vec<RpcReceipt<Eth::NetworkTypes>>>> {
         trace!(target: "rpc::eth", ?block_id, "Serving eth_getBlockReceipts");
+        if self.eth_api.provider().block_by_id(block_id).map_err(EthApiError::from)?.is_none() {
+            return Ok(None);
+        }
         let result =
             adjust_block_receipts(block_id, &*self.eth_api).instrument(engine_span!()).await?;
         Ok(result.map(|(_, receipts)| receipts))

src/addons/mod.rs

@@ -1,3 +1,5 @@
 pub mod call_forwarder;
 pub mod hl_node_compliance;
+pub mod subscribe_fixup;
 pub mod tx_forwarder;
+mod utils;

src/addons/subscribe_fixup.rs

@@ -0,0 +1,54 @@
+use crate::addons::utils::{EthWrapper, new_headers_stream, pipe_from_stream};
+use alloy_rpc_types::pubsub::{Params, SubscriptionKind};
+use async_trait::async_trait;
+use jsonrpsee::PendingSubscriptionSink;
+use jsonrpsee_types::ErrorObject;
+use reth::tasks::TaskSpawner;
+use reth_rpc::EthPubSub;
+use reth_rpc_convert::RpcTransaction;
+use reth_rpc_eth_api::{EthApiTypes, EthPubSubApiServer};
+use std::sync::Arc;
+
+pub struct SubscribeFixup<Eth: EthWrapper> {
+    pubsub: Arc<EthPubSub<Eth>>,
+    provider: Arc<Eth::Provider>,
+    subscription_task_spawner: Box<dyn TaskSpawner + 'static>,
+}
+
+#[async_trait]
+impl<Eth: EthWrapper> EthPubSubApiServer<RpcTransaction<Eth::NetworkTypes>> for SubscribeFixup<Eth>
+where
+    ErrorObject<'static>: From<<Eth as EthApiTypes>::Error>,
+{
+    async fn subscribe(
+        &self,
+        pending: PendingSubscriptionSink,
+        kind: SubscriptionKind,
+        params: Option<Params>,
+    ) -> jsonrpsee::core::SubscriptionResult {
+        let sink = pending.accept().await?;
+        let (pubsub, provider) = (self.pubsub.clone(), self.provider.clone());
+        self.subscription_task_spawner.spawn(Box::pin(async move {
+            if kind == SubscriptionKind::NewHeads {
+                let _ = pipe_from_stream(sink, new_headers_stream::<Eth>(&provider)).await;
+            } else {
+                let _ = pubsub.handle_accepted(sink, kind, params).await;
+            }
+        }));
+        Ok(())
+    }
+}
+
+impl<Eth: EthWrapper> SubscribeFixup<Eth> {
+    pub fn new(
+        pubsub: Arc<EthPubSub<Eth>>,
+        provider: Arc<Eth::Provider>,
+        subscription_task_spawner: Box<dyn TaskSpawner + 'static>,
+    ) -> Self
+    where
+        Eth: EthWrapper,
+        ErrorObject<'static>: From<Eth::Error>,
+    {
+        Self { pubsub, provider, subscription_task_spawner }
+    }
+}

src/addons/tx_forwarder.rs

@@ -2,14 +2,14 @@ use std::time::Duration;
 
 use alloy_json_rpc::RpcObject;
 use alloy_network::Ethereum;
-use alloy_primitives::{Bytes, B256};
+use alloy_primitives::{B256, Bytes};
 use alloy_rpc_types::TransactionRequest;
 use jsonrpsee::{
     http_client::{HttpClient, HttpClientBuilder},
     proc_macros::rpc,
-    types::{error::INTERNAL_ERROR_CODE, ErrorObject},
+    types::{ErrorObject, error::INTERNAL_ERROR_CODE},
 };
-use jsonrpsee_core::{async_trait, client::ClientT, ClientError, RpcResult};
+use jsonrpsee_core::{ClientError, RpcResult, async_trait, client::ClientT};
 use reth::rpc::{result::internal_rpc_err, server_types::eth::EthApiError};
 use reth_rpc_eth_api::RpcReceipt;
 

src/addons/utils.rs

@@ -0,0 +1,90 @@
+use std::sync::Arc;
+
+use crate::{HlBlock, HlPrimitives};
+use alloy_primitives::U256;
+use alloy_rpc_types::Header;
+use futures::StreamExt;
+use jsonrpsee::{SubscriptionMessage, SubscriptionSink};
+use jsonrpsee_types::ErrorObject;
+use reth_primitives::SealedHeader;
+use reth_provider::{BlockReader, CanonStateSubscriptions};
+use reth_rpc::{RpcTypes, eth::pubsub::SubscriptionSerializeError};
+use reth_rpc_convert::{RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, RpcTxReq};
+use reth_rpc_eth_api::{
+    EthApiServer, FullEthApiTypes, RpcNodeCoreExt,
+    helpers::{EthBlocks, EthTransactions, LoadReceipt},
+};
+use serde::Serialize;
+use tokio_stream::Stream;
+
+pub trait EthWrapper:
+    EthApiServer<
+        RpcTxReq<Self::NetworkTypes>,
+        RpcTransaction<Self::NetworkTypes>,
+        RpcBlock<Self::NetworkTypes>,
+        RpcReceipt<Self::NetworkTypes>,
+        RpcHeader<Self::NetworkTypes>,
+    > + FullEthApiTypes<
+        Primitives = HlPrimitives,
+        NetworkTypes: RpcTypes<TransactionResponse = alloy_rpc_types_eth::Transaction>,
+    > + RpcNodeCoreExt<Provider: BlockReader<Block = HlBlock>>
+    + EthBlocks
+    + EthTransactions
+    + LoadReceipt
+    + 'static
+{
+}
+
+impl<T> EthWrapper for T where
+    T: EthApiServer<
+            RpcTxReq<Self::NetworkTypes>,
+            RpcTransaction<Self::NetworkTypes>,
+            RpcBlock<Self::NetworkTypes>,
+            RpcReceipt<Self::NetworkTypes>,
+            RpcHeader<Self::NetworkTypes>,
+        > + FullEthApiTypes<
+            Primitives = HlPrimitives,
+            NetworkTypes: RpcTypes<TransactionResponse = alloy_rpc_types_eth::Transaction>,
+        > + RpcNodeCoreExt<Provider: BlockReader<Block = HlBlock>>
+        + EthBlocks
+        + EthTransactions
+        + LoadReceipt
+        + 'static
+{
+}
+
+pub(super) async fn pipe_from_stream<T: Serialize, St: Stream<Item = T> + Unpin>(
+    sink: SubscriptionSink,
+    mut stream: St,
+) -> Result<(), ErrorObject<'static>> {
+    loop {
+        tokio::select! {
+            _ = sink.closed() => break Ok(()),
+            maybe_item = stream.next() => {
+                let Some(item) = maybe_item else { break Ok(()) };
+                let msg = SubscriptionMessage::new(sink.method_name(), sink.subscription_id(), &item)
+                    .map_err(SubscriptionSerializeError::from)?;
+                if sink.send(msg).await.is_err() { break Ok(()); }
+            }
+        }
+    }
+}
+
+pub(super) fn new_headers_stream<Eth: EthWrapper>(
+    provider: &Arc<Eth::Provider>,
+) -> impl Stream<Item = Header<alloy_consensus::Header>> {
+    provider.canonical_state_stream().flat_map(|new_chain| {
+        let headers = new_chain
+            .committed()
+            .blocks_iter()
+            .map(|block| {
+                Header::from_consensus(
+                    SealedHeader::new(block.header().inner.clone(), block.hash()).into(),
+                    None,
+                    Some(U256::from(block.rlp_length())),
+                )
+            })
+            .collect::<Vec<_>>();
+        futures::stream::iter(headers)
+    })
+}

src/chainspec/hl.rs

@@ -1,5 +1,5 @@
 use alloy_chains::{Chain, NamedChain};
-use alloy_primitives::{b256, Address, Bytes, B256, B64, U256};
+use alloy_primitives::{Address, B64, B256, Bytes, U256, b256};
 use reth_chainspec::{ChainHardforks, ChainSpec, EthereumHardfork, ForkCondition, Hardfork};
 use reth_primitives::{Header, SealedHeader};
 use std::sync::LazyLock;

src/chainspec/mod.rs

@@ -1,8 +1,10 @@
 pub mod hl;
 pub mod parser;
 
-use crate::hardforks::HlHardforks;
-use alloy_consensus::Header;
+use crate::{
+    hardforks::HlHardforks,
+    node::primitives::{HlHeader, header::HlHeaderExtras},
+};
 use alloy_eips::eip7840::BlobParams;
 use alloy_genesis::Genesis;
 use alloy_primitives::{Address, B256, U256};
@@ -20,10 +22,11 @@ pub const TESTNET_CHAIN_ID: u64 = 998;
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct HlChainSpec {
     pub inner: ChainSpec,
+    pub genesis_header: HlHeader,
 }
 
 impl EthChainSpec for HlChainSpec {
-    type Header = Header;
+    type Header = HlHeader;
 
     fn blob_params_at_timestamp(&self, timestamp: u64) -> Option<BlobParams> {
         self.inner.blob_params_at_timestamp(timestamp)
@@ -57,8 +60,8 @@ impl EthChainSpec for HlChainSpec {
         Box::new(self.inner.display_hardforks())
     }
 
-    fn genesis_header(&self) -> &Header {
-        self.inner.genesis_header()
+    fn genesis_header(&self) -> &HlHeader {
+        &self.genesis_header
     }
 
     fn genesis(&self) -> &Genesis {
@@ -127,4 +130,10 @@ impl HlChainSpec {
             _ => unreachable!("Unreachable since ChainSpecParser won't return other chains"),
         }
     }
+
+    fn new(inner: ChainSpec) -> Self {
+        let genesis_header =
+            HlHeader { inner: inner.genesis_header().clone(), extras: HlHeaderExtras::default() };
+        Self { inner, genesis_header }
+    }
 }

src/chainspec/parser.rs

@@ -1,4 +1,4 @@
-use crate::chainspec::{hl::hl_testnet, HlChainSpec};
+use crate::chainspec::{HlChainSpec, hl::hl_testnet};
 
 use super::hl::hl_mainnet;
 use reth_cli::chainspec::ChainSpecParser;
@@ -26,8 +26,8 @@ impl ChainSpecParser for HlChainSpecParser {
 /// Currently only mainnet is supported.
 pub fn chain_value_parser(s: &str) -> eyre::Result<Arc<HlChainSpec>> {
     match s {
-        "mainnet" => Ok(Arc::new(HlChainSpec { inner: hl_mainnet() })),
-        "testnet" => Ok(Arc::new(HlChainSpec { inner: hl_testnet() })),
+        "mainnet" => Ok(Arc::new(HlChainSpec::new(hl_mainnet()))),
+        "testnet" => Ok(Arc::new(HlChainSpec::new(hl_testnet()))),
         _ => Err(eyre::eyre!("Unsupported chain: {}", s)),
     }
 }

src/consensus.rs

@@ -1,4 +1,4 @@
-use alloy_primitives::{BlockNumber, B256};
+use alloy_primitives::{B256, BlockNumber};
 use reth_provider::{BlockNumReader, ProviderError};
 use std::cmp::Ordering;
 

src/evm/api/builder.rs

@@ -2,8 +2,8 @@ use super::HlEvmInner;
 use crate::evm::{spec::HlSpecId, transaction::HlTxTr};
 use reth_revm::context::ContextTr;
 use revm::{
-    context::Cfg, context_interface::Block, handler::instructions::EthInstructions,
-    interpreter::interpreter::EthInterpreter, Context, Database,
+    Context, Database, context::Cfg, context_interface::Block,
+    handler::instructions::EthInstructions, interpreter::interpreter::EthInterpreter,
 };
 
 /// Trait that allows for hl HlEvm to be built.

src/evm/api/ctx.rs

@@ -1,8 +1,8 @@
 use crate::evm::{spec::HlSpecId, transaction::HlTxEnv};
 use revm::{
+    Context, Journal, MainContext,
     context::{BlockEnv, CfgEnv, TxEnv},
     database_interface::EmptyDB,
-    Context, Journal, MainContext,
 };
 
 /// Type alias for the default context type of the HlEvm.

src/evm/api/exec.rs

@@ -1,16 +1,16 @@
 use super::HlEvmInner;
 use crate::evm::{spec::HlSpecId, transaction::HlTxTr};
 use revm::{
-    context::{result::HaltReason, ContextSetters},
-    context_interface::{
-        result::{EVMError, ExecutionResult, ResultAndState},
-        Cfg, ContextTr, Database, JournalTr,
-    },
-    handler::{instructions::EthInstructions, PrecompileProvider},
-    inspector::{InspectCommitEvm, InspectEvm, Inspector, JournalExt},
-    interpreter::{interpreter::EthInterpreter, InterpreterResult},
-    state::EvmState,
     DatabaseCommit, ExecuteCommitEvm, ExecuteEvm,
+    context::{ContextSetters, result::HaltReason},
+    context_interface::{
+        Cfg, ContextTr, Database, JournalTr,
+        result::{EVMError, ExecutionResult, ResultAndState},
+    },
+    handler::{PrecompileProvider, instructions::EthInstructions},
+    inspector::{InspectCommitEvm, InspectEvm, Inspector, JournalExt},
+    interpreter::{InterpreterResult, interpreter::EthInterpreter},
+    state::EvmState,
 };
 
 // Type alias for HL context

src/evm/api/mod.rs

@@ -1,15 +1,15 @@
 use revm::{
+    Inspector,
     bytecode::opcode::BLOCKHASH,
     context::{ContextSetters, Evm, FrameStack},
     context_interface::ContextTr,
     handler::{
+        EthFrame, EthPrecompiles, EvmTr, FrameInitOrResult, FrameTr, PrecompileProvider,
         evm::{ContextDbError, FrameInitResult},
         instructions::{EthInstructions, InstructionProvider},
-        EthFrame, EthPrecompiles, EvmTr, FrameInitOrResult, FrameTr, PrecompileProvider,
     },
     inspector::{InspectorEvmTr, JournalExt},
-    interpreter::{interpreter::EthInterpreter, Instruction, InterpreterResult},
-    Inspector,
+    interpreter::{Instruction, InterpreterResult, interpreter::EthInterpreter},
 };
 
 use crate::chainspec::MAINNET_CHAIN_ID;

src/evm/api/patch.rs

@@ -7,36 +7,12 @@ use alloy_primitives::keccak256;
 use revm::{
     context::Host,
     interpreter::{
-        as_u64_saturated, interpreter_types::StackTr, popn_top, InstructionContext,
-        InterpreterTypes,
+        _count, InstructionContext, InterpreterTypes, as_u64_saturated, interpreter_types::StackTr,
+        popn_top,
     },
     primitives::{BLOCK_HASH_HISTORY, U256},
 };
 
-#[doc(hidden)]
-#[macro_export]
-#[collapse_debuginfo(yes)]
-macro_rules! _count {
-    (@count) => { 0 };
-    (@count $head:tt $($tail:tt)*) => { 1 + _count!(@count $($tail)*) };
-    ($($arg:tt)*) => { _count!(@count $($arg)*) };
-}
-
-/// Pops n values from the stack and returns the top value. Fails the instruction if n values can't
-/// be popped.
-#[macro_export]
-#[collapse_debuginfo(yes)]
-macro_rules! popn_top {
-    ([ $($x:ident),* ], $top:ident, $interpreter:expr $(,$ret:expr)? ) => {
-        // Workaround for https://github.com/rust-lang/rust/issues/144329.
-        if $interpreter.stack.len() < (1 + $crate::_count!($($x)*)) {
-            $interpreter.halt_underflow();
-            return $($ret)?;
-        }
-        let ([$( $x ),*], $top) = unsafe { $interpreter.stack.popn_top().unwrap_unchecked() };
-    };
-}
-
 /// Implements the BLOCKHASH instruction.
 ///
 /// Gets the hash of one of the 256 most recent complete blocks.

src/evm/transaction.rs

@@ -7,7 +7,7 @@ use reth_primitives_traits::SignerRecoverable;
 use revm::{
     context::TxEnv,
     context_interface::transaction::Transaction,
-    primitives::{Address, Bytes, TxKind, B256, U256},
+    primitives::{Address, B256, Bytes, TxKind, U256},
 };
 
 #[auto_impl(&, &mut, Box, Arc)]

src/hardforks/hl.rs

@@ -2,7 +2,7 @@
 use alloy_chains::{Chain, NamedChain};
 use core::any::Any;
 use reth_chainspec::ForkCondition;
-use reth_ethereum_forks::{hardfork, ChainHardforks, EthereumHardfork, Hardfork};
+use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, Hardfork, hardfork};
 
 hardfork!(
     /// The name of a hl hardfork.

src/lib.rs

@@ -7,4 +7,4 @@ pub mod node;
 pub mod pseudo_peer;
 pub mod version;
 
-pub use node::primitives::{HlBlock, HlBlockBody, HlPrimitives};
+pub use node::primitives::{HlBlock, HlBlockBody, HlHeader, HlPrimitives};

src/main.rs

@@ -1,20 +1,26 @@
 use std::sync::Arc;
 
 use clap::Parser;
-use reth::builder::{NodeBuilder, NodeHandle, WithLaunchContext};
+use reth::{
+    builder::{NodeBuilder, NodeHandle, WithLaunchContext},
+    rpc::{api::EthPubSubApiServer, eth::RpcNodeCore},
+};
 use reth_db::DatabaseEnv;
 use reth_hl::{
     addons::{
         call_forwarder::{self, CallForwarderApiServer},
         hl_node_compliance::install_hl_node_compliance,
+        subscribe_fixup::SubscribeFixup,
         tx_forwarder::{self, EthForwarderApiServer},
     },
-    chainspec::{parser::HlChainSpecParser, HlChainSpec},
+    chainspec::{HlChainSpec, parser::HlChainSpecParser},
     node::{
+        HlNode,
         cli::{Cli, HlNodeArgs},
         rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt},
+        spot_meta::init as spot_meta_init,
         storage::tables::Tables,
-        HlNode,
+        types::set_spot_metadata_db,
     },
 };
 use tracing::info;
@@ -27,11 +33,6 @@ static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 fn main() -> eyre::Result<()> {
     reth_cli_util::sigsegv_handler::install();
 
-    // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
-    if std::env::var_os("RUST_BACKTRACE").is_none() {
-        std::env::set_var("RUST_BACKTRACE", "1");
-    }
-
     // Initialize custom version metadata before parsing CLI so --version uses reth-hl values
     reth_hl::version::init_reth_hl_version();
 
@@ -40,8 +41,11 @@ fn main() -> eyre::Result<()> {
          ext: HlNodeArgs| async move {
             let default_upstream_rpc_url = builder.config().chain.official_rpc_url();
 
-            let (node, engine_handle_tx) =
-                HlNode::new(ext.block_source_args.parse().await?, ext.debug_cutoff_height);
+            let (node, engine_handle_tx) = HlNode::new(
+                ext.block_source_args.parse().await?,
+                ext.debug_cutoff_height,
+                ext.allow_network_overrides,
+            );
             let NodeHandle { node, node_exit_future: exit_future } = builder
                 .node(node)
                 .extend_rpc_modules(move |mut ctx| {
@@ -64,6 +68,17 @@ fn main() -> eyre::Result<()> {
                         info!("Call/gas estimation will be forwarded to {}", upstream_rpc_url);
                     }
 
+                    // This is a temporary workaround to fix the issue with custom headers
+                    // affects `eth_subscribe[type=newHeads]`
+                    ctx.modules.replace_configured(
+                        SubscribeFixup::new(
+                            Arc::new(ctx.registry.eth_handlers().pubsub.clone()),
+                            Arc::new(ctx.registry.eth_api().provider().clone()),
+                            Box::new(ctx.node().task_executor.clone()),
+                        )
+                        .into_rpc(),
+                    )?;
+
                     if ext.hl_node_compliant {
                         install_hl_node_compliance(&mut ctx)?;
                         info!("hl-node compliant mode enabled");
@@ -80,8 +95,18 @@ fn main() -> eyre::Result<()> {
 
                     Ok(())
                 })
-                .apply(|builder| {
-                    builder.db().create_tables_for::<Tables>().expect("create tables");
+                .apply(|mut builder| {
+                    builder.db_mut().create_tables_for::<Tables>().expect("create tables");
+
+                    let chain_id = builder.config().chain.inner.chain().id();
+                    let db = builder.db_mut().clone();
+
+                    // Set database handle for on-demand persistence
+                    set_spot_metadata_db(db.clone());
+
+                    // Load spot metadata from database and initialize cache
+                    spot_meta_init::load_spot_metadata_cache(&db, chain_id);
+
                     builder
                 })
                 .launch()

src/node/cli.rs

@@ -1,21 +1,24 @@
 use crate::{
-    chainspec::{parser::HlChainSpecParser, HlChainSpec},
-    node::{consensus::HlConsensus, evm::config::HlEvmConfig, storage::tables::Tables, HlNode},
+    chainspec::{HlChainSpec, parser::HlChainSpecParser},
+    node::{
+        HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator,
+        spot_meta::init as spot_meta_init, storage::tables::Tables,
+    },
     pseudo_peer::BlockSourceArgs,
 };
 use clap::{Args, Parser};
 use reth::{
-    args::LogArgs,
+    CliRunner,
+    args::{DatabaseArgs, DatadirArgs, LogArgs},
     builder::{NodeBuilder, WithLaunchContext},
     cli::Commands,
     prometheus_exporter::install_prometheus_recorder,
     version::version_metadata,
-    CliRunner,
 };
 use reth_chainspec::EthChainSpec;
 use reth_cli::chainspec::ChainSpecParser;
 use reth_cli_commands::{common::EnvironmentArgs, launcher::FnLauncher};
-use reth_db::{init_db, mdbx::init_db_for, DatabaseEnv};
+use reth_db::{DatabaseEnv, init_db, mdbx::init_db_for};
 use reth_tracing::FileWorkerGuard;
 use std::{
     fmt::{self},
@@ -79,6 +82,13 @@ pub struct HlNodeArgs {
     /// * Refers to the Merkle trie used for eth_getProof and state root, not actual state values.
     #[arg(long, env = "EXPERIMENTAL_ETH_GET_PROOF")]
     pub experimental_eth_get_proof: bool,
+
+    /// Allow network configuration overrides from CLI.
+    ///
+    /// When enabled, network settings (discovery_addr, listener_addr, dns_discovery, nat)
+    /// will be taken from CLI arguments instead of being hardcoded to localhost-only defaults.
+    #[arg(long, env = "ALLOW_NETWORK_OVERRIDES")]
+    pub allow_network_overrides: bool,
 }
 
 /// The main reth_hl cli interface.
@@ -136,11 +146,18 @@ where
         // Install the prometheus recorder to be sure to record all metrics
         let _ = install_prometheus_recorder();
 
-        let components =
-            |spec: Arc<C::ChainSpec>| (HlEvmConfig::new(spec.clone()), HlConsensus::new(spec));
+        let components = |spec: Arc<C::ChainSpec>| {
+            (HlEvmConfig::new(spec.clone()), Arc::new(HlConsensus::new(spec)))
+        };
 
         match self.command {
             Commands::Node(command) => runner.run_command_until_exit(|ctx| {
+                // NOTE: This is for one time migration around Oct 10 upgrade:
+                // It's not necessary anymore, an environment variable gate is added here.
+                if std::env::var("CHECK_DB_MIGRATION").is_ok() {
+                    Self::migrate_db(&command.chain, &command.datadir, &command.db)
+                        .expect("Failed to migrate database");
+                }
                 command.execute(ctx, FnLauncher::new::<C, Ext>(launcher))
             }),
             Commands::Init(command) => {
@@ -157,9 +174,6 @@ where
                 runner.run_command_until_exit(|ctx| command.execute::<HlNode, _>(ctx, components))
             }
             Commands::Config(command) => runner.run_until_ctrl_c(command.execute()),
-            Commands::Recover(command) => {
-                runner.run_command_until_exit(|ctx| command.execute::<HlNode>(ctx))
-            }
             Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<HlNode>()),
             Commands::Import(command) => {
                 runner.run_blocking_until_ctrl_c(command.execute::<HlNode, _>(components))
@@ -187,7 +201,21 @@ where
         let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain());
         let db_path = data_dir.db();
         init_db(db_path.clone(), env.db.database_args())?;
-        init_db_for::<_, Tables>(db_path, env.db.database_args())?;
+        init_db_for::<_, Tables>(db_path.clone(), env.db.database_args())?;
+
+        // Initialize spot metadata in database
+        let chain_id = env.chain.chain().id();
+        spot_meta_init::init_spot_metadata(db_path, env.db.database_args(), chain_id)?;
+
+        Ok(())
+    }
+
+    fn migrate_db(
+        chain: &HlChainSpec,
+        datadir: &DatadirArgs,
+        db: &DatabaseArgs,
+    ) -> eyre::Result<()> {
+        Migrator::<HlNode>::new(chain.clone(), datadir.clone(), *db)?.migrate_db()?;
         Ok(())
     }
 }

src/node/consensus/mod.rs

@@ -1,9 +1,12 @@
-use crate::{hardforks::HlHardforks, node::HlNode, HlBlock, HlBlockBody, HlPrimitives};
-use alloy_consensus::Header;
+use crate::{
+    HlBlock, HlBlockBody, HlPrimitives,
+    hardforks::HlHardforks,
+    node::{HlNode, primitives::HlHeader},
+};
 use reth::{
-    api::FullNodeTypes,
+    api::{FullNodeTypes, NodeTypes},
     beacon_consensus::EthBeaconConsensus,
-    builder::{components::ConsensusBuilder, BuilderContext},
+    builder::{BuilderContext, components::ConsensusBuilder},
     consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator},
     consensus_common::validation::{
         validate_against_parent_4844, validate_against_parent_hash_number,
@@ -24,7 +27,7 @@ impl<Node> ConsensusBuilder<Node> for HlConsensusBuilder
 where
     Node: FullNodeTypes<Types = HlNode>,
 {
-    type Consensus = Arc<dyn FullConsensus<HlPrimitives, Error = ConsensusError>>;
+    type Consensus = Arc<HlConsensus<<Node::Types as NodeTypes>::ChainSpec>>;
 
     async fn build_consensus(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Consensus> {
         Ok(Arc::new(HlConsensus::new(ctx.chain_spec())))
@@ -101,14 +104,14 @@ where
 
 impl<ChainSpec> Consensus<HlBlock> for HlConsensus<ChainSpec>
 where
-    ChainSpec: EthChainSpec<Header = Header> + HlHardforks,
+    ChainSpec: EthChainSpec<Header = HlHeader> + HlHardforks,
 {
     type Error = ConsensusError;
 
     fn validate_body_against_header(
         &self,
         body: &HlBlockBody,
-        header: &SealedHeader,
+        header: &SealedHeader<HlHeader>,
     ) -> Result<(), ConsensusError> {
         Consensus::<HlBlock>::validate_body_against_header(&self.inner, body, header)
     }
@@ -148,7 +151,7 @@ mod reth_copy;
 
 impl<ChainSpec> FullConsensus<HlPrimitives> for HlConsensus<ChainSpec>
 where
-    ChainSpec: EthChainSpec<Header = Header> + HlHardforks,
+    ChainSpec: EthChainSpec<Header = HlHeader> + HlHardforks,
 {
     fn validate_block_post_execution(
         &self,

src/node/consensus/reth_copy.rs

@@ -1,21 +1,21 @@
 //! Copy of reth codebase.
 
-use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt};
+use crate::HlBlock;
+use alloy_consensus::{BlockHeader, TxReceipt, proofs::calculate_receipt_root};
 use alloy_eips::eip7685::Requests;
-use alloy_primitives::{Bloom, B256};
+use alloy_primitives::{B256, Bloom};
 use reth::consensus::ConsensusError;
 use reth_chainspec::EthereumHardforks;
-use reth_primitives::{gas_spent_by_transactions, GotExpected, RecoveredBlock};
-use reth_primitives_traits::{Block, Receipt as ReceiptTrait};
+use reth_primitives::{GotExpected, RecoveredBlock, gas_spent_by_transactions};
+use reth_primitives_traits::Receipt as ReceiptTrait;
 
-pub fn validate_block_post_execution<B, R, ChainSpec>(
-    block: &RecoveredBlock<B>,
+pub fn validate_block_post_execution<R, ChainSpec>(
+    block: &RecoveredBlock<HlBlock>,
     chain_spec: &ChainSpec,
     receipts: &[R],
     requests: &Requests,
 ) -> Result<(), ConsensusError>
 where
-    B: Block,
     R: ReceiptTrait,
     ChainSpec: EthereumHardforks,
 {
@@ -42,7 +42,7 @@ where
             receipts.iter().filter(|&r| r.cumulative_gas_used() != 0).cloned().collect::<Vec<_>>();
         if let Err(error) = verify_receipts(
             block.header().receipts_root(),
-            block.header().logs_bloom(),
+            block.header().inner.logs_bloom(),
             &receipts_for_root,
         ) {
             tracing::debug!(%error, ?receipts, "receipts verification failed");

src/node/evm/assembler.rs

@@ -1,8 +1,7 @@
 use crate::{
+    HlBlock, HlHeader,
     node::evm::config::{HlBlockExecutorFactory, HlEvmConfig},
-    HlBlock,
 };
-use alloy_consensus::Header;
 use reth_evm::{
     block::BlockExecutionError,
     execute::{BlockAssembler, BlockAssemblerInput},
@@ -13,7 +12,7 @@ impl BlockAssembler<HlBlockExecutorFactory> for HlEvmConfig {
 
     fn assemble_block(
         &self,
-        input: BlockAssemblerInput<'_, '_, HlBlockExecutorFactory, Header>,
+        input: BlockAssemblerInput<'_, '_, HlBlockExecutorFactory, HlHeader>,
     ) -> Result<Self::Block, BlockExecutionError> {
         let HlBlock { header, body } = self.block_assembler.assemble_block(input)?;
         Ok(HlBlock { header, body })

src/node/evm/config.rs

@@ -1,5 +1,6 @@
 use super::{executor::HlBlockExecutor, factory::HlEvmFactory};
 use crate::{
+    HlBlock, HlBlockBody, HlHeader, HlPrimitives,
     chainspec::HlChainSpec,
     evm::{spec::HlSpecId, transaction::HlTxEnv},
     hardforks::HlHardforks,
@@ -9,31 +10,30 @@ use crate::{
         rpc::engine_api::validator::HlExecutionData,
         types::HlExtras,
     },
-    HlBlock, HlBlockBody, HlPrimitives,
 };
-use alloy_consensus::{BlockHeader, Header, Transaction as _, TxReceipt, EMPTY_OMMER_ROOT_HASH};
-use alloy_eips::{merge::BEACON_NONCE, Encodable2718};
+use alloy_consensus::{BlockHeader, EMPTY_OMMER_ROOT_HASH, Header, Transaction as _, TxReceipt};
+use alloy_eips::{Encodable2718, merge::BEACON_NONCE};
 use alloy_primitives::{Log, U256};
 use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
 use reth_evm::{
-    block::{BlockExecutionError, BlockExecutorFactory, BlockExecutorFor},
-    eth::{receipt_builder::ReceiptBuilder, EthBlockExecutionCtx},
-    execute::{BlockAssembler, BlockAssemblerInput},
-    precompiles::PrecompilesMap,
     ConfigureEngineEvm, ConfigureEvm, EvmEnv, EvmEnvFor, EvmFactory, ExecutableTxIterator,
     ExecutionCtxFor, FromRecoveredTx, FromTxWithEncoded, IntoTxEnv, NextBlockEnvAttributes,
+    block::{BlockExecutionError, BlockExecutorFactory, BlockExecutorFor},
+    eth::{EthBlockExecutionCtx, receipt_builder::ReceiptBuilder},
+    execute::{BlockAssembler, BlockAssemblerInput},
+    precompiles::PrecompilesMap,
 };
 use reth_evm_ethereum::EthBlockAssembler;
 use reth_payload_primitives::NewPayloadError;
-use reth_primitives::{logs_bloom, BlockTy, HeaderTy, Receipt, SealedBlock, SealedHeader};
-use reth_primitives_traits::{proofs, SignerRecoverable, WithEncoded};
+use reth_primitives::{BlockTy, HeaderTy, Receipt, SealedBlock, SealedHeader, logs_bloom};
+use reth_primitives_traits::{SignerRecoverable, WithEncoded, proofs};
 use reth_provider::BlockExecutionResult;
 use reth_revm::State;
 use revm::{
+    Inspector,
     context::{BlockEnv, CfgEnv, TxEnv},
     context_interface::block::BlobExcessGasAndPrice,
     primitives::hardfork::SpecId,
-    Inspector,
 };
 use std::{borrow::Cow, convert::Infallible, sync::Arc};
 
@@ -54,7 +54,7 @@ where
 
     fn assemble_block(
         &self,
-        input: BlockAssemblerInput<'_, '_, F>,
+        input: BlockAssemblerInput<'_, '_, F, HlHeader>,
     ) -> Result<Self::Block, BlockExecutionError> {
         // TODO: Copy of EthBlockAssembler::assemble_block
         let inner = &self.inner;
@@ -106,7 +106,10 @@ where
             } else {
                 // for the first post-fork block, both parent.blob_gas_used and
                 // parent.excess_blob_gas are evaluated as 0
-                Some(alloy_eips::eip7840::BlobParams::cancun().next_block_excess_blob_gas(0, 0))
+                Some(
+                    alloy_eips::eip7840::BlobParams::cancun()
+                        .next_block_excess_blob_gas_osaka(0, 0, 0),
+                )
             };
         }
 
@@ -133,6 +136,9 @@ where
             excess_blob_gas,
             requests_hash,
         };
+        let system_tx_count =
+            transactions.iter().filter(|t| is_system_transaction(t)).count() as u64;
+        let header = HlHeader::from_ethereum_header(header, receipts, system_tx_count);
 
         Ok(Self::Block {
             header,
@@ -266,6 +272,8 @@ where
     }
 }
 
+static EMPTY_OMMERS: [Header; 0] = [];
+
 impl ConfigureEvm for HlEvmConfig
 where
     Self: Send + Sync + Unpin + Clone + 'static,
@@ -284,7 +292,7 @@ where
         self
     }
 
-    fn evm_env(&self, header: &Header) -> EvmEnv<HlSpecId> {
+    fn evm_env(&self, header: &HlHeader) -> Result<EvmEnv<HlSpecId>, Self::Error> {
         let blob_params = self.chain_spec().blob_params_at_timestamp(header.timestamp);
         let spec = revm_spec_by_timestamp_and_block_number(
             self.chain_spec().clone(),
@@ -324,12 +332,12 @@ where
             blob_excess_gas_and_price,
         };
 
-        EvmEnv { cfg_env, block_env }
+        Ok(EvmEnv { cfg_env, block_env })
     }
 
     fn next_evm_env(
         &self,
-        parent: &Header,
+        parent: &HlHeader,
         attributes: &Self::NextBlockEnvCtx,
     ) -> Result<EvmEnv<HlSpecId>, Self::Error> {
         // ensure we're not missing any timestamp based hardforks
@@ -373,28 +381,28 @@ where
     fn context_for_block<'a>(
         &self,
         block: &'a SealedBlock<BlockTy<Self::Primitives>>,
-    ) -> ExecutionCtxFor<'a, Self> {
+    ) -> Result<ExecutionCtxFor<'a, Self>, Self::Error> {
         let block_body = block.body();
-        HlBlockExecutionCtx {
+        Ok(HlBlockExecutionCtx {
             ctx: EthBlockExecutionCtx {
                 parent_hash: block.header().parent_hash,
                 parent_beacon_block_root: block.header().parent_beacon_block_root,
-                ommers: &block.body().ommers,
+                ommers: &EMPTY_OMMERS,
                 withdrawals: block.body().withdrawals.as_ref().map(Cow::Borrowed),
             },
             extras: HlExtras {
                 read_precompile_calls: block_body.read_precompile_calls.clone(),
                 highest_precompile_address: block_body.highest_precompile_address,
             },
-        }
+        })
     }
 
     fn context_for_next_block(
         &self,
         parent: &SealedHeader<HeaderTy<Self::Primitives>>,
         attributes: Self::NextBlockEnvCtx,
-    ) -> ExecutionCtxFor<'_, Self> {
-        HlBlockExecutionCtx {
+    ) -> Result<ExecutionCtxFor<'_, Self>, Self::Error> {
+        Ok(HlBlockExecutionCtx {
             ctx: EthBlockExecutionCtx {
                 parent_hash: parent.hash(),
                 parent_beacon_block_root: attributes.parent_beacon_block_root,
@@ -402,13 +410,13 @@ where
                 withdrawals: attributes.withdrawals.map(Cow::Owned),
             },
             extras: HlExtras::default(), // TODO: hacky, double check if this is correct
-        }
+        })
     }
 }
 
 impl ConfigureEngineEvm<HlExecutionData> for HlEvmConfig {
     fn evm_env_for_payload(&self, payload: &HlExecutionData) -> EvmEnvFor<Self> {
-        self.evm_env(&payload.0.header)
+        self.evm_env(&payload.0.header).unwrap()
     }
 
     fn context_for_payload<'a>(&self, payload: &'a HlExecutionData) -> ExecutionCtxFor<'a, Self> {
@@ -417,7 +425,7 @@ impl ConfigureEngineEvm<HlExecutionData> for HlEvmConfig {
             ctx: EthBlockExecutionCtx {
                 parent_hash: block.header.parent_hash,
                 parent_beacon_block_root: block.header.parent_beacon_block_root,
-                ommers: &block.body.ommers,
+                ommers: &EMPTY_OMMERS,
                 withdrawals: block.body.withdrawals.as_ref().map(Cow::Borrowed),
             },
             extras: HlExtras {

src/node/evm/executor.rs

@@ -8,29 +8,26 @@ use crate::{
     },
 };
 use alloy_consensus::{Transaction, TxReceipt};
-use alloy_eips::{eip7685::Requests, Encodable2718};
+use alloy_eips::{Encodable2718, eip7685::Requests};
 use alloy_evm::{block::ExecutableTx, eth::receipt_builder::ReceiptBuilderCtx};
-use alloy_primitives::{address, hex, Address, Bytes, U160, U256};
+use alloy_primitives::{Address, Bytes, U160, U256, address, hex};
 use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
 use reth_evm::{
-    block::{BlockValidationError, CommitChanges},
+    Database, Evm, FromRecoveredTx, FromTxWithEncoded, IntoTxEnv, OnStateHook,
+    block::BlockValidationError,
     eth::receipt_builder::ReceiptBuilder,
     execute::{BlockExecutionError, BlockExecutor},
     precompiles::{DynPrecompile, PrecompileInput, PrecompilesMap},
-    Database, Evm, FromRecoveredTx, FromTxWithEncoded, IntoTxEnv, OnStateHook,
 };
 use reth_provider::BlockExecutionResult;
 use reth_revm::State;
 use revm::{
-    context::{
-        result::{ExecutionResult, ResultAndState},
-        TxEnv,
-    },
+    DatabaseCommit,
+    context::{TxEnv, result::ResultAndState},
     interpreter::instructions::utility::IntoU256,
     precompile::{PrecompileError, PrecompileOutput, PrecompileResult},
     primitives::HashMap,
     state::Bytecode,
-    DatabaseCommit,
 };
 
 pub fn is_system_transaction(tx: &TransactionSigned) -> bool {
@@ -110,7 +107,9 @@ where
         const COREWRITER_ENABLED_BLOCK_NUMBER: u64 = 7578300;
         const COREWRITER_CONTRACT_ADDRESS: Address =
             address!("0x3333333333333333333333333333333333333333");
-        const COREWRITER_CODE: &[u8] = &hex!("608060405234801561000f575f5ffd5b5060043610610029575f3560e01c806317938e131461002d575b5f5ffd5b61004760048036038101906100429190610123565b610049565b005b5f5f90505b61019081101561006557808060010191505061004e565b503373ffffffffffffffffffffffffffffffffffffffff167f8c7f585fb295f7eb1e6aeb8fba61b23a4fe60beda405f0045073b185c74412e383836040516100ae9291906101c8565b60405180910390a25050565b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5f83601f8401126100e3576100e26100c2565b5b8235905067ffffffffffffffff811115610100576100ff6100c6565b5b60208301915083600182028301111561011c5761011b6100ca565b5b9250929050565b5f5f60208385031215610139576101386100ba565b5b5f83013567ffffffffffffffff811115610156576101556100be565b5b610162858286016100ce565b92509250509250929050565b5f82825260208201905092915050565b828183375f83830152505050565b5f601f19601f8301169050919050565b5f6101a7838561016e565b93506101b483858461017e565b6101bd8361018c565b840190509392505050565b5f6020820190508181035f8301526101e181848661019c565b9050939250505056fea2646970667358221220f01517e1fbaff8af4bd72cb063cccecbacbb00b07354eea7dd52265d355474fb64736f6c634300081c0033");
+        const COREWRITER_CODE: &[u8] = &hex!(
+            "608060405234801561000f575f5ffd5b5060043610610029575f3560e01c806317938e131461002d575b5f5ffd5b61004760048036038101906100429190610123565b610049565b005b5f5f90505b61019081101561006557808060010191505061004e565b503373ffffffffffffffffffffffffffffffffffffffff167f8c7f585fb295f7eb1e6aeb8fba61b23a4fe60beda405f0045073b185c74412e383836040516100ae9291906101c8565b60405180910390a25050565b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5f83601f8401126100e3576100e26100c2565b5b8235905067ffffffffffffffff811115610100576100ff6100c6565b5b60208301915083600182028301111561011c5761011b6100ca565b5b9250929050565b5f5f60208385031215610139576101386100ba565b5b5f83013567ffffffffffffffff811115610156576101556100be565b5b610162858286016100ce565b92509250509250929050565b5f82825260208201905092915050565b828183375f83830152505050565b5f601f19601f8301169050919050565b5f6101a7838561016e565b93506101b483858461017e565b6101bd8361018c565b840190509392505050565b5f6020820190508181035f8301526101e181848661019c565b9050939250505056fea2646970667358221220f01517e1fbaff8af4bd72cb063cccecbacbb00b07354eea7dd52265d355474fb64736f6c634300081c0033"
+        );
 
         if self.evm.block().number != U256::from(COREWRITER_ENABLED_BLOCK_NUMBER) {
             return Ok(());
@@ -161,11 +160,10 @@ where
         Ok(())
     }
 
-    fn execute_transaction_with_commit_condition(
+    fn execute_transaction_without_commit(
         &mut self,
         tx: impl ExecutableTx<Self>,
-        f: impl FnOnce(&ExecutionResult<<Self::Evm as Evm>::HaltReason>) -> CommitChanges,
-    ) -> Result<Option<u64>, BlockExecutionError> {
+    ) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
         // The sum of the transaction's gas limit, Tg, and the gas utilized in this block prior,
         // must be no greater than the block's gasLimit.
         let block_available_gas = self.evm.block().gas_limit - self.gas_used;
@@ -178,16 +176,20 @@ where
             .into());
         }
 
-        // Execute transaction.
-        let ResultAndState { result, mut state } = self
-            .evm
-            .transact(&tx)
-            .map_err(|err| BlockExecutionError::evm(err, tx.tx().trie_hash()))?;
-
-        if !f(&result).should_commit() {
-            return Ok(None);
+        // Execute transaction and return the result
+        self.evm.transact(&tx).map_err(|err| {
+            let hash = tx.tx().trie_hash();
+            BlockExecutionError::evm(err, hash)
+        })
     }
 
+    fn commit_transaction(
+        &mut self,
+        output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
+        tx: impl ExecutableTx<Self>,
+    ) -> Result<u64, BlockExecutionError> {
+        let ResultAndState { result, mut state } = output;
+
         let gas_used = result.gas_used();
 
         // append gas used
@@ -215,7 +217,7 @@ where
         // Commit the state changes.
         self.evm.db_mut().commit(state);
 
-        Ok(Some(gas_used))
+        Ok(gas_used)
     }
 
     fn finish(self) -> Result<(Self::Evm, BlockExecutionResult<R::Receipt>), BlockExecutionError> {

src/node/evm/factory.rs

@@ -7,16 +7,16 @@ use crate::evm::{
     spec::HlSpecId,
     transaction::HlTxEnv,
 };
-use reth_evm::{precompiles::PrecompilesMap, Database, EvmEnv, EvmFactory};
+use reth_evm::{Database, EvmEnv, EvmFactory, precompiles::PrecompilesMap};
 use reth_revm::Context;
 use revm::{
+    Inspector,
     context::{
-        result::{EVMError, HaltReason},
         TxEnv,
+        result::{EVMError, HaltReason},
     },
     inspector::NoOpInspector,
     precompile::{PrecompileSpecId, Precompiles},
-    Inspector,
 };
 
 /// Factory producing [`HlEvm`].

src/node/evm/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
     evm::{
-        api::{ctx::HlContext, HlEvmInner},
+        api::{HlEvmInner, ctx::HlContext},
         spec::HlSpecId,
         transaction::HlTxEnv,
     },
@@ -10,18 +10,18 @@ use alloy_primitives::{Address, Bytes};
 use config::HlEvmConfig;
 use reth::{
     api::FullNodeTypes,
-    builder::{components::ExecutorBuilder, BuilderContext},
+    builder::{BuilderContext, components::ExecutorBuilder},
 };
 use reth_evm::{Database, Evm, EvmEnv};
 use revm::{
-    context::{
-        result::{EVMError, ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
-        BlockEnv, TxEnv,
-    },
-    handler::{instructions::EthInstructions, EthPrecompiles, PrecompileProvider},
-    interpreter::{interpreter::EthInterpreter, InterpreterResult},
-    state::EvmState,
     Context, ExecuteEvm, InspectEvm, Inspector,
+    context::{
+        BlockEnv, TxEnv,
+        result::{EVMError, ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
+    },
+    handler::{EthPrecompiles, PrecompileProvider, instructions::EthInstructions},
+    interpreter::{InterpreterResult, interpreter::EthInterpreter},
+    state::EvmState,
 };
 use std::ops::{Deref, DerefMut};
 
@@ -98,11 +98,7 @@ where
         &mut self,
         tx: Self::Tx,
     ) -> Result<ResultAndState<Self::HaltReason>, Self::Error> {
-        if self.inspect {
-            self.inner.inspect_tx(tx)
-        } else {
-            self.inner.transact(tx)
-        }
+        if self.inspect { self.inner.inspect_tx(tx) } else { self.inner.transact(tx) }
     }
 
     fn transact_system_call(

src/node/evm/patch.rs

@@ -1,4 +1,4 @@
-use alloy_primitives::{address, Address};
+use alloy_primitives::{Address, address};
 use reth_evm::block::BlockExecutionError;
 use revm::{primitives::HashMap, state::Account};
 

src/node/migrate.rs

@@ -0,0 +1,429 @@
+use alloy_consensus::Header;
+use alloy_primitives::{B256, BlockHash, Bytes, U256, b256, hex::ToHexExt};
+use reth::{
+    api::NodeTypesWithDBAdapter,
+    args::{DatabaseArgs, DatadirArgs},
+    dirs::{ChainPath, DataDirPath},
+};
+use reth_chainspec::EthChainSpec;
+use reth_db::{
+    DatabaseEnv,
+    mdbx::{RO, tx::Tx},
+    models::CompactU256,
+    static_file::iter_static_files,
+    table::Decompress,
+    tables,
+};
+use reth_db_api::{
+    cursor::{DbCursorRO, DbCursorRW},
+    transaction::{DbTx, DbTxMut},
+};
+use reth_errors::ProviderResult;
+use reth_ethereum_primitives::EthereumReceipt;
+use reth_provider::{
+    DatabaseProvider, ProviderFactory, ReceiptProvider, StaticFileProviderFactory,
+    StaticFileSegment, StaticFileWriter,
+    providers::{NodeTypesForProvider, StaticFileProvider},
+    static_file::SegmentRangeInclusive,
+};
+use std::{fs::File, io::Write, path::PathBuf, sync::Arc};
+use tracing::{info, warn};
+
+use crate::{HlHeader, HlPrimitives, chainspec::HlChainSpec};
+
+pub(crate) trait HlNodeType:
+    NodeTypesForProvider<ChainSpec = HlChainSpec, Primitives = HlPrimitives>
+{
+}
+impl<N: NodeTypesForProvider<ChainSpec = HlChainSpec, Primitives = HlPrimitives>> HlNodeType for N {}
+
+pub(super) struct Migrator<N: HlNodeType> {
+    data_dir: ChainPath<DataDirPath>,
+    provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
+}
+
+impl<N: HlNodeType> Migrator<N> {
+    const MIGRATION_PATH_SUFFIX: &'static str = "migration-tmp";
+
+    pub fn new(
+        chain_spec: HlChainSpec,
+        datadir: DatadirArgs,
+        database_args: DatabaseArgs,
+    ) -> eyre::Result<Self> {
+        let data_dir = datadir.clone().resolve_datadir(chain_spec.chain());
+        let provider_factory = Self::provider_factory(chain_spec, datadir, database_args)?;
+        Ok(Self { data_dir, provider_factory })
+    }
+
+    pub fn sf_provider(&self) -> StaticFileProvider<HlPrimitives> {
+        self.provider_factory.static_file_provider()
+    }
+
+    pub fn migrate_db(&self) -> eyre::Result<()> {
+        let is_empty = Self::highest_block_number(&self.sf_provider()).is_none();
+
+        if is_empty {
+            return Ok(());
+        }
+
+        self.migrate_db_inner()
+    }
+
+    fn highest_block_number(sf_provider: &StaticFileProvider<HlPrimitives>) -> Option<u64> {
+        sf_provider.get_highest_static_file_block(StaticFileSegment::Headers)
+    }
+
+    fn migrate_db_inner(&self) -> eyre::Result<()> {
+        let migrated_mdbx = MigratorMdbx::<N>(self).migrate_mdbx()?;
+        let migrated_static_files = MigrateStaticFiles::<N>(self).migrate_static_files()?;
+
+        if migrated_mdbx || migrated_static_files {
+            info!("Database migrated successfully");
+        }
+        Ok(())
+    }
+
+    fn conversion_tmp_dir(&self) -> PathBuf {
+        self.data_dir.data_dir().join(Self::MIGRATION_PATH_SUFFIX)
+    }
+
+    fn provider_factory(
+        chain_spec: HlChainSpec,
+        datadir: DatadirArgs,
+        database_args: DatabaseArgs,
+    ) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>> {
+        let data_dir = datadir.clone().resolve_datadir(chain_spec.chain());
+        let db_env = reth_db::init_db(data_dir.db(), database_args.database_args())?;
+        let static_file_provider = StaticFileProvider::read_only(data_dir.static_files(), false)?;
+        let db = Arc::new(db_env);
+        Ok(ProviderFactory::new(db, Arc::new(chain_spec), static_file_provider))
+    }
+}
+
+struct MigratorMdbx<'a, N: HlNodeType>(&'a Migrator<N>);
+
+impl<'a, N: HlNodeType> MigratorMdbx<'a, N> {
+    fn migrate_mdbx(&self) -> eyre::Result<bool> {
+        // if any header is in old format, we need to migrate it, so we pick the first and last one
+        let db_env = self.0.provider_factory.provider()?;
+        let mut cursor = db_env.tx_ref().cursor_read::<tables::Headers<Bytes>>()?;
+
+        let migration_needed = {
+            let first_is_old = match cursor.first()? {
+                Some((number, header)) => using_old_header(number, &header),
+                None => false,
+            };
+            let last_is_old = match cursor.last()? {
+                Some((number, header)) => using_old_header(number, &header),
+                None => false,
+            };
+            first_is_old || last_is_old
+        };
+
+        if !migration_needed {
+            return Ok(false);
+        }
+
+        check_if_migration_enabled()?;
+
+        self.migrate_mdbx_inner()?;
+        Ok(true)
+    }
+
+    fn migrate_mdbx_inner(&self) -> eyre::Result<()> {
+        // There shouldn't be many headers in mdbx, but using file for safety
+        info!("Old database detected, migrating mdbx...");
+        let conversion_tmp = self.0.conversion_tmp_dir();
+        let tmp_path = conversion_tmp.join("headers.rmp");
+
+        if conversion_tmp.exists() {
+            std::fs::remove_dir_all(&conversion_tmp)?;
+        }
+        std::fs::create_dir_all(&conversion_tmp)?;
+
+        let count = self.export_old_headers(&tmp_path)?;
+        self.import_new_headers(tmp_path, count)?;
+        Ok(())
+    }
+
+    fn export_old_headers(&self, tmp_path: &PathBuf) -> Result<i32, eyre::Error> {
+        let db_env = self.0.provider_factory.provider()?;
+        let mut cursor_read = db_env.tx_ref().cursor_read::<tables::Headers<Bytes>>()?;
+        let mut tmp_writer = File::create(tmp_path)?;
+        let mut count = 0;
+        let old_headers = cursor_read.walk(None)?.filter_map(|row| {
+            let (block_number, header) = row.ok()?;
+            if !using_old_header(block_number, &header) {
+                None
+            } else {
+                Some((block_number, Header::decompress(&header).ok()?))
+            }
+        });
+        for (block_number, header) in old_headers {
+            let receipt =
+                db_env.receipts_by_block(block_number.into())?.expect("Receipt not found");
+            let new_header = to_hl_header(receipt, header);
+            tmp_writer.write_all(&rmp_serde::to_vec(&(block_number, new_header))?)?;
+            count += 1;
+        }
+        Ok(count)
+    }
+
+    fn import_new_headers(&self, tmp_path: PathBuf, count: i32) -> Result<(), eyre::Error> {
+        let mut tmp_reader = File::open(tmp_path)?;
+        let db_env = self.0.provider_factory.provider_rw()?;
+        let mut cursor_write = db_env.tx_ref().cursor_write::<tables::Headers<Bytes>>()?;
+        for _ in 0..count {
+            let (number, header) = rmp_serde::from_read::<_, (u64, HlHeader)>(&mut tmp_reader)?;
+            cursor_write.upsert(number, &rmp_serde::to_vec(&header)?.into())?;
+        }
+        db_env.commit()?;
+        Ok(())
+    }
+}
+
+fn check_if_migration_enabled() -> Result<(), eyre::Error> {
+    if std::env::var("EXPERIMENTAL_MIGRATE_DB").is_err() {
+        let err_msg = concat!(
+            "Detected an old database format but experimental database migration is currently disabled. ",
+            "To enable migration, set EXPERIMENTAL_MIGRATE_DB=1, or alternatively, resync your node (safest option)."
+        );
+        warn!("{}", err_msg);
+        return Err(eyre::eyre!("{}", err_msg));
+    }
+    Ok(())
+}
+
+struct MigrateStaticFiles<'a, N: HlNodeType>(&'a Migrator<N>);
+
+impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> {
+    fn iterate_files_for_segment(
+        &self,
+        block_range: SegmentRangeInclusive,
+        dir: &PathBuf,
+    ) -> eyre::Result<Vec<(PathBuf, String)>> {
+        let prefix = StaticFileSegment::Headers.filename(&block_range);
+
+        let entries = std::fs::read_dir(dir)?
+            .map(|res| res.map(|e| e.path()))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(entries
+            .into_iter()
+            .filter_map(|path| {
+                let file_name = path.file_name().and_then(|f| f.to_str())?;
+                if file_name.starts_with(&prefix) {
+                    Some((path.clone(), file_name.to_string()))
+                } else {
+                    None
+                }
+            })
+            .collect())
+    }
+
+    fn create_placeholder(&self, block_range: SegmentRangeInclusive) -> eyre::Result<()> {
+        // The direction is opposite here
+        let src = self.0.data_dir.static_files();
+        let dst = self.0.conversion_tmp_dir();
+
+        for (src_path, file_name) in self.iterate_files_for_segment(block_range, &src)? {
+            let dst_path = dst.join(file_name);
+            if dst_path.exists() {
+                std::fs::remove_file(&dst_path)?;
+            }
+            std::os::unix::fs::symlink(src_path, dst_path)?;
+        }
+
+        Ok(())
+    }
+
+    fn move_static_files_for_segment(
+        &self,
+        block_range: SegmentRangeInclusive,
+    ) -> eyre::Result<()> {
+        let src = self.0.conversion_tmp_dir();
+        let dst = self.0.data_dir.static_files();
+
+        for (src_path, file_name) in self.iterate_files_for_segment(block_range, &src)? {
+            let dst_path = dst.join(file_name);
+            std::fs::remove_file(&dst_path)?;
+            std::fs::rename(&src_path, &dst_path)?;
+        }
+
+        // Still StaticFileProvider needs the file to exist, so we create a symlink
+        self.create_placeholder(block_range)
+    }
+
+    fn migrate_static_files(&self) -> eyre::Result<bool> {
+        let conversion_tmp = self.0.conversion_tmp_dir();
+        let old_path = self.0.data_dir.static_files();
+
+        if conversion_tmp.exists() {
+            std::fs::remove_dir_all(&conversion_tmp)?;
+        }
+        std::fs::create_dir_all(&conversion_tmp)?;
+
+        let mut all_static_files = iter_static_files(&old_path)?;
+        let all_static_files =
+            all_static_files.remove(&StaticFileSegment::Headers).unwrap_or_default();
+
+        let mut first = true;
+
+        for (block_range, _tx_ranges) in all_static_files {
+            let migration_needed = self.using_old_header(block_range.start())? ||
+                self.using_old_header(block_range.end())?;
+            if !migration_needed {
+                // Create a placeholder symlink
+                self.create_placeholder(block_range)?;
+                continue;
+            }
+
+            if first {
+                check_if_migration_enabled()?;
+
+                info!("Old database detected, migrating static files...");
+                first = false;
+            }
+
+            let sf_provider = self.0.sf_provider();
+            let sf_tmp_provider = StaticFileProvider::<HlPrimitives>::read_write(&conversion_tmp)?;
+            let provider = self.0.provider_factory.provider()?;
+            let block_range_for_filename = sf_provider.find_fixed_range(block_range.start());
+            migrate_single_static_file(&sf_tmp_provider, &sf_provider, &provider, block_range)?;
+
+            self.move_static_files_for_segment(block_range_for_filename)?;
+        }
+
+        Ok(!first)
+    }
+
+    fn using_old_header(&self, number: u64) -> eyre::Result<bool> {
+        let sf_provider = self.0.sf_provider();
+        let content = old_headers_range(&sf_provider, number..=number)?;
+
+        let &[row] = &content.as_slice() else {
+            warn!("No header found for block {}", number);
+            return Ok(false);
+        };
+
+        Ok(using_old_header(number, &row[0]))
+    }
+}
+
+// Problem is that decompress just panics when the header is not valid
+// So we need heuristics...
+fn is_old_header(header: &[u8]) -> bool {
+    const SHA3_UNCLE_OFFSET: usize = 0x24;
+    const SHA3_UNCLE_HASH: B256 =
+        b256!("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347");
+    const GENESIS_PREFIX: [u8; 4] = [0x01, 0x20, 0x00, 0xf8];
+    let Some(sha3_uncle_hash) = header.get(SHA3_UNCLE_OFFSET..SHA3_UNCLE_OFFSET + 32) else {
+        return false;
+    };
+    if sha3_uncle_hash == SHA3_UNCLE_HASH {
+        return true;
+    }
+
+    // genesis block might be different
+    if header.starts_with(&GENESIS_PREFIX) {
+        return true;
+    }
+
+    false
+}
+
+fn is_new_header(header: &[u8]) -> bool {
+    rmp_serde::from_slice::<HlHeader>(header).is_ok()
+}
+
+fn migrate_single_static_file<N: HlNodeType>(
+    sf_out: &StaticFileProvider<HlPrimitives>,
+    sf_in: &StaticFileProvider<HlPrimitives>,
+    provider: &DatabaseProvider<Tx<RO>, NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
+    block_range: SegmentRangeInclusive,
+) -> Result<(), eyre::Error> {
+    info!("Migrating block range {}...", block_range);
+
+    // block_ranges into chunks of 50000 blocks
+    const CHUNK_SIZE: u64 = 50000;
+    for chunk in (block_range.start()..=block_range.end()).step_by(CHUNK_SIZE as usize) {
+        let end = std::cmp::min(chunk + CHUNK_SIZE - 1, block_range.end());
+        let block_range = chunk..=end;
+        let headers = old_headers_range(sf_in, block_range.clone())?;
+        let receipts = provider.receipts_by_block_range(block_range.clone())?;
+        assert_eq!(headers.len(), receipts.len());
+        let mut writer = sf_out.get_writer(*block_range.start(), StaticFileSegment::Headers)?;
+        let new_headers = std::iter::zip(headers, receipts)
+            .map(|(header, receipts)| {
+                let eth_header = Header::decompress(&header[0]).unwrap();
+                let hl_header = to_hl_header(receipts, eth_header);
+
+                let difficulty: U256 = CompactU256::decompress(&header[1]).unwrap().into();
+                let hash = BlockHash::decompress(&header[2]).unwrap();
+                (hl_header, difficulty, hash)
+            })
+            .collect::<Vec<_>>();
+        for header in new_headers {
+            writer.append_header(&header.0, header.1, &header.2)?;
+        }
+        writer.commit().unwrap();
+        info!("Migrated block range {:?}...", block_range);
+    }
+    Ok(())
+}
+
+fn to_hl_header(receipts: Vec<EthereumReceipt>, eth_header: Header) -> HlHeader {
+    let system_tx_count = receipts.iter().filter(|r| r.cumulative_gas_used == 0).count();
+    HlHeader::from_ethereum_header(eth_header, &receipts, system_tx_count as u64)
+}
+
+fn old_headers_range(
+    provider: &StaticFileProvider<HlPrimitives>,
+    block_range: impl std::ops::RangeBounds<u64>,
+) -> ProviderResult<Vec<Vec<Vec<u8>>>> {
+    Ok(provider
+        .fetch_range_with_predicate(
+            StaticFileSegment::Headers,
+            to_range(block_range),
+            |cursor, number| {
+                cursor.get(number.into(), 0b111).map(|rows| {
+                    rows.map(|columns| columns.into_iter().map(|column| column.to_vec()).collect())
+                })
+            },
+            |_| true,
+        )?
+        .into_iter()
+        .collect())
+}
+
+// Copied from reth
+fn to_range<R: std::ops::RangeBounds<u64>>(bounds: R) -> std::ops::Range<u64> {
+    let start = match bounds.start_bound() {
+        std::ops::Bound::Included(&v) => v,
+        std::ops::Bound::Excluded(&v) => v + 1,
+        std::ops::Bound::Unbounded => 0,
+    };
+
+    let end = match bounds.end_bound() {
+        std::ops::Bound::Included(&v) => v + 1,
+        std::ops::Bound::Excluded(&v) => v,
+        std::ops::Bound::Unbounded => u64::MAX,
+    };
+
+    start..end
+}
+
+fn using_old_header(number: u64, header: &[u8]) -> bool {
+    let deserialized_old = is_old_header(header);
+    let deserialized_new = is_new_header(header);
+
+    assert!(
+        deserialized_old ^ deserialized_new,
+        "Header is not valid: {} {}\ndeserialized_old: {}\ndeserialized_new: {}",
+        number,
+        header.encode_hex(),
+        deserialized_old,
+        deserialized_new
+    );
+    deserialized_old && !deserialized_new
+}

src/node/mod.rs

@@ -4,11 +4,11 @@ use crate::{
         pool::HlPoolBuilder,
         primitives::{HlBlock, HlPrimitives},
         rpc::{
+            HlEthApiBuilder,
             engine_api::{
                 builder::HlEngineApiBuilder, payload::HlPayloadTypes,
                 validator::HlPayloadValidatorBuilder,
             },
-            HlEthApiBuilder,
         },
         storage::HlStorage,
     },
@@ -20,19 +20,20 @@ use network::HlNetworkBuilder;
 use reth::{
     api::{FullNodeTypes, NodeTypes},
     builder::{
+        Node, NodeAdapter,
         components::{ComponentsBuilder, NoopPayloadServiceBuilder},
         rpc::RpcAddOns,
-        Node, NodeAdapter,
     },
 };
 use reth_engine_primitives::ConsensusEngineHandle;
 use std::{marker::PhantomData, sync::Arc};
-use tokio::sync::{oneshot, Mutex};
+use tokio::sync::{Mutex, oneshot};
 
 pub mod cli;
 pub mod consensus;
 pub mod engine;
 pub mod evm;
+pub mod migrate;
 pub mod network;
 pub mod primitives;
 pub mod rpc;
@@ -50,12 +51,14 @@ pub struct HlNode {
     engine_handle_rx: Arc<Mutex<Option<oneshot::Receiver<ConsensusEngineHandle<HlPayloadTypes>>>>>,
     block_source_config: BlockSourceConfig,
     debug_cutoff_height: Option<u64>,
+    allow_network_overrides: bool,
 }
 
 impl HlNode {
     pub fn new(
         block_source_config: BlockSourceConfig,
         debug_cutoff_height: Option<u64>,
+        allow_network_overrides: bool,
     ) -> (Self, oneshot::Sender<ConsensusEngineHandle<HlPayloadTypes>>) {
         let (tx, rx) = oneshot::channel();
         (
@@ -63,6 +66,7 @@ impl HlNode {
                 engine_handle_rx: Arc::new(Mutex::new(Some(rx))),
                 block_source_config,
                 debug_cutoff_height,
+                allow_network_overrides,
             },
             tx,
         )
@@ -94,6 +98,7 @@ impl HlNode {
                 engine_handle_rx: self.engine_handle_rx.clone(),
                 block_source_config: self.block_source_config.clone(),
                 debug_cutoff_height: self.debug_cutoff_height,
+                allow_network_overrides: self.allow_network_overrides,
             })
             .consensus(HlConsensusBuilder::default())
     }

src/node/network/block_import/mod.rs

@@ -8,7 +8,7 @@ use reth_primitives::NodePrimitives;
 use service::{BlockMsg, ImportEvent, Outcome};
 use std::{
     fmt,
-    task::{ready, Context, Poll},
+    task::{Context, Poll, ready},
 };
 
 use crate::node::network::HlNewBlock;

src/node/network/block_import/service.rs

@@ -1,17 +1,17 @@
 use super::handle::ImportHandle;
 use crate::{
+    HlBlock, HlBlockBody,
     consensus::HlConsensus,
     node::{
         network::HlNewBlock,
         rpc::engine_api::payload::HlPayloadTypes,
         types::{BlockAndReceipts, EvmBlock},
     },
-    HlBlock, HlBlockBody,
 };
 use alloy_consensus::{BlockBody, Header};
 use alloy_primitives::U128;
 use alloy_rpc_types::engine::{ForkchoiceState, PayloadStatusEnum};
-use futures::{future::Either, stream::FuturesUnordered, StreamExt};
+use futures::{StreamExt, future::Either, stream::FuturesUnordered};
 use reth_engine_primitives::{ConsensusEngineHandle, EngineTypes};
 use reth_eth_wire::NewBlock;
 use reth_network::{
@@ -179,7 +179,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::chainspec::hl::hl_mainnet;
+    use crate::{HlHeader, chainspec::hl::hl_mainnet};
 
     use super::*;
     use alloy_primitives::{B256, U128};
@@ -355,7 +355,7 @@ mod tests {
     /// Creates a test block message
     fn create_test_block() -> NewBlockMessage<HlNewBlock> {
         let block = HlBlock {
-            header: Header::default(),
+            header: HlHeader::default(),
             body: HlBlockBody {
                 inner: BlockBody {
                     transactions: Vec::new(),

src/node/network/mod.rs

@@ -1,20 +1,20 @@
 #![allow(clippy::owned_cow)]
 use crate::{
+    HlBlock,
     consensus::HlConsensus,
     node::{
-        network::block_import::{handle::ImportHandle, service::ImportService, HlBlockImport},
+        HlNode,
+        network::block_import::{HlBlockImport, handle::ImportHandle, service::ImportService},
         primitives::HlPrimitives,
         rpc::engine_api::payload::HlPayloadTypes,
         types::ReadPrecompileCalls,
-        HlNode,
     },
-    pseudo_peer::{start_pseudo_peer, BlockSourceConfig},
-    HlBlock,
+    pseudo_peer::{BlockSourceConfig, start_pseudo_peer},
 };
 use alloy_rlp::{Decodable, Encodable};
 use reth::{
     api::{FullNodeTypes, TxTy},
-    builder::{components::NetworkBuilder, BuilderContext},
+    builder::{BuilderContext, components::NetworkBuilder},
     transaction_pool::{PoolTransaction, TransactionPool},
 };
 use reth_discv4::NodeRecord;
@@ -25,8 +25,11 @@ use reth_network::{NetworkConfig, NetworkHandle, NetworkManager};
 use reth_network_api::PeersInfo;
 use reth_provider::StageCheckpointReader;
 use reth_stages_types::StageId;
-use std::sync::Arc;
-use tokio::sync::{mpsc, oneshot, Mutex};
+use std::{
+    net::{Ipv4Addr, SocketAddr},
+    sync::Arc,
+};
+use tokio::sync::{Mutex, mpsc, oneshot};
 use tracing::info;
 
 pub mod block_import;
@@ -38,10 +41,10 @@ pub struct HlNewBlock(pub NewBlock<HlBlock>);
 mod rlp {
     use super::*;
     use crate::{
+        HlBlockBody, HlHeader,
         node::primitives::{BlockBody, TransactionSigned},
-        HlBlockBody,
     };
-    use alloy_consensus::{BlobTransactionSidecar, Header};
+    use alloy_consensus::BlobTransactionSidecar;
     use alloy_primitives::{Address, U128};
     use alloy_rlp::{RlpDecodable, RlpEncodable};
     use alloy_rpc_types::Withdrawals;
@@ -50,9 +53,9 @@ mod rlp {
     #[derive(RlpEncodable, RlpDecodable)]
     #[rlp(trailing)]
     struct BlockHelper<'a> {
-        header: Cow<'a, Header>,
+        header: Cow<'a, HlHeader>,
         transactions: Cow<'a, Vec<TransactionSigned>>,
-        ommers: Cow<'a, Vec<Header>>,
+        ommers: Cow<'a, Vec<HlHeader>>,
         withdrawals: Option<Cow<'a, Withdrawals>>,
     }
 
@@ -144,6 +147,8 @@ pub struct HlNetworkBuilder {
     pub(crate) block_source_config: BlockSourceConfig,
 
     pub(crate) debug_cutoff_height: Option<u64>,
+
+    pub(crate) allow_network_overrides: bool,
 }
 
 impl HlNetworkBuilder {
@@ -174,15 +179,24 @@ impl HlNetworkBuilder {
             ImportService::new(consensus, handle, from_network, to_network).await.unwrap();
         });
 
-        Ok(ctx.build_network_config(
-            ctx.network_config_builder()?
+        let mut config_builder = ctx.network_config_builder()?;
+
+        // Only apply localhost-only network settings if network overrides are NOT allowed
+        if !self.allow_network_overrides {
+            config_builder = config_builder
+                .discovery_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
+                .listener_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
                 .disable_dns_discovery()
-                .disable_nat()
+                .disable_nat();
+        }
+
+        config_builder = config_builder
             .boot_nodes(boot_nodes())
             .set_head(ctx.head())
             .with_pow()
-                .block_import(Box::new(HlBlockImport::new(handle))),
-        ))
+            .block_import(Box::new(HlBlockImport::new(handle)));
+
+        Ok(ctx.build_network_config(config_builder))
     }
 }
 

src/node/pool.rs

@@ -6,12 +6,12 @@
 //! Ethereum transaction pool only supports TransactionSigned (EthereumTxEnvelope<TxEip4844>),
 //! hence this placeholder for the transaction pool.
 
-use crate::node::{primitives::TransactionSigned, HlNode};
+use crate::node::{HlNode, primitives::TransactionSigned};
 use alloy_consensus::{
-    error::ValueError, EthereumTxEnvelope, Transaction as TransactionTrait, TxEip4844,
+    EthereumTxEnvelope, Transaction as TransactionTrait, TxEip4844, error::ValueError,
 };
-use alloy_eips::{eip7702::SignedAuthorization, Typed2718};
-use alloy_primitives::{Address, Bytes, ChainId, TxHash, TxKind, B256, U256};
+use alloy_eips::{Typed2718, eip7702::SignedAuthorization};
+use alloy_primitives::{Address, B256, Bytes, ChainId, TxHash, TxKind, U256};
 use alloy_rpc_types::AccessList;
 use reth::{
     api::FullNodeTypes, builder::components::PoolBuilder, transaction_pool::PoolTransaction,
@@ -19,7 +19,7 @@ use reth::{
 use reth_ethereum_primitives::PooledTransactionVariant;
 use reth_primitives::Recovered;
 use reth_primitives_traits::InMemorySize;
-use reth_transaction_pool::{noop::NoopTransactionPool, EthPoolTransaction};
+use reth_transaction_pool::{EthPoolTransaction, noop::NoopTransactionPool};
 use std::sync::Arc;
 
 pub struct HlPoolBuilder;

src/node/primitives/block.rs

@@ -0,0 +1,49 @@
+use super::{HlBlockBody, HlHeader, rlp};
+use alloy_rlp::Encodable;
+use reth_primitives_traits::{Block, InMemorySize};
+use serde::{Deserialize, Serialize};
+use std::borrow::Cow;
+
+/// Block for HL
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct HlBlock {
+    pub header: HlHeader,
+    pub body: HlBlockBody,
+}
+
+impl InMemorySize for HlBlock {
+    fn size(&self) -> usize {
+        self.header.size() + self.body.size()
+    }
+}
+
+impl Block for HlBlock {
+    type Header = HlHeader;
+    type Body = HlBlockBody;
+
+    fn new(header: Self::Header, body: Self::Body) -> Self {
+        Self { header, body }
+    }
+    fn header(&self) -> &Self::Header {
+        &self.header
+    }
+    fn body(&self) -> &Self::Body {
+        &self.body
+    }
+    fn split(self) -> (Self::Header, Self::Body) {
+        (self.header, self.body)
+    }
+
+    fn rlp_length(header: &Self::Header, body: &Self::Body) -> usize {
+        rlp::BlockHelper {
+            header: Cow::Borrowed(header),
+            transactions: Cow::Borrowed(&body.inner.transactions),
+            ommers: Cow::Borrowed(&body.inner.ommers),
+            withdrawals: body.inner.withdrawals.as_ref().map(Cow::Borrowed),
+            sidecars: body.sidecars.as_ref().map(Cow::Borrowed),
+            read_precompile_calls: body.read_precompile_calls.as_ref().map(Cow::Borrowed),
+            highest_precompile_address: body.highest_precompile_address.as_ref().map(Cow::Borrowed),
+        }
+        .length()
+    }
+}

src/node/primitives/body.rs

@@ -0,0 +1,80 @@
+use alloy_consensus::BlobTransactionSidecar;
+use alloy_primitives::Address;
+use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    HlHeader,
+    node::{
+        primitives::TransactionSigned,
+        types::{ReadPrecompileCall, ReadPrecompileCalls},
+    },
+};
+
+/// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars
+/// for blob transactions.
+#[derive(
+    Debug,
+    Clone,
+    Default,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    derive_more::Deref,
+    derive_more::DerefMut,
+)]
+pub struct HlBlockBody {
+    #[serde(flatten)]
+    #[deref]
+    #[deref_mut]
+    pub inner: BlockBody,
+    pub sidecars: Option<Vec<BlobTransactionSidecar>>,
+    pub read_precompile_calls: Option<ReadPrecompileCalls>,
+    pub highest_precompile_address: Option<Address>,
+}
+
+pub type BlockBody = alloy_consensus::BlockBody<TransactionSigned, HlHeader>;
+
+impl InMemorySize for HlBlockBody {
+    fn size(&self) -> usize {
+        self.inner.size() +
+            self.sidecars
+                .as_ref()
+                .map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>()) +
+            self.read_precompile_calls
+                .as_ref()
+                .map_or(0, |s| s.0.capacity() * core::mem::size_of::<ReadPrecompileCall>())
+    }
+}
+
+impl BlockBodyTrait for HlBlockBody {
+    type Transaction = TransactionSigned;
+    type OmmerHeader = super::HlHeader;
+
+    fn transactions(&self) -> &[Self::Transaction] {
+        BlockBodyTrait::transactions(&self.inner)
+    }
+    fn into_ethereum_body(self) -> BlockBody {
+        self.inner
+    }
+    fn into_transactions(self) -> Vec<Self::Transaction> {
+        self.inner.into_transactions()
+    }
+    fn withdrawals(&self) -> Option<&alloy_rpc_types::Withdrawals> {
+        self.inner.withdrawals()
+    }
+    fn ommers(&self) -> Option<&[Self::OmmerHeader]> {
+        self.inner.ommers()
+    }
+
+    fn calculate_tx_root(&self) -> alloy_primitives::B256 {
+        alloy_consensus::proofs::calculate_transaction_root(
+            &self
+                .transactions()
+                .iter()
+                .filter(|tx| !tx.is_system_transaction())
+                .collect::<Vec<_>>(),
+        )
+    }
+}

src/node/primitives/header.rs

@@ -0,0 +1,246 @@
+use alloy_consensus::Header;
+use alloy_primitives::{Address, B64, B256, BlockNumber, Bloom, Bytes, Sealable, U256};
+use alloy_rlp::{RlpDecodable, RlpEncodable};
+use reth_cli_commands::common::CliHeader;
+use reth_codecs::Compact;
+use reth_ethereum_primitives::EthereumReceipt;
+use reth_primitives::{SealedHeader, logs_bloom};
+use reth_primitives_traits::{BlockHeader, InMemorySize, serde_bincode_compat::RlpBincode};
+use reth_rpc_convert::transaction::FromConsensusHeader;
+use serde::{Deserialize, Serialize};
+
+/// The header type of this node
+///
+/// This type extends the regular ethereum header with an extension.
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    Hash,
+    derive_more::AsRef,
+    derive_more::Deref,
+    Default,
+    RlpEncodable,
+    RlpDecodable,
+    Serialize,
+    Deserialize,
+)]
+#[serde(rename_all = "camelCase")]
+pub struct HlHeader {
+    /// The regular eth header
+    #[as_ref]
+    #[deref]
+    pub inner: Header,
+    /// The extended header fields that is not part of the block hash
+    pub extras: HlHeaderExtras,
+}
+
+#[derive(
+    Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, RlpEncodable, RlpDecodable, Hash,
+)]
+pub struct HlHeaderExtras {
+    pub logs_bloom_with_system_txs: Bloom,
+    pub system_tx_count: u64,
+}
+
+impl HlHeader {
+    pub(crate) fn from_ethereum_header(
+        header: Header,
+        receipts: &[EthereumReceipt],
+        system_tx_count: u64,
+    ) -> HlHeader {
+        let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs));
+        HlHeader {
+            inner: header,
+            extras: HlHeaderExtras { logs_bloom_with_system_txs: logs_bloom, system_tx_count },
+        }
+    }
+}
+
+impl From<Header> for HlHeader {
+    fn from(_value: Header) -> Self {
+        unreachable!()
+    }
+}
+
+impl AsRef<Self> for HlHeader {
+    fn as_ref(&self) -> &Self {
+        self
+    }
+}
+
+impl Sealable for HlHeader {
+    fn hash_slow(&self) -> B256 {
+        self.inner.hash_slow()
+    }
+}
+
+impl alloy_consensus::BlockHeader for HlHeader {
+    fn parent_hash(&self) -> B256 {
+        self.inner.parent_hash()
+    }
+
+    fn ommers_hash(&self) -> B256 {
+        self.inner.ommers_hash()
+    }
+
+    fn beneficiary(&self) -> Address {
+        self.inner.beneficiary()
+    }
+
+    fn state_root(&self) -> B256 {
+        self.inner.state_root()
+    }
+
+    fn transactions_root(&self) -> B256 {
+        self.inner.transactions_root()
+    }
+
+    fn receipts_root(&self) -> B256 {
+        self.inner.receipts_root()
+    }
+
+    fn withdrawals_root(&self) -> Option<B256> {
+        self.inner.withdrawals_root()
+    }
+
+    fn logs_bloom(&self) -> Bloom {
+        self.extras.logs_bloom_with_system_txs
+    }
+
+    fn difficulty(&self) -> U256 {
+        self.inner.difficulty()
+    }
+
+    fn number(&self) -> BlockNumber {
+        self.inner.number()
+    }
+
+    fn gas_limit(&self) -> u64 {
+        self.inner.gas_limit()
+    }
+
+    fn gas_used(&self) -> u64 {
+        self.inner.gas_used()
+    }
+
+    fn timestamp(&self) -> u64 {
+        self.inner.timestamp()
+    }
+
+    fn mix_hash(&self) -> Option<B256> {
+        self.inner.mix_hash()
+    }
+
+    fn nonce(&self) -> Option<B64> {
+        self.inner.nonce()
+    }
+
+    fn base_fee_per_gas(&self) -> Option<u64> {
+        self.inner.base_fee_per_gas()
+    }
+
+    fn blob_gas_used(&self) -> Option<u64> {
+        self.inner.blob_gas_used()
+    }
+
+    fn excess_blob_gas(&self) -> Option<u64> {
+        self.inner.excess_blob_gas()
+    }
+
+    fn parent_beacon_block_root(&self) -> Option<B256> {
+        self.inner.parent_beacon_block_root()
+    }
+
+    fn requests_hash(&self) -> Option<B256> {
+        self.inner.requests_hash()
+    }
+
+    fn extra_data(&self) -> &Bytes {
+        self.inner.extra_data()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.extras.system_tx_count == 0 && self.inner.is_empty()
+    }
+}
+
+impl InMemorySize for HlHeader {
+    fn size(&self) -> usize {
+        self.inner.size() + self.extras.size()
+    }
+}
+
+impl InMemorySize for HlHeaderExtras {
+    fn size(&self) -> usize {
+        self.logs_bloom_with_system_txs.data().len() + self.system_tx_count.size()
+    }
+}
+
+impl reth_codecs::Compact for HlHeader {
+    fn to_compact<B>(&self, buf: &mut B) -> usize
+    where
+        B: alloy_rlp::bytes::BufMut + AsMut<[u8]>,
+    {
+        // Because Header ends with extra_data which is `Bytes`, we can't use to_compact for extras,
+        // because Compact trait requires the Bytes field to be placed at the end of the struct.
+        // Bytes::from_compact just reads all trailing data as the Bytes field.
+        //
+        // Hence we need to use other form of serialization, since extra headers are not
+        // Compact-compatible. We just treat all header fields as rmp-serialized one `Bytes`
+        // field.
+        let result: Bytes = rmp_serde::to_vec(&self).unwrap().into();
+        result.to_compact(buf)
+    }
+
+    fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
+        let (bytes, remaining) = Bytes::from_compact(buf, len);
+        let header: HlHeader = rmp_serde::from_slice(&bytes).unwrap();
+        (header, remaining)
+    }
+}
+
+impl reth_db_api::table::Compress for HlHeader {
+    type Compressed = Vec<u8>;
+
+    fn compress_to_buf<B: alloy_primitives::bytes::BufMut + AsMut<[u8]>>(&self, buf: &mut B) {
+        let _ = Compact::to_compact(self, buf);
+    }
+}
+
+impl reth_db_api::table::Decompress for HlHeader {
+    fn decompress(value: &[u8]) -> Result<Self, reth_db_api::DatabaseError> {
+        let (obj, _) = Compact::from_compact(value, value.len());
+        Ok(obj)
+    }
+}
+
+impl BlockHeader for HlHeader {}
+
+impl RlpBincode for HlHeader {}
+
+impl CliHeader for HlHeader {
+    fn set_number(&mut self, number: u64) {
+        self.inner.set_number(number);
+    }
+}
+
+impl From<HlHeader> for Header {
+    fn from(value: HlHeader) -> Self {
+        value.inner
+    }
+}
+
+pub fn to_ethereum_ommers(ommers: &[HlHeader]) -> Vec<Header> {
+    ommers.iter().map(|ommer| ommer.clone().into()).collect()
+}
+
+impl FromConsensusHeader<HlHeader> for alloy_rpc_types::Header {
+    fn from_consensus_header(header: SealedHeader<HlHeader>, block_size: usize) -> Self {
+        FromConsensusHeader::<Header>::from_consensus_header(
+            SealedHeader::<Header>::new(header.inner.clone(), header.hash()),
+            block_size,
+        )
+    }
+}

src/node/primitives/mod.rs

@@ -1,17 +1,18 @@
-#![allow(clippy::owned_cow)]
-use alloy_consensus::{BlobTransactionSidecar, Header};
-use alloy_primitives::Address;
-use alloy_rlp::{Encodable, RlpDecodable, RlpEncodable};
 use reth_ethereum_primitives::Receipt;
 use reth_primitives::NodePrimitives;
-use reth_primitives_traits::{Block, BlockBody as BlockBodyTrait, InMemorySize};
-use serde::{Deserialize, Serialize};
-use std::borrow::Cow;
 
-use crate::node::types::{ReadPrecompileCall, ReadPrecompileCalls};
+pub mod transaction;
+pub use transaction::TransactionSigned;
 
-pub mod tx_wrapper;
-pub use tx_wrapper::{BlockBody, TransactionSigned};
+pub mod block;
+pub use block::HlBlock;
+pub mod body;
+pub use body::{BlockBody, HlBlockBody};
+pub mod header;
+pub use header::HlHeader;
+
+pub mod rlp;
+pub mod serde_bincode_compat;
 
 /// Primitive types for HyperEVM.
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -20,321 +21,8 @@ pub struct HlPrimitives;
 
 impl NodePrimitives for HlPrimitives {
     type Block = HlBlock;
-    type BlockHeader = Header;
+    type BlockHeader = HlHeader;
     type BlockBody = HlBlockBody;
     type SignedTx = TransactionSigned;
     type Receipt = Receipt;
 }
-
-/// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars
-/// for blob transactions.
-#[derive(
-    Debug,
-    Clone,
-    Default,
-    PartialEq,
-    Eq,
-    Serialize,
-    Deserialize,
-    derive_more::Deref,
-    derive_more::DerefMut,
-)]
-pub struct HlBlockBody {
-    #[serde(flatten)]
-    #[deref]
-    #[deref_mut]
-    pub inner: BlockBody,
-    pub sidecars: Option<Vec<BlobTransactionSidecar>>,
-    pub read_precompile_calls: Option<ReadPrecompileCalls>,
-    pub highest_precompile_address: Option<Address>,
-}
-
-impl InMemorySize for HlBlockBody {
-    fn size(&self) -> usize {
-        self.inner.size() +
-            self.sidecars
-                .as_ref()
-                .map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>()) +
-            self.read_precompile_calls
-                .as_ref()
-                .map_or(0, |s| s.0.capacity() * core::mem::size_of::<ReadPrecompileCall>())
-    }
-}
-
-impl BlockBodyTrait for HlBlockBody {
-    type Transaction = TransactionSigned;
-    type OmmerHeader = Header;
-
-    fn transactions(&self) -> &[Self::Transaction] {
-        BlockBodyTrait::transactions(&self.inner)
-    }
-    fn into_ethereum_body(self) -> BlockBody {
-        self.inner
-    }
-    fn into_transactions(self) -> Vec<Self::Transaction> {
-        self.inner.into_transactions()
-    }
-    fn withdrawals(&self) -> Option<&alloy_rpc_types::Withdrawals> {
-        self.inner.withdrawals()
-    }
-    fn ommers(&self) -> Option<&[Self::OmmerHeader]> {
-        self.inner.ommers()
-    }
-
-    fn calculate_tx_root(&self) -> alloy_primitives::B256 {
-        alloy_consensus::proofs::calculate_transaction_root(
-            &self
-                .transactions()
-                .iter()
-                .filter(|tx| !tx.is_system_transaction())
-                .collect::<Vec<_>>(),
-        )
-    }
-}
-
-/// Block for HL
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
-pub struct HlBlock {
-    pub header: Header,
-    pub body: HlBlockBody,
-}
-
-impl InMemorySize for HlBlock {
-    fn size(&self) -> usize {
-        self.header.size() + self.body.size()
-    }
-}
-
-impl Block for HlBlock {
-    type Header = Header;
-    type Body = HlBlockBody;
-
-    fn new(header: Self::Header, body: Self::Body) -> Self {
-        Self { header, body }
-    }
-    fn header(&self) -> &Self::Header {
-        &self.header
-    }
-    fn body(&self) -> &Self::Body {
-        &self.body
-    }
-    fn split(self) -> (Self::Header, Self::Body) {
-        (self.header, self.body)
-    }
-
-    fn rlp_length(header: &Self::Header, body: &Self::Body) -> usize {
-        rlp::BlockHelper {
-            header: Cow::Borrowed(header),
-            transactions: Cow::Borrowed(&body.inner.transactions),
-            ommers: Cow::Borrowed(&body.inner.ommers),
-            withdrawals: body.inner.withdrawals.as_ref().map(Cow::Borrowed),
-            sidecars: body.sidecars.as_ref().map(Cow::Borrowed),
-            read_precompile_calls: body.read_precompile_calls.as_ref().map(Cow::Borrowed),
-            highest_precompile_address: body.highest_precompile_address.as_ref().map(Cow::Borrowed),
-        }
-        .length()
-    }
-}
-
-mod rlp {
-    use super::*;
-    use alloy_eips::eip4895::Withdrawals;
-    use alloy_rlp::Decodable;
-
-    #[derive(RlpEncodable, RlpDecodable)]
-    #[rlp(trailing)]
-    struct BlockBodyHelper<'a> {
-        transactions: Cow<'a, Vec<TransactionSigned>>,
-        ommers: Cow<'a, Vec<Header>>,
-        withdrawals: Option<Cow<'a, Withdrawals>>,
-        sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
-        read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
-        highest_precompile_address: Option<Cow<'a, Address>>,
-    }
-
-    #[derive(RlpEncodable, RlpDecodable)]
-    #[rlp(trailing)]
-    pub(crate) struct BlockHelper<'a> {
-        pub(crate) header: Cow<'a, Header>,
-        pub(crate) transactions: Cow<'a, Vec<TransactionSigned>>,
-        pub(crate) ommers: Cow<'a, Vec<Header>>,
-        pub(crate) withdrawals: Option<Cow<'a, Withdrawals>>,
-        pub(crate) sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
-        pub(crate) read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
-        pub(crate) highest_precompile_address: Option<Cow<'a, Address>>,
-    }
-
-    impl<'a> From<&'a HlBlockBody> for BlockBodyHelper<'a> {
-        fn from(value: &'a HlBlockBody) -> Self {
-            let HlBlockBody {
-                inner: BlockBody { transactions, ommers, withdrawals },
-                sidecars,
-                read_precompile_calls,
-                highest_precompile_address,
-            } = value;
-            Self {
-                transactions: Cow::Borrowed(transactions),
-                ommers: Cow::Borrowed(ommers),
-                withdrawals: withdrawals.as_ref().map(Cow::Borrowed),
-                sidecars: sidecars.as_ref().map(Cow::Borrowed),
-                read_precompile_calls: read_precompile_calls.as_ref().map(Cow::Borrowed),
-                highest_precompile_address: highest_precompile_address.as_ref().map(Cow::Borrowed),
-            }
-        }
-    }
-
-    impl<'a> From<&'a HlBlock> for BlockHelper<'a> {
-        fn from(value: &'a HlBlock) -> Self {
-            let HlBlock {
-                header,
-                body:
-                    HlBlockBody {
-                        inner: BlockBody { transactions, ommers, withdrawals },
-                        sidecars,
-                        read_precompile_calls,
-                        highest_precompile_address,
-                    },
-            } = value;
-            Self {
-                header: Cow::Borrowed(header),
-                transactions: Cow::Borrowed(transactions),
-                ommers: Cow::Borrowed(ommers),
-                withdrawals: withdrawals.as_ref().map(Cow::Borrowed),
-                sidecars: sidecars.as_ref().map(Cow::Borrowed),
-                read_precompile_calls: read_precompile_calls.as_ref().map(Cow::Borrowed),
-                highest_precompile_address: highest_precompile_address.as_ref().map(Cow::Borrowed),
-            }
-        }
-    }
-
-    impl Encodable for HlBlockBody {
-        fn encode(&self, out: &mut dyn bytes::BufMut) {
-            BlockBodyHelper::from(self).encode(out);
-        }
-        fn length(&self) -> usize {
-            BlockBodyHelper::from(self).length()
-        }
-    }
-
-    impl Decodable for HlBlockBody {
-        fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
-            let BlockBodyHelper {
-                transactions,
-                ommers,
-                withdrawals,
-                sidecars,
-                read_precompile_calls,
-                highest_precompile_address,
-            } = BlockBodyHelper::decode(buf)?;
-            Ok(Self {
-                inner: BlockBody {
-                    transactions: transactions.into_owned(),
-                    ommers: ommers.into_owned(),
-                    withdrawals: withdrawals.map(|w| w.into_owned()),
-                },
-                sidecars: sidecars.map(|s| s.into_owned()),
-                read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
-                highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
-            })
-        }
-    }
-
-    impl Encodable for HlBlock {
-        fn encode(&self, out: &mut dyn bytes::BufMut) {
-            BlockHelper::from(self).encode(out);
-        }
-        fn length(&self) -> usize {
-            BlockHelper::from(self).length()
-        }
-    }
-
-    impl Decodable for HlBlock {
-        fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
-            let BlockHelper {
-                header,
-                transactions,
-                ommers,
-                withdrawals,
-                sidecars,
-                read_precompile_calls,
-                highest_precompile_address,
-            } = BlockHelper::decode(buf)?;
-            Ok(Self {
-                header: header.into_owned(),
-                body: HlBlockBody {
-                    inner: BlockBody {
-                        transactions: transactions.into_owned(),
-                        ommers: ommers.into_owned(),
-                        withdrawals: withdrawals.map(|w| w.into_owned()),
-                    },
-                    sidecars: sidecars.map(|s| s.into_owned()),
-                    read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
-                    highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
-                },
-            })
-        }
-    }
-}
-
-pub mod serde_bincode_compat {
-    use super::*;
-    use reth_primitives_traits::serde_bincode_compat::{BincodeReprFor, SerdeBincodeCompat};
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct HlBlockBodyBincode<'a> {
-        inner: BincodeReprFor<'a, BlockBody>,
-        sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
-        read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
-        highest_precompile_address: Option<Cow<'a, Address>>,
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct HlBlockBincode<'a> {
-        header: BincodeReprFor<'a, Header>,
-        body: BincodeReprFor<'a, HlBlockBody>,
-    }
-
-    impl SerdeBincodeCompat for HlBlockBody {
-        type BincodeRepr<'a> = HlBlockBodyBincode<'a>;
-
-        fn as_repr(&self) -> Self::BincodeRepr<'_> {
-            HlBlockBodyBincode {
-                inner: self.inner.as_repr(),
-                sidecars: self.sidecars.as_ref().map(Cow::Borrowed),
-                read_precompile_calls: self.read_precompile_calls.as_ref().map(Cow::Borrowed),
-                highest_precompile_address: self
-                    .highest_precompile_address
-                    .as_ref()
-                    .map(Cow::Borrowed),
-            }
-        }
-
-        fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
-            let HlBlockBodyBincode {
-                inner,
-                sidecars,
-                read_precompile_calls,
-                highest_precompile_address,
-            } = repr;
-            Self {
-                inner: BlockBody::from_repr(inner),
-                sidecars: sidecars.map(|s| s.into_owned()),
-                read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
-                highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
-            }
-        }
-    }
-
-    impl SerdeBincodeCompat for HlBlock {
-        type BincodeRepr<'a> = HlBlockBincode<'a>;
-
-        fn as_repr(&self) -> Self::BincodeRepr<'_> {
-            HlBlockBincode { header: self.header.as_repr(), body: self.body.as_repr() }
-        }
-
-        fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
-            let HlBlockBincode { header, body } = repr;
-            Self { header: Header::from_repr(header), body: HlBlockBody::from_repr(body) }
-        }
-    }
-}

src/node/primitives/rlp.rs

@@ -0,0 +1,142 @@
+#![allow(clippy::owned_cow)]
+use super::{HlBlock, HlBlockBody, TransactionSigned};
+use crate::{HlHeader, node::types::ReadPrecompileCalls};
+use alloy_consensus::{BlobTransactionSidecar, BlockBody};
+use alloy_eips::eip4895::Withdrawals;
+use alloy_primitives::Address;
+use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
+use std::borrow::Cow;
+
+#[derive(RlpEncodable, RlpDecodable)]
+#[rlp(trailing)]
+struct BlockBodyHelper<'a> {
+    transactions: Cow<'a, Vec<TransactionSigned>>,
+    ommers: Cow<'a, Vec<HlHeader>>,
+    withdrawals: Option<Cow<'a, Withdrawals>>,
+    sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
+    read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
+    highest_precompile_address: Option<Cow<'a, Address>>,
+}
+
+#[derive(RlpEncodable, RlpDecodable)]
+#[rlp(trailing)]
+pub(crate) struct BlockHelper<'a> {
+    pub(crate) header: Cow<'a, HlHeader>,
+    pub(crate) transactions: Cow<'a, Vec<TransactionSigned>>,
+    pub(crate) ommers: Cow<'a, Vec<HlHeader>>,
+    pub(crate) withdrawals: Option<Cow<'a, Withdrawals>>,
+    pub(crate) sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
+    pub(crate) read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
+    pub(crate) highest_precompile_address: Option<Cow<'a, Address>>,
+}
+
+impl<'a> From<&'a HlBlockBody> for BlockBodyHelper<'a> {
+    fn from(value: &'a HlBlockBody) -> Self {
+        let HlBlockBody {
+            inner: BlockBody { transactions, ommers, withdrawals },
+            sidecars,
+            read_precompile_calls,
+            highest_precompile_address,
+        } = value;
+        Self {
+            transactions: Cow::Borrowed(transactions),
+            ommers: Cow::Borrowed(ommers),
+            withdrawals: withdrawals.as_ref().map(Cow::Borrowed),
+            sidecars: sidecars.as_ref().map(Cow::Borrowed),
+            read_precompile_calls: read_precompile_calls.as_ref().map(Cow::Borrowed),
+            highest_precompile_address: highest_precompile_address.as_ref().map(Cow::Borrowed),
+        }
+    }
+}
+
+impl<'a> From<&'a HlBlock> for BlockHelper<'a> {
+    fn from(value: &'a HlBlock) -> Self {
+        let HlBlock {
+            header,
+            body:
+                HlBlockBody {
+                    inner: BlockBody { transactions, ommers, withdrawals },
+                    sidecars,
+                    read_precompile_calls,
+                    highest_precompile_address,
+                },
+        } = value;
+        Self {
+            header: Cow::Borrowed(header),
+            transactions: Cow::Borrowed(transactions),
+            ommers: Cow::Borrowed(ommers),
+            withdrawals: withdrawals.as_ref().map(Cow::Borrowed),
+            sidecars: sidecars.as_ref().map(Cow::Borrowed),
+            read_precompile_calls: read_precompile_calls.as_ref().map(Cow::Borrowed),
+            highest_precompile_address: highest_precompile_address.as_ref().map(Cow::Borrowed),
+        }
+    }
+}
+
+impl Encodable for HlBlockBody {
+    fn encode(&self, out: &mut dyn bytes::BufMut) {
+        BlockBodyHelper::from(self).encode(out);
+    }
+    fn length(&self) -> usize {
+        BlockBodyHelper::from(self).length()
+    }
+}
+
+impl Decodable for HlBlockBody {
+    fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
+        let BlockBodyHelper {
+            transactions,
+            ommers,
+            withdrawals,
+            sidecars,
+            read_precompile_calls,
+            highest_precompile_address,
+        } = BlockBodyHelper::decode(buf)?;
+        Ok(Self {
+            inner: BlockBody {
+                transactions: transactions.into_owned(),
+                ommers: ommers.into_owned(),
+                withdrawals: withdrawals.map(|w| w.into_owned()),
+            },
+            sidecars: sidecars.map(|s| s.into_owned()),
+            read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
+            highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
+        })
+    }
+}
+
+impl Encodable for HlBlock {
+    fn encode(&self, out: &mut dyn bytes::BufMut) {
+        BlockHelper::from(self).encode(out);
+    }
+    fn length(&self) -> usize {
+        BlockHelper::from(self).length()
+    }
+}
+
+impl Decodable for HlBlock {
+    fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
+        let BlockHelper {
+            header,
+            transactions,
+            ommers,
+            withdrawals,
+            sidecars,
+            read_precompile_calls,
+            highest_precompile_address,
+        } = BlockHelper::decode(buf)?;
+        Ok(Self {
+            header: header.into_owned(),
+            body: HlBlockBody {
+                inner: BlockBody {
+                    transactions: transactions.into_owned(),
+                    ommers: ommers.into_owned(),
+                    withdrawals: withdrawals.map(|w| w.into_owned()),
+                },
+                sidecars: sidecars.map(|s| s.into_owned()),
+                read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
+                highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
+            },
+        })
+    }
+}

src/node/primitives/serde_bincode_compat.rs

@@ -0,0 +1,67 @@
+#![allow(clippy::owned_cow)]
+use alloy_consensus::BlobTransactionSidecar;
+use alloy_primitives::Address;
+use reth_primitives_traits::serde_bincode_compat::{BincodeReprFor, SerdeBincodeCompat};
+use serde::{Deserialize, Serialize};
+use std::borrow::Cow;
+
+use super::{HlBlock, HlBlockBody};
+use crate::{
+    HlHeader,
+    node::{primitives::BlockBody, types::ReadPrecompileCalls},
+};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct HlBlockBodyBincode<'a> {
+    inner: BincodeReprFor<'a, BlockBody>,
+    sidecars: Option<Cow<'a, Vec<BlobTransactionSidecar>>>,
+    read_precompile_calls: Option<Cow<'a, ReadPrecompileCalls>>,
+    highest_precompile_address: Option<Cow<'a, Address>>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct HlBlockBincode<'a> {
+    header: BincodeReprFor<'a, HlHeader>,
+    body: BincodeReprFor<'a, HlBlockBody>,
+}
+
+impl SerdeBincodeCompat for HlBlockBody {
+    type BincodeRepr<'a> = HlBlockBodyBincode<'a>;
+
+    fn as_repr(&self) -> Self::BincodeRepr<'_> {
+        HlBlockBodyBincode {
+            inner: self.inner.as_repr(),
+            sidecars: self.sidecars.as_ref().map(Cow::Borrowed),
+            read_precompile_calls: self.read_precompile_calls.as_ref().map(Cow::Borrowed),
+            highest_precompile_address: self.highest_precompile_address.as_ref().map(Cow::Borrowed),
+        }
+    }
+
+    fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
+        let HlBlockBodyBincode {
+            inner,
+            sidecars,
+            read_precompile_calls,
+            highest_precompile_address,
+        } = repr;
+        Self {
+            inner: BlockBody::from_repr(inner),
+            sidecars: sidecars.map(|s| s.into_owned()),
+            read_precompile_calls: read_precompile_calls.map(|s| s.into_owned()),
+            highest_precompile_address: highest_precompile_address.map(|s| s.into_owned()),
+        }
+    }
+}
+
+impl SerdeBincodeCompat for HlBlock {
+    type BincodeRepr<'a> = HlBlockBincode<'a>;
+
+    fn as_repr(&self) -> Self::BincodeRepr<'_> {
+        HlBlockBincode { header: self.header.as_repr(), body: self.body.as_repr() }
+    }
+
+    fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
+        let HlBlockBincode { header, body } = repr;
+        Self { header: HlHeader::from_repr(header), body: HlBlockBody::from_repr(body) }
+    }
+}

src/node/primitives/transaction.rs

@@ -1,30 +1,32 @@
 //! HlNodePrimitives::TransactionSigned; it's the same as ethereum transaction type,
 //! except that it supports pseudo signer for system transactions.
+use std::convert::Infallible;
+
 use crate::evm::transaction::HlTxEnv;
 use alloy_consensus::{
-    crypto::RecoveryError, error::ValueError, SignableTransaction, Signed,
-    Transaction as TransactionTrait, TransactionEnvelope, TxEip1559, TxEip2930, TxEip4844,
-    TxEip7702, TxLegacy, TxType, TypedTransaction,
+    SignableTransaction, Signed, Transaction as TransactionTrait, TransactionEnvelope, TxEip1559,
+    TxEip2930, TxEip4844, TxEip7702, TxLegacy, TxType, TypedTransaction, crypto::RecoveryError,
+    error::ValueError, transaction::TxHashRef,
 };
 use alloy_eips::Encodable2718;
 use alloy_network::TxSigner;
-use alloy_primitives::{address, Address, TxHash, U256};
+use alloy_primitives::{Address, TxHash, U256, address};
 use alloy_rpc_types::{Transaction, TransactionInfo, TransactionRequest};
 use alloy_signer::Signature;
 use reth_codecs::alloy::transaction::{Envelope, FromTxCompact};
 use reth_db::{
-    table::{Compress, Decompress},
     DatabaseError,
+    table::{Compress, Decompress},
 };
 use reth_ethereum_primitives::PooledTransactionVariant;
 use reth_evm::FromRecoveredTx;
 use reth_primitives::Recovered;
 use reth_primitives_traits::{
-    serde_bincode_compat::SerdeBincodeCompat, InMemorySize, SignedTransaction, SignerRecoverable,
+    InMemorySize, SignedTransaction, SignerRecoverable, serde_bincode_compat::SerdeBincodeCompat,
 };
 use reth_rpc_eth_api::{
-    transaction::{FromConsensusTx, TryIntoTxEnv},
     EthTxEnvError, SignTxRequestError, SignableTxRequest, TryIntoSimTx,
+    transaction::{FromConsensusTx, TryIntoTxEnv},
 };
 use revm::context::{BlockEnv, CfgEnv, TxEnv};
 
@@ -46,6 +48,12 @@ fn s_to_address(s: U256) -> Address {
     Address::from_slice(&buf)
 }
 
+impl TxHashRef for TransactionSigned {
+    fn tx_hash(&self) -> &TxHash {
+        self.inner().tx_hash()
+    }
+}
+
 impl SignerRecoverable for TransactionSigned {
     fn recover_signer(&self) -> Result<Address, RecoveryError> {
         if self.is_system_transaction() {
@@ -69,11 +77,7 @@ impl SignerRecoverable for TransactionSigned {
     }
 }
 
-impl SignedTransaction for TransactionSigned {
-    fn tx_hash(&self) -> &TxHash {
-        self.inner().tx_hash()
-    }
-}
+impl SignedTransaction for TransactionSigned {}
 
 // ------------------------------------------------------------
 // NOTE: All lines below are just wrappers for the inner type.
@@ -177,8 +181,6 @@ impl SerdeBincodeCompat for TransactionSigned {
     }
 }
 
-pub type BlockBody = alloy_consensus::BlockBody<TransactionSigned>;
-
 impl TryFrom<TransactionSigned> for PooledTransactionVariant {
     type Error = <InnerType as TryInto<PooledTransactionVariant>>::Error;
 
@@ -207,22 +209,6 @@ impl Decompress for TransactionSigned {
     }
 }
 
-pub fn convert_to_eth_block_body(value: BlockBody) -> alloy_consensus::BlockBody<InnerType> {
-    alloy_consensus::BlockBody {
-        transactions: value.transactions.into_iter().map(|tx| tx.into_inner()).collect(),
-        ommers: value.ommers,
-        withdrawals: value.withdrawals,
-    }
-}
-
-pub fn convert_to_hl_block_body(value: alloy_consensus::BlockBody<InnerType>) -> BlockBody {
-    BlockBody {
-        transactions: value.transactions.into_iter().map(TransactionSigned::Default).collect(),
-        ommers: value.ommers,
-        withdrawals: value.withdrawals,
-    }
-}
-
 impl TryIntoSimTx<TransactionSigned> for TransactionRequest {
     fn try_into_sim_tx(self) -> Result<TransactionSigned, ValueError<Self>> {
         let tx = self
@@ -250,9 +236,17 @@ impl TryIntoTxEnv<HlTxEnv<TxEnv>> for TransactionRequest {
 
 impl FromConsensusTx<TransactionSigned> for Transaction {
     type TxInfo = TransactionInfo;
+    type Err = Infallible;
 
-    fn from_consensus_tx(tx: TransactionSigned, signer: Address, tx_info: Self::TxInfo) -> Self {
-        Self::from_transaction(Recovered::new_unchecked(tx.into_inner().into(), signer), tx_info)
+    fn from_consensus_tx(
+        tx: TransactionSigned,
+        signer: Address,
+        tx_info: Self::TxInfo,
+    ) -> Result<Self, Self::Err> {
+        Ok(Self::from_transaction(
+            Recovered::new_unchecked(tx.into_inner().into(), signer),
+            tx_info,
+        ))
     }
 }
 

src/node/rpc/block.rs

@@ -1,12 +1,12 @@
 use crate::node::rpc::{HlEthApi, HlRpcNodeCore};
 use reth::rpc::server_types::eth::{
-    builder::config::PendingBlockKind, error::FromEvmError, EthApiError, PendingBlock,
+    EthApiError, PendingBlock, builder::config::PendingBlockKind, error::FromEvmError,
 };
 use reth_rpc_eth_api::{
-    helpers::{
-        pending_block::PendingEnvBuilder, EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt,
-    },
     RpcConvert,
+    helpers::{
+        EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, pending_block::PendingEnvBuilder,
+    },
 };
 
 impl<N, Rpc> EthBlocks for HlEthApi<N, Rpc>
@@ -29,7 +29,7 @@ impl<N, Rpc> LoadPendingBlock for HlEthApi<N, Rpc>
 where
     N: HlRpcNodeCore,
     EthApiError: FromEvmError<N::Evm>,
-    Rpc: RpcConvert<Primitives = N::Primitives, Error = EthApiError>,
+    Rpc: RpcConvert<Primitives = N::Primitives>,
 {
     #[inline]
     fn pending_block(&self) -> &tokio::sync::Mutex<Option<PendingBlock<N::Primitives>>> {
@@ -50,7 +50,6 @@ where
 impl<N, Rpc> LoadReceipt for HlEthApi<N, Rpc>
 where
     N: HlRpcNodeCore,
-    EthApiError: FromEvmError<N::Evm>,
     Rpc: RpcConvert<Primitives = N::Primitives, Error = EthApiError>,
 {
 }

src/node/rpc/call.rs

@@ -1,19 +1,19 @@
 use core::fmt;
 
 use super::{HlEthApi, HlRpcNodeCore};
-use crate::{node::evm::apply_precompiles, HlBlock};
+use crate::{HlBlock, node::evm::apply_precompiles};
+use alloy_consensus::transaction::TxHashRef;
 use alloy_evm::Evm;
 use alloy_primitives::B256;
 use reth::rpc::server_types::eth::EthApiError;
 use reth_evm::{ConfigureEvm, Database, EvmEnvFor, HaltReasonFor, InspectorFor, SpecFor, TxEnvFor};
 use reth_primitives::{NodePrimitives, Recovered};
-use reth_primitives_traits::SignedTransaction;
 use reth_provider::{ProviderError, ProviderTx};
 use reth_rpc_eth_api::{
-    helpers::{Call, EthCall},
     FromEvmError, RpcConvert, RpcNodeCore,
+    helpers::{Call, EthCall},
 };
-use revm::{context::result::ResultAndState, DatabaseCommit};
+use revm::{DatabaseCommit, context::result::ResultAndState};
 
 impl<N> HlRpcNodeCore for N where N: RpcNodeCore<Primitives: NodePrimitives<Block = HlBlock>> {}
 

src/node/rpc/engine_api/validator.rs

@@ -9,7 +9,7 @@ use alloy_primitives::B256;
 use alloy_rpc_types_engine::PayloadError;
 use reth::{
     api::{FullNodeComponents, NodeTypes},
-    builder::{rpc::PayloadValidatorBuilder, AddOnsContext},
+    builder::{AddOnsContext, rpc::PayloadValidatorBuilder},
 };
 use reth_engine_primitives::{ExecutionPayload, PayloadValidator};
 use reth_payload_primitives::NewPayloadError;

src/node/rpc/estimate.rs

@@ -1,5 +1,5 @@
-use super::{apply_precompiles, HlEthApi, HlRpcNodeCore};
-use alloy_evm::overrides::{apply_state_overrides, StateOverrideError};
+use super::{HlEthApi, HlRpcNodeCore, apply_precompiles};
+use alloy_evm::overrides::{StateOverrideError, apply_state_overrides};
 use alloy_network::TransactionBuilder;
 use alloy_primitives::{TxKind, U256};
 use alloy_rpc_types_eth::state::StateOverride;
@@ -9,19 +9,19 @@ use reth_evm::{ConfigureEvm, Evm, EvmEnvFor, SpecFor, TransactionEnv, TxEnvFor};
 use reth_revm::{database::StateProviderDatabase, db::CacheDB};
 use reth_rpc_convert::{RpcConvert, RpcTxReq};
 use reth_rpc_eth_api::{
-    helpers::{
-        estimate::{update_estimated_gas_range, EstimateCall},
-        Call,
-    },
     AsEthApiError, IntoEthApiError, RpcNodeCore,
+    helpers::{
+        Call,
+        estimate::{EstimateCall, update_estimated_gas_range},
+    },
 };
 use reth_rpc_eth_types::{
-    error::{api::FromEvmHalt, FromEvmError},
     EthApiError, RevertError, RpcInvalidTransactionError,
+    error::{FromEvmError, api::FromEvmHalt},
 };
 use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GAS_ERROR_RATIO};
 use reth_storage_api::StateProvider;
-use revm::context_interface::{result::ExecutionResult, Transaction};
+use revm::context_interface::{Transaction, result::ExecutionResult};
 use tracing::trace;
 
 impl<N, Rpc> EstimateCall for HlEthApi<N, Rpc>
@@ -82,13 +82,12 @@ where
         let mut tx_env = self.create_txn_env(&evm_env, request, &mut db)?;
 
         let mut is_basic_transfer = false;
-        if tx_env.input().is_empty() {
-            if let TxKind::Call(to) = tx_env.kind() {
-                if let Ok(code) = db.db.account_code(&to) {
+        if tx_env.input().is_empty() &&
+            let TxKind::Call(to) = tx_env.kind() &&
+            let Ok(code) = db.db.account_code(&to)
+        {
             is_basic_transfer = code.map(|code| code.is_empty()).unwrap_or(true);
         }
-            }
-        }
 
         if tx_env.gas_price() > 0 {
             highest_gas_limit =
@@ -107,12 +106,12 @@ where
             let mut min_tx_env = tx_env.clone();
             min_tx_env.set_gas_limit(MIN_TRANSACTION_GAS);
 
-            if let Ok(res) = evm.transact(min_tx_env).map_err(Self::Error::from_evm_err) {
-                if res.result.is_success() {
+            if let Ok(res) = evm.transact(min_tx_env).map_err(Self::Error::from_evm_err) &&
+                res.result.is_success()
+            {
                 return Ok(U256::from(MIN_TRANSACTION_GAS));
             }
         }
-        }
 
         trace!(target: "rpc::eth::estimate", ?tx_env, gas_limit = tx_env.gas_limit(), is_basic_transfer, "Starting gas estimation");
 

src/node/rpc/mod.rs

@@ -1,7 +1,7 @@
 use crate::{
+    HlBlock, HlPrimitives,
     chainspec::HlChainSpec,
     node::{evm::apply_precompiles, types::HlExtras},
-    HlBlock, HlPrimitives,
 };
 use alloy_eips::BlockId;
 use alloy_evm::Evm;
@@ -10,19 +10,19 @@ use alloy_primitives::U256;
 use reth::{
     api::{FullNodeTypes, HeaderTy, NodeTypes, PrimitivesTy},
     builder::{
-        rpc::{EthApiBuilder, EthApiCtx},
         FullNodeComponents,
+        rpc::{EthApiBuilder, EthApiCtx},
     },
     rpc::{
-        eth::{core::EthApiInner, DevSigner, FullEthApiServer},
+        eth::{DevSigner, FullEthApiServer, core::EthApiInner},
         server_types::eth::{
-            receipt::EthReceiptConverter, EthApiError, EthStateCache, FeeHistoryCache,
-            GasPriceOracle,
+            EthApiError, EthStateCache, FeeHistoryCache, GasPriceOracle,
+            receipt::EthReceiptConverter,
         },
     },
     tasks::{
-        pool::{BlockingTaskGuard, BlockingTaskPool},
         TaskSpawner,
+        pool::{BlockingTaskGuard, BlockingTaskPool},
     },
 };
 use reth_evm::{ConfigureEvm, Database, EvmEnvFor, HaltReasonFor, InspectorFor, TxEnvFor};
@@ -32,12 +32,12 @@ use reth_provider::{
 };
 use reth_rpc::RpcTypes;
 use reth_rpc_eth_api::{
-    helpers::{
-        pending_block::BuildPendingEnv, spec::SignersForApi, AddDevSigners, EthApiSpec, EthFees,
-        EthState, LoadFee, LoadPendingBlock, LoadState, SpawnBlocking, Trace,
-    },
     EthApiTypes, FromEvmError, RpcConvert, RpcConverter, RpcNodeCore, RpcNodeCoreExt,
     SignableTxRequest,
+    helpers::{
+        AddDevSigners, EthApiSpec, EthFees, EthState, LoadFee, LoadPendingBlock, LoadState,
+        SpawnBlocking, Trace, pending_block::BuildPendingEnv, spec::SignersForApi,
+    },
 };
 use revm::context::result::ResultAndState;
 use std::{fmt, marker::PhantomData, sync::Arc};
@@ -60,12 +60,17 @@ pub(crate) struct HlEthApiInner<N: HlRpcNodeCore, Rpc: RpcConvert> {
 type HlRpcConvert<N, NetworkT> =
     RpcConverter<NetworkT, <N as FullNodeComponents>::Evm, EthReceiptConverter<HlChainSpec>>;
 
-#[derive(Clone)]
 pub struct HlEthApi<N: HlRpcNodeCore, Rpc: RpcConvert> {
     /// Gateway to node's core components.
     pub(crate) inner: Arc<HlEthApiInner<N, Rpc>>,
 }
 
+impl<N: HlRpcNodeCore, Rpc: RpcConvert> Clone for HlEthApi<N, Rpc> {
+    fn clone(&self) -> Self {
+        Self { inner: self.inner.clone() }
+    }
+}
+
 impl<N, Rpc> fmt::Debug for HlEthApi<N, Rpc>
 where
     N: HlRpcNodeCore,
@@ -79,7 +84,7 @@ where
 impl<N, Rpc> EthApiTypes for HlEthApi<N, Rpc>
 where
     N: HlRpcNodeCore,
-    Rpc: RpcConvert<Primitives = N::Primitives, Error = EthApiError>,
+    Rpc: RpcConvert<Primitives = N::Primitives>,
 {
     type Error = EthApiError;
     type NetworkTypes = Rpc::Network;
@@ -155,7 +160,7 @@ where
 impl<N, Rpc> SpawnBlocking for HlEthApi<N, Rpc>
 where
     N: HlRpcNodeCore,
-    Rpc: RpcConvert<Primitives = N::Primitives, Error = EthApiError>,
+    Rpc: RpcConvert<Primitives = N::Primitives>,
 {
     #[inline]
     fn io_task_spawner(&self) -> impl TaskSpawner {

src/node/rpc/precompile.rs

@@ -1,6 +1,6 @@
 use alloy_eips::BlockId;
 use jsonrpsee::proc_macros::rpc;
-use jsonrpsee_core::{async_trait, RpcResult};
+use jsonrpsee_core::{RpcResult, async_trait};
 use reth_rpc_convert::RpcConvert;
 use reth_rpc_eth_types::EthApiError;
 use tracing::trace;

src/node/rpc/transaction.rs

@@ -1,9 +1,11 @@
+use std::time::Duration;
+
 use crate::node::rpc::{HlEthApi, HlRpcNodeCore};
-use alloy_primitives::{Bytes, B256};
+use alloy_primitives::{B256, Bytes};
 use reth::rpc::server_types::eth::EthApiError;
 use reth_rpc_eth_api::{
-    helpers::{spec::SignersForRpc, EthTransactions, LoadTransaction},
     RpcConvert,
+    helpers::{EthTransactions, LoadTransaction, spec::SignersForRpc},
 };
 
 impl<N, Rpc> LoadTransaction for HlEthApi<N, Rpc>
@@ -25,4 +27,8 @@ where
     async fn send_raw_transaction(&self, _tx: Bytes) -> Result<B256, Self::Error> {
         unreachable!()
     }
+
+    fn send_raw_transaction_sync_timeout(&self) -> Duration {
+        self.inner.eth_api.send_raw_transaction_sync_timeout()
+    }
 }

src/node/spot_meta/init.rs

@@ -0,0 +1,103 @@
+use crate::node::{
+    spot_meta::{SpotId, erc20_contract_to_spot_token},
+    storage::tables::{self, SPOT_METADATA_KEY},
+    types::reth_compat,
+};
+use alloy_primitives::Address;
+use reth_db::{
+    DatabaseEnv,
+    cursor::DbCursorRO,
+};
+use reth_db_api::{
+    Database,
+    transaction::DbTx,
+};
+use std::{collections::BTreeMap, sync::Arc};
+use tracing::info;
+
+/// Load spot metadata from database and initialize cache
+pub fn load_spot_metadata_cache(db: &Arc<DatabaseEnv>, chain_id: u64) {
+    // Try to read from database
+    let data = match db.view(|tx| -> Result<Option<Vec<u8>>, reth_db::DatabaseError> {
+        let mut cursor = tx.cursor_read::<tables::SpotMetadata>()?;
+        Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.map(|(_, data)| data.to_vec()))
+    }) {
+        Ok(Ok(data)) => data,
+        Ok(Err(e)) => {
+            info!(
+                "Failed to read spot metadata from database: {}. Will fetch on-demand from API.",
+                e
+            );
+            return;
+        }
+        Err(e) => {
+            info!(
+                "Database view error while loading spot metadata: {}. Will fetch on-demand from API.",
+                e
+            );
+            return;
+        }
+    };
+
+    // Check if data exists
+    let Some(data) = data else {
+        info!(
+            "No spot metadata found in database for chain {}. Run 'init-state' to populate, or it will be fetched on-demand from API.",
+            chain_id
+        );
+        return;
+    };
+
+    // Deserialize metadata
+    let serializable_map = match rmp_serde::from_slice::<BTreeMap<Address, u64>>(&data) {
+        Ok(map) => map,
+        Err(e) => {
+            info!("Failed to deserialize spot metadata: {}. Will fetch on-demand from API.", e);
+            return;
+        }
+    };
+
+    // Convert and initialize cache
+    let metadata: BTreeMap<Address, SpotId> =
+        serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect();
+
+    info!("Loaded spot metadata from database ({} entries)", metadata.len());
+    reth_compat::initialize_spot_metadata_cache(metadata);
+}
+
+/// Initialize spot metadata in database from API
+pub fn init_spot_metadata(
+    db_path: impl AsRef<std::path::Path>,
+    db_args: reth_db::mdbx::DatabaseArguments,
+    chain_id: u64,
+) -> eyre::Result<()> {
+    info!("Initializing spot metadata for chain {}", chain_id);
+
+    let db = Arc::new(reth_db::open_db(db_path.as_ref(), db_args)?);
+
+    // Check if spot metadata already exists
+    let exists = db.view(|tx| -> Result<bool, reth_db::DatabaseError> {
+        let mut cursor = tx.cursor_read::<tables::SpotMetadata>()?;
+        Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.is_some())
+    })??;
+
+    if exists {
+        info!("Spot metadata already exists in database");
+        return Ok(());
+    }
+
+    // Fetch from API
+    let metadata = match erc20_contract_to_spot_token(chain_id) {
+        Ok(m) => m,
+        Err(e) => {
+            info!("Failed to fetch spot metadata from API: {}. Will be fetched on-demand.", e);
+            return Ok(());
+        }
+    };
+
+    // Store to database
+    reth_compat::store_spot_metadata(&db, &metadata)?;
+
+    info!("Successfully fetched and stored spot metadata for chain {}", chain_id);
+    Ok(())
+}

src/node/spot_meta/mod.rs

@@ -5,6 +5,7 @@ use std::collections::BTreeMap;
 
 use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID};
 
+pub mod init;
 mod patch;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -25,7 +26,7 @@ pub struct SpotMeta {
 }
 
 #[derive(Debug, Clone)]
-pub(crate) struct SpotId {
+pub struct SpotId {
     pub index: u64,
 }
 

src/node/spot_meta/patch.rs

@@ -1,5 +1,5 @@
 use crate::node::spot_meta::SpotId;
-use alloy_primitives::{address, Address};
+use alloy_primitives::{Address, address};
 use std::collections::BTreeMap;
 
 /// Testnet-specific fix for #67

src/node/storage/mod.rs

@@ -1,29 +1,27 @@
 use crate::{
-    node::{
-        primitives::tx_wrapper::{convert_to_eth_block_body, convert_to_hl_block_body},
-        types::HlExtras,
-    },
-    HlBlock, HlBlockBody, HlPrimitives,
+    HlBlock, HlBlockBody, HlHeader, HlPrimitives,
+    node::{primitives::TransactionSigned, types::HlExtras},
 };
 use alloy_consensus::BlockHeader;
 use alloy_primitives::Bytes;
 use reth_chainspec::EthereumHardforks;
 use reth_db::{
+    DbTxUnwindExt,
     cursor::{DbCursorRO, DbCursorRW},
     transaction::{DbTx, DbTxMut},
-    DbTxUnwindExt,
 };
+use reth_primitives_traits::Block;
 use reth_provider::{
-    providers::{ChainStorage, NodeTypesForProvider},
     BlockBodyReader, BlockBodyWriter, ChainSpecProvider, ChainStorageReader, ChainStorageWriter,
     DBProvider, DatabaseProvider, EthStorage, ProviderResult, ReadBodyInput, StorageLocation,
+    providers::{ChainStorage, NodeTypesForProvider},
 };
 
 pub mod tables;
 
 #[derive(Debug, Clone, Default)]
 #[non_exhaustive]
-pub struct HlStorage(EthStorage);
+pub struct HlStorage(EthStorage<TransactionSigned, HlHeader>);
 
 impl HlStorage {
     fn write_precompile_calls<Provider>(
@@ -89,30 +87,17 @@ where
         let mut read_precompile_calls = Vec::with_capacity(bodies.len());
 
         for (block_number, body) in bodies {
-            match body {
+            let (inner_opt, extras) = match body {
                 Some(HlBlockBody {
                     inner,
                     sidecars: _,
-                    read_precompile_calls: rpc,
+                    read_precompile_calls,
                     highest_precompile_address,
-                }) => {
-                    eth_bodies.push((block_number, Some(convert_to_eth_block_body(inner))));
-                    read_precompile_calls.push((
-                        block_number,
-                        HlExtras { read_precompile_calls: rpc, highest_precompile_address },
-                    ));
-                }
-                None => {
-                    eth_bodies.push((block_number, None));
-                    read_precompile_calls.push((
-                        block_number,
-                        HlExtras {
-                            read_precompile_calls: Default::default(),
-                            highest_precompile_address: None,
-                        },
-                    ));
-                }
-            }
+                }) => (Some(inner), HlExtras { read_precompile_calls, highest_precompile_address }),
+                None => Default::default(),
+            };
+            eth_bodies.push((block_number, inner_opt));
+            read_precompile_calls.push((block_number, extras));
         }
 
         self.0.write_block_bodies(provider, eth_bodies, write_to)?;
@@ -146,22 +131,16 @@ where
         inputs: Vec<ReadBodyInput<'_, Self::Block>>,
     ) -> ProviderResult<Vec<HlBlockBody>> {
         let read_precompile_calls = self.read_precompile_calls(provider, &inputs)?;
-        let eth_bodies = self.0.read_block_bodies(
-            provider,
-            inputs
-                .into_iter()
-                .map(|(header, transactions)| {
-                    (header, transactions.into_iter().map(|tx| tx.into_inner()).collect())
-                })
-                .collect(),
-        )?;
+        let inputs: Vec<(&<Self::Block as Block>::Header, _)> = inputs;
+        let eth_bodies = self.0.read_block_bodies(provider, inputs)?;
+        let eth_bodies: Vec<alloy_consensus::BlockBody<_, HlHeader>> = eth_bodies;
 
         // NOTE: sidecars are not used in HyperEVM yet.
         Ok(eth_bodies
             .into_iter()
             .zip(read_precompile_calls)
             .map(|(inner, extra)| HlBlockBody {
-                inner: convert_to_hl_block_body(inner),
+                inner,
                 sidecars: None,
                 read_precompile_calls: extra.read_precompile_calls,
                 highest_precompile_address: extra.highest_precompile_address,

src/node/storage/tables.rs

@@ -1,11 +1,22 @@
 use alloy_primitives::{BlockNumber, Bytes};
-use reth_db::{table::TableInfo, tables, TableSet, TableType, TableViewer};
+use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables};
 use std::fmt;
 
+/// Static key used for spot metadata, as the database is unique to each chain.
+/// This may later serve as a versioning key to assist with future database migrations.
+pub const SPOT_METADATA_KEY: u64 = 0;
+
 tables! {
     /// Read precompile calls for each block.
     table BlockReadPrecompileCalls {
         type Key = BlockNumber;
         type Value = Bytes;
     }
+
+    /// Spot metadata mapping (EVM address to spot token index).
+    /// Uses a constant key since the database is chain-specific.
+    table SpotMetadata {
+        type Key = u64;
+        type Value = Bytes;
+    }
 }

src/node/types/mod.rs

@@ -2,26 +2,39 @@
 //!
 //! Changes:
 //! - ReadPrecompileCalls supports RLP encoding / decoding
-use alloy_primitives::{Address, Bytes, Log, B256};
+use alloy_consensus::TxType;
+use alloy_primitives::{Address, B256, Bytes, Log};
 use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
 use bytes::BufMut;
+use reth_ethereum_primitives::EthereumReceipt;
+use reth_primitives_traits::InMemorySize;
 use serde::{Deserialize, Serialize};
 
 use crate::HlBlock;
 
 pub type ReadPrecompileCall = (Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>);
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Default, Hash)]
 pub struct ReadPrecompileCalls(pub Vec<ReadPrecompileCall>);
 
 pub(crate) mod reth_compat;
 
+// Re-export spot metadata functions
+pub use reth_compat::{initialize_spot_metadata_cache, set_spot_metadata_db};
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct HlExtras {
     pub read_precompile_calls: Option<ReadPrecompileCalls>,
     pub highest_precompile_address: Option<Address>,
 }
 
+impl InMemorySize for HlExtras {
+    fn size(&self) -> usize {
+        self.read_precompile_calls.as_ref().map_or(0, |s| s.0.len()) +
+            self.highest_precompile_address.as_ref().map_or(0, |_| 20)
+    }
+}
+
 impl Encodable for ReadPrecompileCalls {
     fn encode(&self, out: &mut dyn BufMut) {
         let buf: Bytes = rmp_serde::to_vec(&self.0).unwrap().into();
@@ -56,6 +69,7 @@ impl BlockAndReceipts {
             self.read_precompile_calls.clone(),
             self.highest_precompile_address,
             self.system_txs.clone(),
+            self.receipts.clone(),
             chain_id,
         )
     }
@@ -84,6 +98,23 @@ pub struct LegacyReceipt {
     logs: Vec<Log>,
 }
 
+impl From<LegacyReceipt> for EthereumReceipt {
+    fn from(r: LegacyReceipt) -> Self {
+        EthereumReceipt {
+            tx_type: match r.tx_type {
+                LegacyTxType::Legacy => TxType::Legacy,
+                LegacyTxType::Eip2930 => TxType::Eip2930,
+                LegacyTxType::Eip1559 => TxType::Eip1559,
+                LegacyTxType::Eip4844 => TxType::Eip4844,
+                LegacyTxType::Eip7702 => TxType::Eip7702,
+            },
+            success: r.success,
+            cumulative_gas_used: r.cumulative_gas_used,
+            logs: r.logs,
+        }
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
 enum LegacyTxType {
     Legacy = 0,
@@ -99,6 +130,19 @@ pub struct SystemTx {
     pub receipt: Option<LegacyReceipt>,
 }
 
+impl SystemTx {
+    pub fn gas_limit(&self) -> u64 {
+        use reth_compat::Transaction;
+        match &self.tx {
+            Transaction::Legacy(tx) => tx.gas_limit,
+            Transaction::Eip2930(tx) => tx.gas_limit,
+            Transaction::Eip1559(tx) => tx.gas_limit,
+            Transaction::Eip4844(tx) => tx.gas_limit,
+            Transaction::Eip7702(tx) => tx.gas_limit,
+        }
+    }
+}
+
 #[derive(
     Debug,
     Clone,
@@ -117,7 +161,7 @@ pub struct ReadPrecompileInput {
     pub gas_limit: u64,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Hash)]
 pub enum ReadPrecompileResult {
     Ok { gas_used: u64, bytes: Bytes },
     OutOfGas,

src/node/types/reth_compat.rs

@@ -1,21 +1,24 @@
 //! Copy of reth codebase to preserve serialization compatibility
+use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata};
 use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy};
-use alloy_primitives::{Address, BlockHash, Signature, TxKind, U256};
+use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256};
+use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW};
+use reth_db_api::{Database, transaction::DbTxMut};
 use reth_primitives::TransactionSigned as RethTxSigned;
 use serde::{Deserialize, Serialize};
 use std::{
     collections::BTreeMap,
-    sync::{Arc, LazyLock, RwLock},
+    sync::{Arc, LazyLock, Mutex, RwLock},
 };
 use tracing::info;
 
 use crate::{
+    HlBlock, HlBlockBody, HlHeader,
     node::{
         primitives::TransactionSigned as TxSigned,
-        spot_meta::{erc20_contract_to_spot_token, SpotId},
-        types::{ReadPrecompileCalls, SystemTx},
+        spot_meta::{SpotId, erc20_contract_to_spot_token},
+        types::{LegacyReceipt, ReadPrecompileCalls, SystemTx},
     },
-    HlBlock, HlBlockBody,
 };
 
 /// A raw transaction.
@@ -81,10 +84,56 @@ pub struct SealedBlock {
     pub body: BlockBody,
 }
 
-fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned {
-    static EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
+static SPOT_EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
     LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new())));
-    {
+
+// Optional database handle for persisting on-demand fetches
+static DB_HANDLE: LazyLock<Mutex<Option<Arc<DatabaseEnv>>>> = LazyLock::new(|| Mutex::new(None));
+
+/// Set the database handle for persisting spot metadata
+pub fn set_spot_metadata_db(db: Arc<DatabaseEnv>) {
+    *DB_HANDLE.lock().unwrap() = Some(db);
+}
+
+/// Initialize the spot metadata cache with data loaded from database.
+/// This should be called during node initialization.
+pub fn initialize_spot_metadata_cache(metadata: BTreeMap<Address, SpotId>) {
+    *SPOT_EVM_MAP.write().unwrap() = metadata;
+}
+
+/// Helper function to serialize and store spot metadata to database
+pub fn store_spot_metadata(
+    db: &Arc<DatabaseEnv>,
+    metadata: &BTreeMap<Address, SpotId>,
+) -> Result<(), DatabaseError> {
+    db.update(|tx| {
+        let mut cursor = tx.cursor_write::<SpotMetadata>()?;
+
+        // Serialize to BTreeMap<Address, u64>
+        let serializable_map: BTreeMap<Address, u64> =
+            metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect();
+
+        cursor.upsert(
+            SPOT_METADATA_KEY,
+            &Bytes::from(
+                rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"),
+            ),
+        )?;
+        Ok(())
+    })?
+}
+
+/// Persist spot metadata to database if handle is available
+fn persist_spot_metadata_to_db(metadata: &BTreeMap<Address, SpotId>) {
+    if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() {
+        match store_spot_metadata(db, metadata) {
+            Ok(_) => info!("Persisted spot metadata to database"),
+            Err(e) => info!("Failed to persist spot metadata to database: {}", e),
+        }
+    }
+}
+
+fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned {
     let Transaction::Legacy(tx) = &transaction.tx else {
         panic!("Unexpected transaction type");
     };
@@ -95,17 +144,19 @@ fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSig
         U256::from(0x1)
     } else {
         loop {
-                if let Some(spot) = EVM_MAP.read().unwrap().get(&to) {
+            if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) {
                 break spot.to_s();
             }
 
-                info!("Contract not found: {to:?} from spot mapping, fetching again...");
-                *EVM_MAP.write().unwrap() = erc20_contract_to_spot_token(chain_id).unwrap();
+            // Cache miss - fetch from API, update cache, and persist to database
+            info!("Contract not found: {to:?} from spot mapping, fetching from API...");
+            let metadata = erc20_contract_to_spot_token(chain_id).unwrap();
+            *SPOT_EVM_MAP.write().unwrap() = metadata.clone();
+            persist_spot_metadata_to_db(&metadata);
         }
     };
     let signature = Signature::new(U256::from(0x1), s, true);
     TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature)))
-    }
 }
 
 impl SealedBlock {
@@ -113,23 +164,40 @@ impl SealedBlock {
         &self,
         read_precompile_calls: ReadPrecompileCalls,
         highest_precompile_address: Option<Address>,
-        system_txs: Vec<super::SystemTx>,
+        mut system_txs: Vec<super::SystemTx>,
+        receipts: Vec<LegacyReceipt>,
         chain_id: u64,
     ) -> HlBlock {
+        // NOTE: These types of transactions are tracked at #97.
+        system_txs.retain(|tx| tx.receipt.is_some());
+
         let mut merged_txs = vec![];
         merged_txs.extend(system_txs.iter().map(|tx| system_tx_to_reth_transaction(tx, chain_id)));
         merged_txs.extend(self.body.transactions.iter().map(|tx| tx.to_reth_transaction()));
+
+        let mut merged_receipts = vec![];
+        merged_receipts.extend(system_txs.iter().map(|tx| tx.receipt.clone().unwrap().into()));
+        merged_receipts.extend(receipts.into_iter().map(From::from));
+
         let block_body = HlBlockBody {
             inner: reth_primitives::BlockBody {
                 transactions: merged_txs,
                 withdrawals: self.body.withdrawals.clone(),
-                ommers: self.body.ommers.clone(),
+                ommers: vec![],
             },
             sidecars: None,
             read_precompile_calls: Some(read_precompile_calls),
             highest_precompile_address,
         };
 
-        HlBlock { header: self.header.header.clone(), body: block_body }
+        let system_tx_count = system_txs.len() as u64;
+        HlBlock {
+            header: HlHeader::from_ethereum_header(
+                self.header.header.clone(),
+                &merged_receipts,
+                system_tx_count,
+            ),
+            body: block_body,
+        }
     }
 }

src/pseudo_peer/config.rs

@@ -46,7 +46,7 @@ impl BlockSourceConfig {
                     .expect("home dir not found")
                     .join("hl")
                     .join("data")
-                    .join("evm_blocks_and_receipts"),
+                    .join("evm_block_and_receipts"),
             },
             block_source_from_node: None,
         }

src/pseudo_peer/network.rs

@@ -1,8 +1,8 @@
 use super::service::{BlockHashCache, BlockPoller};
-use crate::{chainspec::HlChainSpec, node::network::HlNetworkPrimitives, HlPrimitives};
+use crate::{HlPrimitives, chainspec::HlChainSpec, node::network::HlNetworkPrimitives};
 use reth_network::{
-    config::{rng_secret_key, SecretKey},
     NetworkConfig, NetworkManager, PeersConfig,
+    config::{SecretKey, rng_secret_key},
 };
 use reth_network_peers::TrustedPeer;
 use reth_provider::test_utils::NoopProvider;

src/pseudo_peer/service.rs

@@ -81,13 +81,13 @@ impl BlockPoller {
             .await
             .ok_or(eyre::eyre!("Failed to find latest block number"))?;
 
-        if let Some(debug_cutoff_height) = debug_cutoff_height {
-            if next_block_number > debug_cutoff_height {
+        loop {
+            if let Some(debug_cutoff_height) = debug_cutoff_height &&
+                next_block_number > debug_cutoff_height
+            {
                 next_block_number = debug_cutoff_height;
             }
-        }
 
-        loop {
             match block_source.collect_block(next_block_number).await {
                 Ok(block) => {
                     block_tx.send((next_block_number, block)).await?;

src/pseudo_peer/sources/cached.rs

@@ -1,6 +1,6 @@
 use super::{BlockSource, BlockSourceBoxed};
 use crate::node::types::BlockAndReceipts;
-use futures::{future::BoxFuture, FutureExt};
+use futures::{FutureExt, future::BoxFuture};
 use reth_network::cache::LruMap;
 use std::sync::{Arc, RwLock};
 

src/pseudo_peer/sources/hl_node/cache.rs

@@ -27,7 +27,7 @@ impl LocalBlocksCache {
     }
 
     pub fn get_block(&mut self, height: u64) -> Option<BlockAndReceipts> {
-        self.cache.remove(&height)
+        self.cache.get(&height).cloned()
     }
 
     pub fn get_path_for_height(&self, height: u64) -> Option<PathBuf> {

src/pseudo_peer/sources/hl_node/file_ops.rs

@@ -1,4 +1,4 @@
-use super::{scan::Scanner, time_utils::TimeUtils, HOURLY_SUBDIR};
+use super::{HOURLY_SUBDIR, scan::Scanner, time_utils::TimeUtils};
 use crate::node::types::BlockAndReceipts;
 use std::{
     fs::File,

src/pseudo_peer/sources/hl_node/mod.rs

@@ -8,13 +8,13 @@ mod time_utils;
 use self::{
     cache::LocalBlocksCache,
     file_ops::FileOperations,
-    scan::{ScanOptions, Scanner},
+    scan::{LineStream, ScanOptions, Scanner},
     time_utils::TimeUtils,
 };
 use super::{BlockSource, BlockSourceBoxed};
 use crate::node::types::BlockAndReceipts;
 use futures::future::BoxFuture;
-use reth_metrics::{metrics, metrics::Counter, Metrics};
+use reth_metrics::{Metrics, metrics, metrics::Counter};
 use std::{
     path::{Path, PathBuf},
     sync::Arc,
@@ -52,6 +52,8 @@ pub struct HlNodeBlockSourceMetrics {
     pub fetched_from_hl_node: Counter,
     /// How many times the HL node block source is fetched from the fallback
     pub fetched_from_fallback: Counter,
+    /// How many times `try_collect_local_block` was faster than ingest loop
+    pub file_read_triggered: Counter,
 }
 
 impl BlockSource for HlNodeBlockSource {
@@ -64,7 +66,9 @@ impl BlockSource for HlNodeBlockSource {
         Box::pin(async move {
             let now = OffsetDateTime::now_utc();
 
-            if let Some(block) = Self::try_collect_local_block(local_blocks_cache, height).await {
+            if let Some(block) =
+                Self::try_collect_local_block(&metrics, local_blocks_cache, height).await
+            {
                 Self::update_last_fetch(last_local_fetch, height, now).await;
                 metrics.fetched_from_hl_node.increment(1);
                 return Ok(block);
@@ -120,6 +124,28 @@ impl BlockSource for HlNodeBlockSource {
     }
 }
 
+struct CurrentFile {
+    path: PathBuf,
+    line_stream: Option<LineStream>,
+}
+
+impl CurrentFile {
+    pub fn from_datetime(dt: OffsetDateTime, root: &Path) -> Self {
+        let (hour, day_str) = (dt.hour(), TimeUtils::date_from_datetime(dt));
+        let path = root.join(HOURLY_SUBDIR).join(&day_str).join(format!("{}", hour));
+        Self { path, line_stream: None }
+    }
+
+    pub fn open(&mut self) -> eyre::Result<()> {
+        if self.line_stream.is_some() {
+            return Ok(());
+        }
+
+        self.line_stream = Some(LineStream::from_path(&self.path)?);
+        Ok(())
+    }
+}
+
 impl HlNodeBlockSource {
     async fn update_last_fetch(
         last_local_fetch: Arc<Mutex<Option<(u64, OffsetDateTime)>>>,
@@ -133,6 +159,7 @@ impl HlNodeBlockSource {
     }
 
     async fn try_collect_local_block(
+        metrics: &HlNodeBlockSourceMetrics,
         local_blocks_cache: Arc<Mutex<LocalBlocksCache>>,
         height: u64,
     ) -> Option<BlockAndReceipts> {
@@ -142,9 +169,10 @@ impl HlNodeBlockSource {
         }
         let path = u_cache.get_path_for_height(height)?;
         info!("Loading block data from {:?}", path);
+        metrics.file_read_triggered.increment(1);
+        let mut line_stream = LineStream::from_path(&path).ok()?;
         let scan_result = Scanner::scan_hour_file(
-            &path,
-            &mut 0,
+            &mut line_stream,
             ScanOptions { start_height: 0, only_load_ranges: false },
         );
         u_cache.load_scan_result(scan_result);
@@ -165,9 +193,10 @@ impl HlNodeBlockSource {
             } else {
                 warn!("Failed to parse last line of file: {:?}", subfile);
             }
+            let mut line_stream =
+                LineStream::from_path(&subfile).expect("Failed to open line stream");
             let mut scan_result = Scanner::scan_hour_file(
-                &subfile,
-                &mut 0,
+                &mut line_stream,
                 ScanOptions { start_height: cutoff_height, only_load_ranges: true },
             );
             scan_result.new_blocks.clear(); // Only store ranges, load data lazily
@@ -188,15 +217,13 @@ impl HlNodeBlockSource {
                 }
                 tokio::time::sleep(TAIL_INTERVAL).await;
             };
-            let (mut hour, mut day_str, mut last_line) =
-                (dt.hour(), TimeUtils::date_from_datetime(dt), 0);
+            let mut current_file = CurrentFile::from_datetime(dt, &root);
             info!("Starting local ingest loop from height: {}", current_head);
             loop {
-                let hour_file = root.join(HOURLY_SUBDIR).join(&day_str).join(format!("{hour}"));
-                if hour_file.exists() {
+                let _ = current_file.open();
+                if let Some(line_stream) = &mut current_file.line_stream {
                     let scan_result = Scanner::scan_hour_file(
-                        &hour_file,
-                        &mut last_line,
+                        line_stream,
                         ScanOptions { start_height: next_height, only_load_ranges: false },
                     );
                     next_height = scan_result.next_expected_height;
@@ -205,11 +232,8 @@ impl HlNodeBlockSource {
                 let now = OffsetDateTime::now_utc();
                 if dt + ONE_HOUR < now {
                     dt += ONE_HOUR;
-                    (hour, day_str, last_line) = (dt.hour(), TimeUtils::date_from_datetime(dt), 0);
-                    info!(
-                        "Moving to new file: {:?}",
-                        root.join(HOURLY_SUBDIR).join(&day_str).join(format!("{hour}"))
-                    );
+                    current_file = CurrentFile::from_datetime(dt, &root);
+                    info!("Moving to new file: {:?}", current_file.path);
                     continue;
                 }
                 tokio::time::sleep(TAIL_INTERVAL).await;

src/pseudo_peer/sources/hl_node/scan.rs

@@ -2,7 +2,7 @@ use crate::node::types::{BlockAndReceipts, EvmBlock};
 use serde::{Deserialize, Serialize};
 use std::{
     fs::File,
-    io::{BufRead, BufReader},
+    io::{BufRead, BufReader, Seek, SeekFrom},
     ops::RangeInclusive,
     path::{Path, PathBuf},
 };
@@ -25,6 +25,57 @@ pub struct ScanOptions {
 
 pub struct Scanner;
 
+/// Stream for sequentially reading lines from a file.
+///
+/// This struct allows sequential iteration over lines over [Self::next] method.
+/// It is resilient to cases where the line producer process is interrupted while writing:
+/// - If a line is incomplete but still ends with a line ending, it is skipped: later, the fallback
+///   block source will be used to retrieve the missing block.
+/// - If a line does not end with a newline (i.e., the write was incomplete), the method returns
+///   `None` to break out of the loop and avoid reading partial data.
+/// - If a temporary I/O error occurs, the stream exits the loop without rewinding the cursor, which
+///   will result in skipping ahead to the next unread bytes.
+pub struct LineStream {
+    path: PathBuf,
+    reader: BufReader<File>,
+}
+
+impl LineStream {
+    pub fn from_path(path: &Path) -> std::io::Result<Self> {
+        let reader = BufReader::with_capacity(1024 * 1024, File::open(path)?);
+        Ok(Self { path: path.to_path_buf(), reader })
+    }
+
+    pub fn next(&mut self) -> Option<String> {
+        let mut line_buffer = vec![];
+        let Ok(size) = self.reader.read_until(b'\n', &mut line_buffer) else {
+            // Temporary I/O error; restart the loop
+            return None;
+        };
+
+        // Now cursor is right after the end of the line
+        // On UTF-8 error, skip the line
+        let Ok(mut line) = String::from_utf8(line_buffer) else {
+            return Some(String::new());
+        };
+
+        // If line is not completed yet, return None so that we can break the loop
+        if line.ends_with('\n') {
+            if line.ends_with('\r') {
+                line.pop();
+            }
+            line.pop();
+            return Some(line);
+        }
+
+        // info!("Line is not completed yet: {}", line);
+        if size != 0 {
+            self.reader.seek(SeekFrom::Current(-(size as i64))).unwrap();
+        }
+        None
+    }
+}
+
 impl Scanner {
     pub fn line_to_evm_block(line: &str) -> serde_json::Result<(BlockAndReceipts, u64)> {
         let LocalBlockAndReceipts(_, parsed_block): LocalBlockAndReceipts =
@@ -35,31 +86,20 @@ impl Scanner {
         Ok((parsed_block, height))
     }
 
-    pub fn scan_hour_file(path: &Path, last_line: &mut usize, options: ScanOptions) -> ScanResult {
-        let lines: Vec<String> =
-            BufReader::new(File::open(path).expect("Failed to open hour file"))
-                .lines()
-                .collect::<Result<_, _>>()
-                .unwrap();
-        let skip = if *last_line == 0 { 0 } else { *last_line - 1 };
+    pub fn scan_hour_file(line_stream: &mut LineStream, options: ScanOptions) -> ScanResult {
         let mut new_blocks = Vec::new();
         let mut last_height = options.start_height;
         let mut block_ranges = Vec::new();
         let mut current_range: Option<(u64, u64)> = None;
 
-        for (line_idx, line) in lines.iter().enumerate().skip(skip) {
-            if line_idx < *last_line || line.trim().is_empty() {
-                continue;
-            }
-
-            match Self::line_to_evm_block(line) {
+        while let Some(line) = line_stream.next() {
+            match Self::line_to_evm_block(&line) {
                 Ok((parsed_block, height)) => {
                     if height >= options.start_height {
                         last_height = last_height.max(height);
                         if !options.only_load_ranges {
                             new_blocks.push(parsed_block);
                         }
-                        *last_line = line_idx;
                     }
 
                     match current_range {
@@ -74,16 +114,17 @@ impl Scanner {
                         }
                     }
                 }
-                Err(_) => warn!("Failed to parse line: {}...", line.get(0..50).unwrap_or(line)),
+                Err(_) => warn!("Failed to parse line: {}...", line.get(0..50).unwrap_or(&line)),
             }
         }
 
         if let Some((start, end)) = current_range {
             block_ranges.push(start..=end);
         }
+
         ScanResult {
-            path: path.to_path_buf(),
-            next_expected_height: last_height + 1,
+            path: line_stream.path.clone(),
+            next_expected_height: last_height + current_range.is_some() as u64,
             new_blocks,
             new_block_ranges: block_ranges,
         }

src/pseudo_peer/sources/hl_node/tests.rs

@@ -1,10 +1,10 @@
 use super::*;
 use crate::{
-    node::types::{reth_compat, ReadPrecompileCalls},
-    pseudo_peer::sources::{hl_node::scan::LocalBlockAndReceipts, LocalBlockSource},
+    node::types::{ReadPrecompileCalls, reth_compat},
+    pseudo_peer::sources::{LocalBlockSource, hl_node::scan::LocalBlockAndReceipts},
 };
 use alloy_consensus::{BlockBody, Header};
-use alloy_primitives::{Address, Bloom, Bytes, B256, B64, U256};
+use alloy_primitives::{Address, B64, B256, Bloom, Bytes, U256};
 use std::{io::Write, time::Duration};
 
 const DEFAULT_FALLBACK_THRESHOLD_FOR_TEST: Duration = Duration::from_millis(5000);

src/pseudo_peer/sources/hl_node/time_utils.rs

@@ -1,5 +1,5 @@
 use std::path::Path;
-use time::{macros::format_description, Date, OffsetDateTime, Time};
+use time::{Date, OffsetDateTime, Time, macros::format_description};
 
 pub struct TimeUtils;
 

src/pseudo_peer/sources/local.rs

@@ -1,8 +1,8 @@
-use super::{utils, BlockSource};
+use super::{BlockSource, utils};
 use crate::node::types::BlockAndReceipts;
 use eyre::Context;
-use futures::{future::BoxFuture, FutureExt};
-use reth_metrics::{metrics, metrics::Counter, Metrics};
+use futures::{FutureExt, future::BoxFuture};
+use reth_metrics::{Metrics, metrics, metrics::Counter};
 use std::path::PathBuf;
 use tracing::info;
 

src/pseudo_peer/sources/s3.rs

@@ -1,8 +1,8 @@
-use super::{utils, BlockSource};
+use super::{BlockSource, utils};
 use crate::node::types::BlockAndReceipts;
 use aws_sdk_s3::types::RequestPayer;
-use futures::{future::BoxFuture, FutureExt};
-use reth_metrics::{metrics, metrics::Counter, Metrics};
+use futures::{FutureExt, future::BoxFuture};
+use reth_metrics::{Metrics, metrics, metrics::Counter};
 use std::{sync::Arc, time::Duration};
 use tracing::info;
 

src/version.rs

@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 
-use reth_node_core::version::{try_init_version_metadata, RethCliVersionConsts};
+use reth_node_core::version::{RethCliVersionConsts, try_init_version_metadata};
 
 pub fn init_reth_hl_version() {
     let cargo_pkg_version = env!("CARGO_PKG_VERSION").to_string();

tests/run_tests.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e
+
+export ETH_RPC_URL="${ETH_RPC_URL:-wss://hl-archive-node.xyz}"
+
+success() {
+    echo "Success: $1"
+}
+
+fail() {
+    echo "Failed: $1"
+    exit 1
+}
+
+ensure_cmd() {
+    command -v "$1" > /dev/null 2>&1 || fail "$1 is required"
+}
+
+ensure_cmd jq
+ensure_cmd cast
+ensure_cmd wscat
+
+if [[ ! "$ETH_RPC_URL" =~ ^wss?:// ]]; then
+    fail "ETH_RPC_URL must be a websocket url"
+fi
+
+TITLE="Issue #78 - eth_getLogs should return system transactions"
+cast logs \
+    --rpc-url "$ETH_RPC_URL" \
+    --from-block 15312567 \
+    --to-block 15312570 \
+    --address 0x9fdbda0a5e284c32744d2f17ee5c74b284993463 \
+    0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef \
+    | grep -q "0x00000000000000000000000020000000000000000000000000000000000000c5" \
+    && success "$TITLE" || fail "$TITLE"
+
+TITLE="Issue #78 - eth_getBlockByNumber should return the same logsBloom as official RPC"
+OFFICIAL_RPC="https://rpc.hyperliquid.xyz/evm"
+A=$(cast block 1394092 --rpc-url "$ETH_RPC_URL" -f logsBloom | md5sum)
+B=$(cast block 1394092 --rpc-url "$OFFICIAL_RPC" -f logsBloom | md5sum)
+echo node "$A"
+echo rpc\  "$B"
+[[ "$A" == "$B" ]] && success "$TITLE" || fail "$TITLE"
+
+TITLE="eth_subscribe newHeads via wscat"
+CMD='{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}'
+wscat -w 2 -c "$ETH_RPC_URL" -x "$CMD" | tail -1 | jq -r .params.result.nonce | grep 0x \
+    && success "$TITLE" || fail "$TITLE"