doc: Update testnet block number to 34112653

feat: cache spot metadata in database to reduce API calls

README.md

@@ -60,19 +60,19 @@ $ reth-hl node --http --http.addr 0.0.0.0 --http.api eth,ots,net,web3 \
 
 ## How to run (testnet)
 
-Testnet is supported since block 30281484.
+Testnet is supported since block 34112653.
 
 ```sh
-# Get testnet genesis at block 30281484
+# Get testnet genesis at block 34112653
 $ cd ~
 $ git clone https://github.com/sprites0/hl-testnet-genesis
 $ zstd --rm -d ~/hl-testnet-genesis/*.zst
 
 # Init node
 $ make install
-$ reth-hl init-state --without-evm --chain testnet --header ~/hl-testnet-genesis/30281484.rlp \
+$ reth-hl init-state --without-evm --chain testnet --header ~/hl-testnet-genesis/34112653.rlp \
-  --header-hash 0x147cc3c09e9ddbb11799c826758db284f77099478ab5f528d3a57a6105516c21 \
+  --header-hash 0xeb79aca618ab9fda6d463fddd3ad439045deada1f539cbab1c62d7e6a0f5859a \
-  ~/hl-testnet-genesis/30281484.jsonl --total-difficulty 0 
+  ~/hl-testnet-genesis/34112653.jsonl --total-difficulty 0 
 
 # Run node
 $ reth-hl node --chain testnet --http --http.addr 0.0.0.0 --http.api eth,ots,net,web3 \

src/addons/mod.rs

@@ -1,5 +1,5 @@
 pub mod call_forwarder;
 pub mod hl_node_compliance;
-pub mod tx_forwarder;
 pub mod subscribe_fixup;
+pub mod tx_forwarder;
 mod utils;

src/chainspec/mod.rs

@@ -1,7 +1,10 @@
 pub mod hl;
 pub mod parser;
 
-use crate::{hardforks::HlHardforks, node::primitives::{header::HlHeaderExtras, HlHeader}};
+use crate::{
+    hardforks::HlHardforks,
+    node::primitives::{HlHeader, header::HlHeaderExtras},
+};
 use alloy_eips::eip7840::BlobParams;
 use alloy_genesis::Genesis;
 use alloy_primitives::{Address, B256, U256};

src/main.rs

@@ -41,8 +41,11 @@ fn main() -> eyre::Result<()> {
          ext: HlNodeArgs| async move {
             let default_upstream_rpc_url = builder.config().chain.official_rpc_url();
 
-            let (node, engine_handle_tx) =
+            let (node, engine_handle_tx) = HlNode::new(
-                HlNode::new(ext.block_source_args.parse().await?, ext.debug_cutoff_height);
+                ext.block_source_args.parse().await?,
+                ext.debug_cutoff_height,
+                ext.allow_network_overrides,
+            );
             let NodeHandle { node, node_exit_future: exit_future } = builder
                 .node(node)
                 .extend_rpc_modules(move |mut ctx| {

src/node/cli.rs

@@ -1,12 +1,8 @@
 use crate::{
     chainspec::{HlChainSpec, parser::HlChainSpecParser},
     node::{
-        HlNode,
+        HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator,
-        consensus::HlConsensus,
+        spot_meta::init as spot_meta_init, storage::tables::Tables,
-        evm::config::HlEvmConfig,
-        migrate::Migrator,
-        spot_meta::init as spot_meta_init,
-        storage::tables::Tables,
     },
     pseudo_peer::BlockSourceArgs,
 };
@@ -24,7 +20,10 @@ use reth_cli::chainspec::ChainSpecParser;
 use reth_cli_commands::{common::EnvironmentArgs, launcher::FnLauncher};
 use reth_db::{DatabaseEnv, init_db, mdbx::init_db_for};
 use reth_tracing::FileWorkerGuard;
-use std::{fmt::{self}, sync::Arc};
+use std::{
+    fmt::{self},
+    sync::Arc,
+};
 use tracing::info;
 
 macro_rules! not_applicable {
@@ -83,6 +82,13 @@ pub struct HlNodeArgs {
     /// * Refers to the Merkle trie used for eth_getProof and state root, not actual state values.
     #[arg(long, env = "EXPERIMENTAL_ETH_GET_PROOF")]
     pub experimental_eth_get_proof: bool,
+
+    /// Allow network configuration overrides from CLI.
+    ///
+    /// When enabled, network settings (discovery_addr, listener_addr, dns_discovery, nat)
+    /// will be taken from CLI arguments instead of being hardcoded to localhost-only defaults.
+    #[arg(long, env = "ALLOW_NETWORK_OVERRIDES")]
+    pub allow_network_overrides: bool,
 }
 
 /// The main reth_hl cli interface.

src/node/consensus/mod.rs

@@ -1,4 +1,8 @@
-use crate::{hardforks::HlHardforks, node::{primitives::HlHeader, HlNode}, HlBlock, HlBlockBody, HlPrimitives};
+use crate::{
+    HlBlock, HlBlockBody, HlPrimitives,
+    hardforks::HlHardforks,
+    node::{HlNode, primitives::HlHeader},
+};
 use reth::{
     api::{FullNodeTypes, NodeTypes},
     beacon_consensus::EthBeaconConsensus,

src/node/evm/assembler.rs

@@ -1,5 +1,6 @@
 use crate::{
-    node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, HlBlock, HlHeader
+    HlBlock, HlHeader,
+    node::evm::config::{HlBlockExecutorFactory, HlEvmConfig},
 };
 use reth_evm::{
     block::BlockExecutionError,

src/node/migrate.rs

@@ -270,8 +270,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> {
         let mut first = true;
 
         for (block_range, _tx_ranges) in all_static_files {
-            let migration_needed = self.using_old_header(block_range.start())?
+            let migration_needed = self.using_old_header(block_range.start())? ||
-                || self.using_old_header(block_range.end())?;
+                self.using_old_header(block_range.end())?;
             if !migration_needed {
                 // Create a placeholder symlink
                 self.create_placeholder(block_range)?;

src/node/mod.rs

@@ -51,12 +51,14 @@ pub struct HlNode {
     engine_handle_rx: Arc<Mutex<Option<oneshot::Receiver<ConsensusEngineHandle<HlPayloadTypes>>>>>,
     block_source_config: BlockSourceConfig,
     debug_cutoff_height: Option<u64>,
+    allow_network_overrides: bool,
 }
 
 impl HlNode {
     pub fn new(
         block_source_config: BlockSourceConfig,
         debug_cutoff_height: Option<u64>,
+        allow_network_overrides: bool,
     ) -> (Self, oneshot::Sender<ConsensusEngineHandle<HlPayloadTypes>>) {
         let (tx, rx) = oneshot::channel();
         (
@@ -64,6 +66,7 @@ impl HlNode {
                 engine_handle_rx: Arc::new(Mutex::new(Some(rx))),
                 block_source_config,
                 debug_cutoff_height,
+                allow_network_overrides,
             },
             tx,
         )
@@ -95,6 +98,7 @@ impl HlNode {
                 engine_handle_rx: self.engine_handle_rx.clone(),
                 block_source_config: self.block_source_config.clone(),
                 debug_cutoff_height: self.debug_cutoff_height,
+                allow_network_overrides: self.allow_network_overrides,
             })
             .consensus(HlConsensusBuilder::default())
     }

src/node/network/block_import/service.rs

@@ -179,7 +179,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::{chainspec::hl::hl_mainnet, HlHeader};
+    use crate::{HlHeader, chainspec::hl::hl_mainnet};
 
     use super::*;
     use alloy_primitives::{B256, U128};

src/node/network/mod.rs

@@ -25,7 +25,10 @@ use reth_network::{NetworkConfig, NetworkHandle, NetworkManager};
 use reth_network_api::PeersInfo;
 use reth_provider::StageCheckpointReader;
 use reth_stages_types::StageId;
-use std::sync::Arc;
+use std::{
+    net::{Ipv4Addr, SocketAddr},
+    sync::Arc,
+};
 use tokio::sync::{Mutex, mpsc, oneshot};
 use tracing::info;
 
@@ -144,6 +147,8 @@ pub struct HlNetworkBuilder {
     pub(crate) block_source_config: BlockSourceConfig,
 
     pub(crate) debug_cutoff_height: Option<u64>,
+
+    pub(crate) allow_network_overrides: bool,
 }
 
 impl HlNetworkBuilder {
@@ -174,15 +179,24 @@ impl HlNetworkBuilder {
             ImportService::new(consensus, handle, from_network, to_network).await.unwrap();
         });
 
-        Ok(ctx.build_network_config(
+        let mut config_builder = ctx.network_config_builder()?;
-            ctx.network_config_builder()?
+
+        // Only apply localhost-only network settings if network overrides are NOT allowed
+        if !self.allow_network_overrides {
+            config_builder = config_builder
+                .discovery_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
+                .listener_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
                 .disable_dns_discovery()
-                .disable_nat()
+                .disable_nat();
-                .boot_nodes(boot_nodes())
+        }
-                .set_head(ctx.head())
+
-                .with_pow()
+        config_builder = config_builder
-                .block_import(Box::new(HlBlockImport::new(handle))),
+            .boot_nodes(boot_nodes())
-        ))
+            .set_head(ctx.head())
+            .with_pow()
+            .block_import(Box::new(HlBlockImport::new(handle)));
+
+        Ok(ctx.build_network_config(config_builder))
     }
 }
 

src/node/primitives/body.rs

@@ -3,8 +3,13 @@ use alloy_primitives::Address;
 use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize};
 use serde::{Deserialize, Serialize};
 
-use crate::node::types::{ReadPrecompileCall, ReadPrecompileCalls};
+use crate::{
-use crate::{HlHeader, node::primitives::TransactionSigned};
+    HlHeader,
+    node::{
+        primitives::TransactionSigned,
+        types::{ReadPrecompileCall, ReadPrecompileCalls},
+    },
+};
 
 /// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars
 /// for blob transactions.
@@ -33,13 +38,11 @@ pub type BlockBody = alloy_consensus::BlockBody<TransactionSigned, HlHeader>;
 
 impl InMemorySize for HlBlockBody {
     fn size(&self) -> usize {
-        self.inner.size()
+        self.inner.size() +
-            + self
+            self.sidecars
-                .sidecars
                 .as_ref()
-                .map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>())
+                .map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>()) +
-            + self
+            self.read_precompile_calls
-                .read_precompile_calls
                 .as_ref()
                 .map_or(0, |s| s.0.capacity() * core::mem::size_of::<ReadPrecompileCall>())
     }

src/node/primitives/header.rs

@@ -45,7 +45,11 @@ pub struct HlHeaderExtras {
 }
 
 impl HlHeader {
-    pub(crate) fn from_ethereum_header(header: Header, receipts: &[EthereumReceipt], system_tx_count: u64) -> HlHeader {
+    pub(crate) fn from_ethereum_header(
+        header: Header,
+        receipts: &[EthereumReceipt],
+        system_tx_count: u64,
+    ) -> HlHeader {
         let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs));
         HlHeader {
             inner: header,
@@ -183,8 +187,9 @@ impl reth_codecs::Compact for HlHeader {
         // because Compact trait requires the Bytes field to be placed at the end of the struct.
         // Bytes::from_compact just reads all trailing data as the Bytes field.
         //
-        // Hence we need to use other form of serialization, since extra headers are not Compact-compatible.
+        // Hence we need to use other form of serialization, since extra headers are not
-        // We just treat all header fields as rmp-serialized one `Bytes` field.
+        // Compact-compatible. We just treat all header fields as rmp-serialized one `Bytes`
+        // field.
         let result: Bytes = rmp_serde::to_vec(&self).unwrap().into();
         result.to_compact(buf)
     }

src/node/primitives/rlp.rs

@@ -1,6 +1,6 @@
 #![allow(clippy::owned_cow)]
 use super::{HlBlock, HlBlockBody, TransactionSigned};
-use crate::{node::types::ReadPrecompileCalls, HlHeader};
+use crate::{HlHeader, node::types::ReadPrecompileCalls};
 use alloy_consensus::{BlobTransactionSidecar, BlockBody};
 use alloy_eips::eip4895::Withdrawals;
 use alloy_primitives::Address;

src/node/primitives/serde_bincode_compat.rs

@@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 
 use super::{HlBlock, HlBlockBody};
-use crate::{node::{primitives::BlockBody, types::ReadPrecompileCalls}, HlHeader};
+use crate::{
+    HlHeader,
+    node::{primitives::BlockBody, types::ReadPrecompileCalls},
+};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct HlBlockBodyBincode<'a> {

src/node/spot_meta/init.rs

@@ -3,9 +3,15 @@ use crate::node::{
     storage::tables::{self, SPOT_METADATA_KEY},
     types::reth_compat,
 };
-use alloy_primitives::{Address, Bytes};
+use alloy_primitives::Address;
-use reth_db::{DatabaseEnv, cursor::{DbCursorRO, DbCursorRW}};
+use reth_db::{
-use reth_db_api::{Database, transaction::{DbTx, DbTxMut}};
+    DatabaseEnv,
+    cursor::DbCursorRO,
+};
+use reth_db_api::{
+    Database,
+    transaction::DbTx,
+};
 use std::{collections::BTreeMap, sync::Arc};
 use tracing::info;
 
@@ -18,11 +24,17 @@ pub fn load_spot_metadata_cache(db: &Arc<DatabaseEnv>, chain_id: u64) {
     }) {
         Ok(Ok(data)) => data,
         Ok(Err(e)) => {
-            info!("Failed to read spot metadata from database: {}. Will fetch on-demand from API.", e);
+            info!(
+                "Failed to read spot metadata from database: {}. Will fetch on-demand from API.",
+                e
+            );
             return;
         }
         Err(e) => {
-            info!("Database view error while loading spot metadata: {}. Will fetch on-demand from API.", e);
+            info!(
+                "Database view error while loading spot metadata: {}. Will fetch on-demand from API.",
+                e
+            );
             return;
         }
     };
@@ -46,10 +58,8 @@ pub fn load_spot_metadata_cache(db: &Arc<DatabaseEnv>, chain_id: u64) {
     };
 
     // Convert and initialize cache
-    let metadata: BTreeMap<Address, SpotId> = serializable_map
+    let metadata: BTreeMap<Address, SpotId> =
-        .into_iter()
+        serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect();
-        .map(|(addr, index)| (addr, SpotId { index }))
-        .collect();
 
     info!("Loaded spot metadata from database ({} entries)", metadata.len());
     reth_compat::initialize_spot_metadata_cache(metadata);
@@ -85,21 +95,8 @@ pub fn init_spot_metadata(
         }
     };
 
-    // Serialize and store
+    // Store to database
-    let serializable_map: BTreeMap<Address, u64> =
+    reth_compat::store_spot_metadata(&db, &metadata)?;
-        metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect();
-
-    db.update(|tx| -> Result<(), reth_db::DatabaseError> {
-        let mut cursor = tx.cursor_write::<tables::SpotMetadata>()?;
-        cursor.upsert(
-            SPOT_METADATA_KEY,
-            &Bytes::from(
-                rmp_serde::to_vec(&serializable_map)
-                    .expect("Failed to serialize spot metadata"),
-            ),
-        )?;
-        Ok(())
-    })??;
 
     info!("Successfully fetched and stored spot metadata for chain {}", chain_id);
     Ok(())

src/node/types/reth_compat.rs

@@ -2,7 +2,7 @@
 use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata};
 use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy};
 use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256};
-use reth_db::cursor::DbCursorRW;
+use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW};
 use reth_db_api::{Database, transaction::DbTxMut};
 use reth_primitives::TransactionSigned as RethTxSigned;
 use serde::{Deserialize, Serialize};
@@ -84,45 +84,49 @@ pub struct SealedBlock {
     pub body: BlockBody,
 }
 
-static EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
+static SPOT_EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
     LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new())));
 
 // Optional database handle for persisting on-demand fetches
-static DB_HANDLE: LazyLock<Mutex<Option<Arc<reth_db::DatabaseEnv>>>> =
+static DB_HANDLE: LazyLock<Mutex<Option<Arc<DatabaseEnv>>>> = LazyLock::new(|| Mutex::new(None));
-    LazyLock::new(|| Mutex::new(None));
 
 /// Set the database handle for persisting spot metadata
-pub fn set_spot_metadata_db(db: Arc<reth_db::DatabaseEnv>) {
+pub fn set_spot_metadata_db(db: Arc<DatabaseEnv>) {
     *DB_HANDLE.lock().unwrap() = Some(db);
 }
 
 /// Initialize the spot metadata cache with data loaded from database.
 /// This should be called during node initialization.
 pub fn initialize_spot_metadata_cache(metadata: BTreeMap<Address, SpotId>) {
-    *EVM_MAP.write().unwrap() = metadata;
+    *SPOT_EVM_MAP.write().unwrap() = metadata;
+}
+
+/// Helper function to serialize and store spot metadata to database
+pub fn store_spot_metadata(
+    db: &Arc<DatabaseEnv>,
+    metadata: &BTreeMap<Address, SpotId>,
+) -> Result<(), DatabaseError> {
+    db.update(|tx| {
+        let mut cursor = tx.cursor_write::<SpotMetadata>()?;
+
+        // Serialize to BTreeMap<Address, u64>
+        let serializable_map: BTreeMap<Address, u64> =
+            metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect();
+
+        cursor.upsert(
+            SPOT_METADATA_KEY,
+            &Bytes::from(
+                rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"),
+            ),
+        )?;
+        Ok(())
+    })?
 }
 
 /// Persist spot metadata to database if handle is available
 fn persist_spot_metadata_to_db(metadata: &BTreeMap<Address, SpotId>) {
     if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() {
-        let result = db.update(|tx| -> Result<(), reth_db::DatabaseError> {
+        match store_spot_metadata(db, metadata) {
-            let mut cursor = tx.cursor_write::<SpotMetadata>()?;
-
-            // Serialize to BTreeMap<Address, u64>
-            let serializable_map: BTreeMap<Address, u64> =
-                metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect();
-
-            cursor.upsert(
-                SPOT_METADATA_KEY,
-                &Bytes::from(
-                    rmp_serde::to_vec(&serializable_map)
-                        .expect("Failed to serialize spot metadata"),
-                ),
-            )?;
-            Ok(())
-        });
-
-        match result {
             Ok(_) => info!("Persisted spot metadata to database"),
             Err(e) => info!("Failed to persist spot metadata to database: {}", e),
         }
@@ -140,14 +144,14 @@ fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSig
         U256::from(0x1)
     } else {
         loop {
-            if let Some(spot) = EVM_MAP.read().unwrap().get(&to) {
+            if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) {
                 break spot.to_s();
             }
 
             // Cache miss - fetch from API, update cache, and persist to database
             info!("Contract not found: {to:?} from spot mapping, fetching from API...");
             let metadata = erc20_contract_to_spot_token(chain_id).unwrap();
-            *EVM_MAP.write().unwrap() = metadata.clone();
+            *SPOT_EVM_MAP.write().unwrap() = metadata.clone();
             persist_spot_metadata_to_db(&metadata);
         }
     };
@@ -160,13 +164,12 @@ impl SealedBlock {
         &self,
         read_precompile_calls: ReadPrecompileCalls,
         highest_precompile_address: Option<Address>,
-        system_txs: Vec<super::SystemTx>,
+        mut system_txs: Vec<super::SystemTx>,
         receipts: Vec<LegacyReceipt>,
         chain_id: u64,
     ) -> HlBlock {
-        // NOTE: Filter out system transactions that may be rejected by the EVM (tracked by #97,
+        // NOTE: These types of transactions are tracked at #97.
-        // testnet only).
+        system_txs.retain(|tx| tx.receipt.is_some());
-        let system_txs: Vec<_> = system_txs.into_iter().filter(|tx| tx.gas_limit() != 0).collect();
 
         let mut merged_txs = vec![];
         merged_txs.extend(system_txs.iter().map(|tx| system_tx_to_reth_transaction(tx, chain_id)));

src/pseudo_peer/service.rs

@@ -82,8 +82,8 @@ impl BlockPoller {
             .ok_or(eyre::eyre!("Failed to find latest block number"))?;
 
         loop {
-            if let Some(debug_cutoff_height) = debug_cutoff_height
+            if let Some(debug_cutoff_height) = debug_cutoff_height &&
-                && next_block_number > debug_cutoff_height
+                next_block_number > debug_cutoff_height
             {
                 next_block_number = debug_cutoff_height;
             }