From 95c653cf146b08e16389524bb849cd76c7a02898 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:46:22 +0000 Subject: [PATCH 1/2] chore: fmt --- src/addons/mod.rs | 2 +- src/chainspec/mod.rs | 5 ++++- src/node/consensus/mod.rs | 6 +++++- src/node/evm/assembler.rs | 3 ++- src/node/migrate.rs | 4 ++-- src/node/network/block_import/service.rs | 2 +- src/node/primitives/body.rs | 19 +++++++++++-------- src/node/primitives/header.rs | 11 ++++++++--- src/node/primitives/rlp.rs | 2 +- src/node/primitives/serde_bincode_compat.rs | 5 ++++- src/pseudo_peer/service.rs | 4 ++-- 11 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/addons/mod.rs b/src/addons/mod.rs index 71e488f8d..6cfed64eb 100644 --- a/src/addons/mod.rs +++ b/src/addons/mod.rs @@ -1,5 +1,5 @@ pub mod call_forwarder; pub mod hl_node_compliance; -pub mod tx_forwarder; pub mod subscribe_fixup; +pub mod tx_forwarder; mod utils; diff --git a/src/chainspec/mod.rs b/src/chainspec/mod.rs index 19ae4bdc0..52b8a9324 100644 --- a/src/chainspec/mod.rs +++ b/src/chainspec/mod.rs @@ -1,7 +1,10 @@ pub mod hl; pub mod parser; -use crate::{hardforks::HlHardforks, node::primitives::{header::HlHeaderExtras, HlHeader}}; +use crate::{ + hardforks::HlHardforks, + node::primitives::{HlHeader, header::HlHeaderExtras}, +}; use alloy_eips::eip7840::BlobParams; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256, U256}; diff --git a/src/node/consensus/mod.rs b/src/node/consensus/mod.rs index aec2d76db..38bf6c8eb 100644 --- a/src/node/consensus/mod.rs +++ b/src/node/consensus/mod.rs @@ -1,4 +1,8 @@ -use crate::{hardforks::HlHardforks, node::{primitives::HlHeader, HlNode}, HlBlock, HlBlockBody, HlPrimitives}; +use crate::{ + HlBlock, HlBlockBody, HlPrimitives, + hardforks::HlHardforks, + node::{HlNode, primitives::HlHeader}, +}; use reth::{ api::{FullNodeTypes, NodeTypes}, beacon_consensus::EthBeaconConsensus, diff --git a/src/node/evm/assembler.rs b/src/node/evm/assembler.rs index 62aee3250..ecbaeb8d4 100644 --- a/src/node/evm/assembler.rs +++ b/src/node/evm/assembler.rs @@ -1,5 +1,6 @@ use crate::{ - node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, HlBlock, HlHeader + HlBlock, HlHeader, + node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, }; use reth_evm::{ block::BlockExecutionError, diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 55b289574..c8f79ac9d 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -270,8 +270,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { let mut first = true; for (block_range, _tx_ranges) in all_static_files { - let migration_needed = self.using_old_header(block_range.start())? - || self.using_old_header(block_range.end())?; + let migration_needed = self.using_old_header(block_range.start())? || + self.using_old_header(block_range.end())?; if !migration_needed { // Create a placeholder symlink self.create_placeholder(block_range)?; diff --git a/src/node/network/block_import/service.rs b/src/node/network/block_import/service.rs index 359d9ed72..33280ffbe 100644 --- a/src/node/network/block_import/service.rs +++ b/src/node/network/block_import/service.rs @@ -179,7 +179,7 @@ where #[cfg(test)] mod tests { - use crate::{chainspec::hl::hl_mainnet, HlHeader}; + use crate::{HlHeader, chainspec::hl::hl_mainnet}; use super::*; use alloy_primitives::{B256, U128}; diff --git a/src/node/primitives/body.rs b/src/node/primitives/body.rs index abe696282..adc6cb09f 100644 --- a/src/node/primitives/body.rs +++ b/src/node/primitives/body.rs @@ -3,8 +3,13 @@ use alloy_primitives::Address; use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize}; use serde::{Deserialize, Serialize}; -use crate::node::types::{ReadPrecompileCall, ReadPrecompileCalls}; -use crate::{HlHeader, node::primitives::TransactionSigned}; +use crate::{ + HlHeader, + node::{ + primitives::TransactionSigned, + types::{ReadPrecompileCall, ReadPrecompileCalls}, + }, +}; /// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars /// for blob transactions. @@ -33,13 +38,11 @@ pub type BlockBody = alloy_consensus::BlockBody; impl InMemorySize for HlBlockBody { fn size(&self) -> usize { - self.inner.size() - + self - .sidecars + self.inner.size() + + self.sidecars .as_ref() - .map_or(0, |s| s.capacity() * core::mem::size_of::()) - + self - .read_precompile_calls + .map_or(0, |s| s.capacity() * core::mem::size_of::()) + + self.read_precompile_calls .as_ref() .map_or(0, |s| s.0.capacity() * core::mem::size_of::()) } diff --git a/src/node/primitives/header.rs b/src/node/primitives/header.rs index 3734ba2e5..0b9d58516 100644 --- a/src/node/primitives/header.rs +++ b/src/node/primitives/header.rs @@ -45,7 +45,11 @@ pub struct HlHeaderExtras { } impl HlHeader { - pub(crate) fn from_ethereum_header(header: Header, receipts: &[EthereumReceipt], system_tx_count: u64) -> HlHeader { + pub(crate) fn from_ethereum_header( + header: Header, + receipts: &[EthereumReceipt], + system_tx_count: u64, + ) -> HlHeader { let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs)); HlHeader { inner: header, @@ -183,8 +187,9 @@ impl reth_codecs::Compact for HlHeader { // because Compact trait requires the Bytes field to be placed at the end of the struct. // Bytes::from_compact just reads all trailing data as the Bytes field. // - // Hence we need to use other form of serialization, since extra headers are not Compact-compatible. - // We just treat all header fields as rmp-serialized one `Bytes` field. + // Hence we need to use other form of serialization, since extra headers are not + // Compact-compatible. We just treat all header fields as rmp-serialized one `Bytes` + // field. let result: Bytes = rmp_serde::to_vec(&self).unwrap().into(); result.to_compact(buf) } diff --git a/src/node/primitives/rlp.rs b/src/node/primitives/rlp.rs index 43cdaf640..cd6124ebc 100644 --- a/src/node/primitives/rlp.rs +++ b/src/node/primitives/rlp.rs @@ -1,6 +1,6 @@ #![allow(clippy::owned_cow)] use super::{HlBlock, HlBlockBody, TransactionSigned}; -use crate::{node::types::ReadPrecompileCalls, HlHeader}; +use crate::{HlHeader, node::types::ReadPrecompileCalls}; use alloy_consensus::{BlobTransactionSidecar, BlockBody}; use alloy_eips::eip4895::Withdrawals; use alloy_primitives::Address; diff --git a/src/node/primitives/serde_bincode_compat.rs b/src/node/primitives/serde_bincode_compat.rs index 958ede8a9..fbab6ca7f 100644 --- a/src/node/primitives/serde_bincode_compat.rs +++ b/src/node/primitives/serde_bincode_compat.rs @@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use super::{HlBlock, HlBlockBody}; -use crate::{node::{primitives::BlockBody, types::ReadPrecompileCalls}, HlHeader}; +use crate::{ + HlHeader, + node::{primitives::BlockBody, types::ReadPrecompileCalls}, +}; #[derive(Debug, Serialize, Deserialize)] pub struct HlBlockBodyBincode<'a> { diff --git a/src/pseudo_peer/service.rs b/src/pseudo_peer/service.rs index ddca88f1b..82f3efbd3 100644 --- a/src/pseudo_peer/service.rs +++ b/src/pseudo_peer/service.rs @@ -82,8 +82,8 @@ impl BlockPoller { .ok_or(eyre::eyre!("Failed to find latest block number"))?; loop { - if let Some(debug_cutoff_height) = debug_cutoff_height - && next_block_number > debug_cutoff_height + if let Some(debug_cutoff_height) = debug_cutoff_height && + next_block_number > debug_cutoff_height { next_block_number = debug_cutoff_height; } From 98cc4ce30b35aae49db3f4b677d10c9ae0c21230 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:48:42 +0000 Subject: [PATCH 2/2] feat: cache spot metadata in database to reduce API calls Implements persistent caching of ERC20 contract address to spot token ID mappings in the database to minimize API requests and improve performance. Changes: - Add SpotMetadata database table for persistent storage - Implement load_spot_metadata_cache() to initialize cache on startup - Add init_spot_metadata() for init-state command to pre-populate cache - Extract store_spot_metadata() helper to DRY serialization logic - Enable on-demand API fetches with automatic database persistence - Integrate cache loading in main node startup flow The cache falls back to on-demand API fetches if database is empty, with automatic persistence of fetched data for future use. --- src/main.rs | 12 ++++ src/node/cli.rs | 9 ++- src/node/spot_meta/init.rs | 103 ++++++++++++++++++++++++++++++++++ src/node/spot_meta/mod.rs | 3 +- src/node/storage/tables.rs | 11 ++++ src/node/types/mod.rs | 3 + src/node/types/reth_compat.rs | 101 ++++++++++++++++++++++++--------- 7 files changed, 214 insertions(+), 28 deletions(-) create mode 100644 src/node/spot_meta/init.rs diff --git a/src/main.rs b/src/main.rs index 9c446f297..601549156 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,9 @@ use reth_hl::{ HlNode, cli::{Cli, HlNodeArgs}, rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt}, + spot_meta::init as spot_meta_init, storage::tables::Tables, + types::set_spot_metadata_db, }, }; use tracing::info; @@ -95,6 +97,16 @@ fn main() -> eyre::Result<()> { }) .apply(|mut builder| { builder.db_mut().create_tables_for::().expect("create tables"); + + let chain_id = builder.config().chain.inner.chain().id(); + let db = builder.db_mut().clone(); + + // Set database handle for on-demand persistence + set_spot_metadata_db(db.clone()); + + // Load spot metadata from database and initialize cache + spot_meta_init::load_spot_metadata_cache(&db, chain_id); + builder }) .launch() diff --git a/src/node/cli.rs b/src/node/cli.rs index 67995eb2f..b3da7a346 100644 --- a/src/node/cli.rs +++ b/src/node/cli.rs @@ -2,7 +2,7 @@ use crate::{ chainspec::{HlChainSpec, parser::HlChainSpecParser}, node::{ HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator, - storage::tables::Tables, + spot_meta::init as spot_meta_init, storage::tables::Tables, }, pseudo_peer::BlockSourceArgs, }; @@ -201,7 +201,12 @@ where let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain()); let db_path = data_dir.db(); init_db(db_path.clone(), env.db.database_args())?; - init_db_for::<_, Tables>(db_path, env.db.database_args())?; + init_db_for::<_, Tables>(db_path.clone(), env.db.database_args())?; + + // Initialize spot metadata in database + let chain_id = env.chain.chain().id(); + spot_meta_init::init_spot_metadata(db_path, env.db.database_args(), chain_id)?; + Ok(()) } diff --git a/src/node/spot_meta/init.rs b/src/node/spot_meta/init.rs new file mode 100644 index 000000000..195ec058b --- /dev/null +++ b/src/node/spot_meta/init.rs @@ -0,0 +1,103 @@ +use crate::node::{ + spot_meta::{SpotId, erc20_contract_to_spot_token}, + storage::tables::{self, SPOT_METADATA_KEY}, + types::reth_compat, +}; +use alloy_primitives::Address; +use reth_db::{ + DatabaseEnv, + cursor::DbCursorRO, +}; +use reth_db_api::{ + Database, + transaction::DbTx, +}; +use std::{collections::BTreeMap, sync::Arc}; +use tracing::info; + +/// Load spot metadata from database and initialize cache +pub fn load_spot_metadata_cache(db: &Arc, chain_id: u64) { + // Try to read from database + let data = match db.view(|tx| -> Result>, reth_db::DatabaseError> { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.map(|(_, data)| data.to_vec())) + }) { + Ok(Ok(data)) => data, + Ok(Err(e)) => { + info!( + "Failed to read spot metadata from database: {}. Will fetch on-demand from API.", + e + ); + return; + } + Err(e) => { + info!( + "Database view error while loading spot metadata: {}. Will fetch on-demand from API.", + e + ); + return; + } + }; + + // Check if data exists + let Some(data) = data else { + info!( + "No spot metadata found in database for chain {}. Run 'init-state' to populate, or it will be fetched on-demand from API.", + chain_id + ); + return; + }; + + // Deserialize metadata + let serializable_map = match rmp_serde::from_slice::>(&data) { + Ok(map) => map, + Err(e) => { + info!("Failed to deserialize spot metadata: {}. Will fetch on-demand from API.", e); + return; + } + }; + + // Convert and initialize cache + let metadata: BTreeMap = + serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect(); + + info!("Loaded spot metadata from database ({} entries)", metadata.len()); + reth_compat::initialize_spot_metadata_cache(metadata); +} + +/// Initialize spot metadata in database from API +pub fn init_spot_metadata( + db_path: impl AsRef, + db_args: reth_db::mdbx::DatabaseArguments, + chain_id: u64, +) -> eyre::Result<()> { + info!("Initializing spot metadata for chain {}", chain_id); + + let db = Arc::new(reth_db::open_db(db_path.as_ref(), db_args)?); + + // Check if spot metadata already exists + let exists = db.view(|tx| -> Result { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.is_some()) + })??; + + if exists { + info!("Spot metadata already exists in database"); + return Ok(()); + } + + // Fetch from API + let metadata = match erc20_contract_to_spot_token(chain_id) { + Ok(m) => m, + Err(e) => { + info!("Failed to fetch spot metadata from API: {}. Will be fetched on-demand.", e); + return Ok(()); + } + }; + + // Store to database + reth_compat::store_spot_metadata(&db, &metadata)?; + + info!("Successfully fetched and stored spot metadata for chain {}", chain_id); + Ok(()) +} diff --git a/src/node/spot_meta/mod.rs b/src/node/spot_meta/mod.rs index cebfee732..f0d2b67d3 100644 --- a/src/node/spot_meta/mod.rs +++ b/src/node/spot_meta/mod.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID}; +pub mod init; mod patch; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -25,7 +26,7 @@ pub struct SpotMeta { } #[derive(Debug, Clone)] -pub(crate) struct SpotId { +pub struct SpotId { pub index: u64, } diff --git a/src/node/storage/tables.rs b/src/node/storage/tables.rs index 8efa02c2e..2369bf7ad 100644 --- a/src/node/storage/tables.rs +++ b/src/node/storage/tables.rs @@ -2,10 +2,21 @@ use alloy_primitives::{BlockNumber, Bytes}; use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables}; use std::fmt; +/// Static key used for spot metadata, as the database is unique to each chain. +/// This may later serve as a versioning key to assist with future database migrations. +pub const SPOT_METADATA_KEY: u64 = 0; + tables! { /// Read precompile calls for each block. table BlockReadPrecompileCalls { type Key = BlockNumber; type Value = Bytes; } + + /// Spot metadata mapping (EVM address to spot token index). + /// Uses a constant key since the database is chain-specific. + table SpotMetadata { + type Key = u64; + type Value = Bytes; + } } diff --git a/src/node/types/mod.rs b/src/node/types/mod.rs index b25eac527..478f76d23 100644 --- a/src/node/types/mod.rs +++ b/src/node/types/mod.rs @@ -19,6 +19,9 @@ pub struct ReadPrecompileCalls(pub Vec); pub(crate) mod reth_compat; +// Re-export spot metadata functions +pub use reth_compat::{initialize_spot_metadata_cache, set_spot_metadata_db}; + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct HlExtras { pub read_precompile_calls: Option, diff --git a/src/node/types/reth_compat.rs b/src/node/types/reth_compat.rs index 016ef9371..aab96c7e1 100644 --- a/src/node/types/reth_compat.rs +++ b/src/node/types/reth_compat.rs @@ -1,11 +1,14 @@ //! Copy of reth codebase to preserve serialization compatibility +use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata}; use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; -use alloy_primitives::{Address, BlockHash, Signature, TxKind, U256}; +use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256}; +use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW}; +use reth_db_api::{Database, transaction::DbTxMut}; use reth_primitives::TransactionSigned as RethTxSigned; use serde::{Deserialize, Serialize}; use std::{ collections::BTreeMap, - sync::{Arc, LazyLock, RwLock}, + sync::{Arc, LazyLock, Mutex, RwLock}, }; use tracing::info; @@ -81,33 +84,81 @@ pub struct SealedBlock { pub body: BlockBody, } -fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { - static EVM_MAP: LazyLock>>> = - LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - { - let Transaction::Legacy(tx) = &transaction.tx else { - panic!("Unexpected transaction type"); - }; - let TxKind::Call(to) = tx.to else { - panic!("Unexpected contract creation"); - }; - let s = if tx.input.is_empty() { - U256::from(0x1) - } else { - loop { - if let Some(spot) = EVM_MAP.read().unwrap().get(&to) { - break spot.to_s(); - } +static SPOT_EVM_MAP: LazyLock>>> = + LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - info!("Contract not found: {to:?} from spot mapping, fetching again..."); - *EVM_MAP.write().unwrap() = erc20_contract_to_spot_token(chain_id).unwrap(); - } - }; - let signature = Signature::new(U256::from(0x1), s, true); - TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +// Optional database handle for persisting on-demand fetches +static DB_HANDLE: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); + +/// Set the database handle for persisting spot metadata +pub fn set_spot_metadata_db(db: Arc) { + *DB_HANDLE.lock().unwrap() = Some(db); +} + +/// Initialize the spot metadata cache with data loaded from database. +/// This should be called during node initialization. +pub fn initialize_spot_metadata_cache(metadata: BTreeMap) { + *SPOT_EVM_MAP.write().unwrap() = metadata; +} + +/// Helper function to serialize and store spot metadata to database +pub fn store_spot_metadata( + db: &Arc, + metadata: &BTreeMap, +) -> Result<(), DatabaseError> { + db.update(|tx| { + let mut cursor = tx.cursor_write::()?; + + // Serialize to BTreeMap + let serializable_map: BTreeMap = + metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect(); + + cursor.upsert( + SPOT_METADATA_KEY, + &Bytes::from( + rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"), + ), + )?; + Ok(()) + })? +} + +/// Persist spot metadata to database if handle is available +fn persist_spot_metadata_to_db(metadata: &BTreeMap) { + if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() { + match store_spot_metadata(db, metadata) { + Ok(_) => info!("Persisted spot metadata to database"), + Err(e) => info!("Failed to persist spot metadata to database: {}", e), + } } } +fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { + let Transaction::Legacy(tx) = &transaction.tx else { + panic!("Unexpected transaction type"); + }; + let TxKind::Call(to) = tx.to else { + panic!("Unexpected contract creation"); + }; + let s = if tx.input.is_empty() { + U256::from(0x1) + } else { + loop { + if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) { + break spot.to_s(); + } + + // Cache miss - fetch from API, update cache, and persist to database + info!("Contract not found: {to:?} from spot mapping, fetching from API..."); + let metadata = erc20_contract_to_spot_token(chain_id).unwrap(); + *SPOT_EVM_MAP.write().unwrap() = metadata.clone(); + persist_spot_metadata_to_db(&metadata); + } + }; + let signature = Signature::new(U256::from(0x1), s, true); + TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +} + impl SealedBlock { pub fn to_reth_block( &self,