diff --git a/src/addons/mod.rs b/src/addons/mod.rs index 71e488f8d..6cfed64eb 100644 --- a/src/addons/mod.rs +++ b/src/addons/mod.rs @@ -1,5 +1,5 @@ pub mod call_forwarder; pub mod hl_node_compliance; -pub mod tx_forwarder; pub mod subscribe_fixup; +pub mod tx_forwarder; mod utils; diff --git a/src/chainspec/mod.rs b/src/chainspec/mod.rs index 19ae4bdc0..52b8a9324 100644 --- a/src/chainspec/mod.rs +++ b/src/chainspec/mod.rs @@ -1,7 +1,10 @@ pub mod hl; pub mod parser; -use crate::{hardforks::HlHardforks, node::primitives::{header::HlHeaderExtras, HlHeader}}; +use crate::{ + hardforks::HlHardforks, + node::primitives::{HlHeader, header::HlHeaderExtras}, +}; use alloy_eips::eip7840::BlobParams; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256, U256}; diff --git a/src/main.rs b/src/main.rs index 9c446f297..601549156 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,9 @@ use reth_hl::{ HlNode, cli::{Cli, HlNodeArgs}, rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt}, + spot_meta::init as spot_meta_init, storage::tables::Tables, + types::set_spot_metadata_db, }, }; use tracing::info; @@ -95,6 +97,16 @@ fn main() -> eyre::Result<()> { }) .apply(|mut builder| { builder.db_mut().create_tables_for::().expect("create tables"); + + let chain_id = builder.config().chain.inner.chain().id(); + let db = builder.db_mut().clone(); + + // Set database handle for on-demand persistence + set_spot_metadata_db(db.clone()); + + // Load spot metadata from database and initialize cache + spot_meta_init::load_spot_metadata_cache(&db, chain_id); + builder }) .launch() diff --git a/src/node/cli.rs b/src/node/cli.rs index 67995eb2f..b3da7a346 100644 --- a/src/node/cli.rs +++ b/src/node/cli.rs @@ -2,7 +2,7 @@ use crate::{ chainspec::{HlChainSpec, parser::HlChainSpecParser}, node::{ HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator, - storage::tables::Tables, + spot_meta::init as spot_meta_init, storage::tables::Tables, }, pseudo_peer::BlockSourceArgs, }; @@ -201,7 +201,12 @@ where let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain()); let db_path = data_dir.db(); init_db(db_path.clone(), env.db.database_args())?; - init_db_for::<_, Tables>(db_path, env.db.database_args())?; + init_db_for::<_, Tables>(db_path.clone(), env.db.database_args())?; + + // Initialize spot metadata in database + let chain_id = env.chain.chain().id(); + spot_meta_init::init_spot_metadata(db_path, env.db.database_args(), chain_id)?; + Ok(()) } diff --git a/src/node/consensus/mod.rs b/src/node/consensus/mod.rs index aec2d76db..38bf6c8eb 100644 --- a/src/node/consensus/mod.rs +++ b/src/node/consensus/mod.rs @@ -1,4 +1,8 @@ -use crate::{hardforks::HlHardforks, node::{primitives::HlHeader, HlNode}, HlBlock, HlBlockBody, HlPrimitives}; +use crate::{ + HlBlock, HlBlockBody, HlPrimitives, + hardforks::HlHardforks, + node::{HlNode, primitives::HlHeader}, +}; use reth::{ api::{FullNodeTypes, NodeTypes}, beacon_consensus::EthBeaconConsensus, diff --git a/src/node/evm/assembler.rs b/src/node/evm/assembler.rs index 62aee3250..ecbaeb8d4 100644 --- a/src/node/evm/assembler.rs +++ b/src/node/evm/assembler.rs @@ -1,5 +1,6 @@ use crate::{ - node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, HlBlock, HlHeader + HlBlock, HlHeader, + node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, }; use reth_evm::{ block::BlockExecutionError, diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 55b289574..c8f79ac9d 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -270,8 +270,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { let mut first = true; for (block_range, _tx_ranges) in all_static_files { - let migration_needed = self.using_old_header(block_range.start())? - || self.using_old_header(block_range.end())?; + let migration_needed = self.using_old_header(block_range.start())? || + self.using_old_header(block_range.end())?; if !migration_needed { // Create a placeholder symlink self.create_placeholder(block_range)?; diff --git a/src/node/network/block_import/service.rs b/src/node/network/block_import/service.rs index 359d9ed72..33280ffbe 100644 --- a/src/node/network/block_import/service.rs +++ b/src/node/network/block_import/service.rs @@ -179,7 +179,7 @@ where #[cfg(test)] mod tests { - use crate::{chainspec::hl::hl_mainnet, HlHeader}; + use crate::{HlHeader, chainspec::hl::hl_mainnet}; use super::*; use alloy_primitives::{B256, U128}; diff --git a/src/node/primitives/body.rs b/src/node/primitives/body.rs index abe696282..adc6cb09f 100644 --- a/src/node/primitives/body.rs +++ b/src/node/primitives/body.rs @@ -3,8 +3,13 @@ use alloy_primitives::Address; use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize}; use serde::{Deserialize, Serialize}; -use crate::node::types::{ReadPrecompileCall, ReadPrecompileCalls}; -use crate::{HlHeader, node::primitives::TransactionSigned}; +use crate::{ + HlHeader, + node::{ + primitives::TransactionSigned, + types::{ReadPrecompileCall, ReadPrecompileCalls}, + }, +}; /// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars /// for blob transactions. @@ -33,13 +38,11 @@ pub type BlockBody = alloy_consensus::BlockBody; impl InMemorySize for HlBlockBody { fn size(&self) -> usize { - self.inner.size() - + self - .sidecars + self.inner.size() + + self.sidecars .as_ref() - .map_or(0, |s| s.capacity() * core::mem::size_of::()) - + self - .read_precompile_calls + .map_or(0, |s| s.capacity() * core::mem::size_of::()) + + self.read_precompile_calls .as_ref() .map_or(0, |s| s.0.capacity() * core::mem::size_of::()) } diff --git a/src/node/primitives/header.rs b/src/node/primitives/header.rs index 3734ba2e5..0b9d58516 100644 --- a/src/node/primitives/header.rs +++ b/src/node/primitives/header.rs @@ -45,7 +45,11 @@ pub struct HlHeaderExtras { } impl HlHeader { - pub(crate) fn from_ethereum_header(header: Header, receipts: &[EthereumReceipt], system_tx_count: u64) -> HlHeader { + pub(crate) fn from_ethereum_header( + header: Header, + receipts: &[EthereumReceipt], + system_tx_count: u64, + ) -> HlHeader { let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs)); HlHeader { inner: header, @@ -183,8 +187,9 @@ impl reth_codecs::Compact for HlHeader { // because Compact trait requires the Bytes field to be placed at the end of the struct. // Bytes::from_compact just reads all trailing data as the Bytes field. // - // Hence we need to use other form of serialization, since extra headers are not Compact-compatible. - // We just treat all header fields as rmp-serialized one `Bytes` field. + // Hence we need to use other form of serialization, since extra headers are not + // Compact-compatible. We just treat all header fields as rmp-serialized one `Bytes` + // field. let result: Bytes = rmp_serde::to_vec(&self).unwrap().into(); result.to_compact(buf) } diff --git a/src/node/primitives/rlp.rs b/src/node/primitives/rlp.rs index 43cdaf640..cd6124ebc 100644 --- a/src/node/primitives/rlp.rs +++ b/src/node/primitives/rlp.rs @@ -1,6 +1,6 @@ #![allow(clippy::owned_cow)] use super::{HlBlock, HlBlockBody, TransactionSigned}; -use crate::{node::types::ReadPrecompileCalls, HlHeader}; +use crate::{HlHeader, node::types::ReadPrecompileCalls}; use alloy_consensus::{BlobTransactionSidecar, BlockBody}; use alloy_eips::eip4895::Withdrawals; use alloy_primitives::Address; diff --git a/src/node/primitives/serde_bincode_compat.rs b/src/node/primitives/serde_bincode_compat.rs index 958ede8a9..fbab6ca7f 100644 --- a/src/node/primitives/serde_bincode_compat.rs +++ b/src/node/primitives/serde_bincode_compat.rs @@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use super::{HlBlock, HlBlockBody}; -use crate::{node::{primitives::BlockBody, types::ReadPrecompileCalls}, HlHeader}; +use crate::{ + HlHeader, + node::{primitives::BlockBody, types::ReadPrecompileCalls}, +}; #[derive(Debug, Serialize, Deserialize)] pub struct HlBlockBodyBincode<'a> { diff --git a/src/node/spot_meta/init.rs b/src/node/spot_meta/init.rs new file mode 100644 index 000000000..195ec058b --- /dev/null +++ b/src/node/spot_meta/init.rs @@ -0,0 +1,103 @@ +use crate::node::{ + spot_meta::{SpotId, erc20_contract_to_spot_token}, + storage::tables::{self, SPOT_METADATA_KEY}, + types::reth_compat, +}; +use alloy_primitives::Address; +use reth_db::{ + DatabaseEnv, + cursor::DbCursorRO, +}; +use reth_db_api::{ + Database, + transaction::DbTx, +}; +use std::{collections::BTreeMap, sync::Arc}; +use tracing::info; + +/// Load spot metadata from database and initialize cache +pub fn load_spot_metadata_cache(db: &Arc, chain_id: u64) { + // Try to read from database + let data = match db.view(|tx| -> Result>, reth_db::DatabaseError> { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.map(|(_, data)| data.to_vec())) + }) { + Ok(Ok(data)) => data, + Ok(Err(e)) => { + info!( + "Failed to read spot metadata from database: {}. Will fetch on-demand from API.", + e + ); + return; + } + Err(e) => { + info!( + "Database view error while loading spot metadata: {}. Will fetch on-demand from API.", + e + ); + return; + } + }; + + // Check if data exists + let Some(data) = data else { + info!( + "No spot metadata found in database for chain {}. Run 'init-state' to populate, or it will be fetched on-demand from API.", + chain_id + ); + return; + }; + + // Deserialize metadata + let serializable_map = match rmp_serde::from_slice::>(&data) { + Ok(map) => map, + Err(e) => { + info!("Failed to deserialize spot metadata: {}. Will fetch on-demand from API.", e); + return; + } + }; + + // Convert and initialize cache + let metadata: BTreeMap = + serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect(); + + info!("Loaded spot metadata from database ({} entries)", metadata.len()); + reth_compat::initialize_spot_metadata_cache(metadata); +} + +/// Initialize spot metadata in database from API +pub fn init_spot_metadata( + db_path: impl AsRef, + db_args: reth_db::mdbx::DatabaseArguments, + chain_id: u64, +) -> eyre::Result<()> { + info!("Initializing spot metadata for chain {}", chain_id); + + let db = Arc::new(reth_db::open_db(db_path.as_ref(), db_args)?); + + // Check if spot metadata already exists + let exists = db.view(|tx| -> Result { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.is_some()) + })??; + + if exists { + info!("Spot metadata already exists in database"); + return Ok(()); + } + + // Fetch from API + let metadata = match erc20_contract_to_spot_token(chain_id) { + Ok(m) => m, + Err(e) => { + info!("Failed to fetch spot metadata from API: {}. Will be fetched on-demand.", e); + return Ok(()); + } + }; + + // Store to database + reth_compat::store_spot_metadata(&db, &metadata)?; + + info!("Successfully fetched and stored spot metadata for chain {}", chain_id); + Ok(()) +} diff --git a/src/node/spot_meta/mod.rs b/src/node/spot_meta/mod.rs index cebfee732..f0d2b67d3 100644 --- a/src/node/spot_meta/mod.rs +++ b/src/node/spot_meta/mod.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID}; +pub mod init; mod patch; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -25,7 +26,7 @@ pub struct SpotMeta { } #[derive(Debug, Clone)] -pub(crate) struct SpotId { +pub struct SpotId { pub index: u64, } diff --git a/src/node/storage/tables.rs b/src/node/storage/tables.rs index 8efa02c2e..2369bf7ad 100644 --- a/src/node/storage/tables.rs +++ b/src/node/storage/tables.rs @@ -2,10 +2,21 @@ use alloy_primitives::{BlockNumber, Bytes}; use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables}; use std::fmt; +/// Static key used for spot metadata, as the database is unique to each chain. +/// This may later serve as a versioning key to assist with future database migrations. +pub const SPOT_METADATA_KEY: u64 = 0; + tables! { /// Read precompile calls for each block. table BlockReadPrecompileCalls { type Key = BlockNumber; type Value = Bytes; } + + /// Spot metadata mapping (EVM address to spot token index). + /// Uses a constant key since the database is chain-specific. + table SpotMetadata { + type Key = u64; + type Value = Bytes; + } } diff --git a/src/node/types/mod.rs b/src/node/types/mod.rs index b25eac527..478f76d23 100644 --- a/src/node/types/mod.rs +++ b/src/node/types/mod.rs @@ -19,6 +19,9 @@ pub struct ReadPrecompileCalls(pub Vec); pub(crate) mod reth_compat; +// Re-export spot metadata functions +pub use reth_compat::{initialize_spot_metadata_cache, set_spot_metadata_db}; + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct HlExtras { pub read_precompile_calls: Option, diff --git a/src/node/types/reth_compat.rs b/src/node/types/reth_compat.rs index 016ef9371..aab96c7e1 100644 --- a/src/node/types/reth_compat.rs +++ b/src/node/types/reth_compat.rs @@ -1,11 +1,14 @@ //! Copy of reth codebase to preserve serialization compatibility +use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata}; use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; -use alloy_primitives::{Address, BlockHash, Signature, TxKind, U256}; +use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256}; +use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW}; +use reth_db_api::{Database, transaction::DbTxMut}; use reth_primitives::TransactionSigned as RethTxSigned; use serde::{Deserialize, Serialize}; use std::{ collections::BTreeMap, - sync::{Arc, LazyLock, RwLock}, + sync::{Arc, LazyLock, Mutex, RwLock}, }; use tracing::info; @@ -81,33 +84,81 @@ pub struct SealedBlock { pub body: BlockBody, } -fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { - static EVM_MAP: LazyLock>>> = - LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - { - let Transaction::Legacy(tx) = &transaction.tx else { - panic!("Unexpected transaction type"); - }; - let TxKind::Call(to) = tx.to else { - panic!("Unexpected contract creation"); - }; - let s = if tx.input.is_empty() { - U256::from(0x1) - } else { - loop { - if let Some(spot) = EVM_MAP.read().unwrap().get(&to) { - break spot.to_s(); - } +static SPOT_EVM_MAP: LazyLock>>> = + LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - info!("Contract not found: {to:?} from spot mapping, fetching again..."); - *EVM_MAP.write().unwrap() = erc20_contract_to_spot_token(chain_id).unwrap(); - } - }; - let signature = Signature::new(U256::from(0x1), s, true); - TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +// Optional database handle for persisting on-demand fetches +static DB_HANDLE: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); + +/// Set the database handle for persisting spot metadata +pub fn set_spot_metadata_db(db: Arc) { + *DB_HANDLE.lock().unwrap() = Some(db); +} + +/// Initialize the spot metadata cache with data loaded from database. +/// This should be called during node initialization. +pub fn initialize_spot_metadata_cache(metadata: BTreeMap) { + *SPOT_EVM_MAP.write().unwrap() = metadata; +} + +/// Helper function to serialize and store spot metadata to database +pub fn store_spot_metadata( + db: &Arc, + metadata: &BTreeMap, +) -> Result<(), DatabaseError> { + db.update(|tx| { + let mut cursor = tx.cursor_write::()?; + + // Serialize to BTreeMap + let serializable_map: BTreeMap = + metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect(); + + cursor.upsert( + SPOT_METADATA_KEY, + &Bytes::from( + rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"), + ), + )?; + Ok(()) + })? +} + +/// Persist spot metadata to database if handle is available +fn persist_spot_metadata_to_db(metadata: &BTreeMap) { + if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() { + match store_spot_metadata(db, metadata) { + Ok(_) => info!("Persisted spot metadata to database"), + Err(e) => info!("Failed to persist spot metadata to database: {}", e), + } } } +fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { + let Transaction::Legacy(tx) = &transaction.tx else { + panic!("Unexpected transaction type"); + }; + let TxKind::Call(to) = tx.to else { + panic!("Unexpected contract creation"); + }; + let s = if tx.input.is_empty() { + U256::from(0x1) + } else { + loop { + if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) { + break spot.to_s(); + } + + // Cache miss - fetch from API, update cache, and persist to database + info!("Contract not found: {to:?} from spot mapping, fetching from API..."); + let metadata = erc20_contract_to_spot_token(chain_id).unwrap(); + *SPOT_EVM_MAP.write().unwrap() = metadata.clone(); + persist_spot_metadata_to_db(&metadata); + } + }; + let signature = Signature::new(U256::from(0x1), s, true); + TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +} + impl SealedBlock { pub fn to_reth_block( &self, diff --git a/src/pseudo_peer/service.rs b/src/pseudo_peer/service.rs index ddca88f1b..82f3efbd3 100644 --- a/src/pseudo_peer/service.rs +++ b/src/pseudo_peer/service.rs @@ -82,8 +82,8 @@ impl BlockPoller { .ok_or(eyre::eyre!("Failed to find latest block number"))?; loop { - if let Some(debug_cutoff_height) = debug_cutoff_height - && next_block_number > debug_cutoff_height + if let Some(debug_cutoff_height) = debug_cutoff_height && + next_block_number > debug_cutoff_height { next_block_number = debug_cutoff_height; }