From 98cc4ce30b35aae49db3f4b677d10c9ae0c21230 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:48:42 +0000 Subject: [PATCH] feat: cache spot metadata in database to reduce API calls Implements persistent caching of ERC20 contract address to spot token ID mappings in the database to minimize API requests and improve performance. Changes: - Add SpotMetadata database table for persistent storage - Implement load_spot_metadata_cache() to initialize cache on startup - Add init_spot_metadata() for init-state command to pre-populate cache - Extract store_spot_metadata() helper to DRY serialization logic - Enable on-demand API fetches with automatic database persistence - Integrate cache loading in main node startup flow The cache falls back to on-demand API fetches if database is empty, with automatic persistence of fetched data for future use. --- src/main.rs | 12 ++++ src/node/cli.rs | 9 ++- src/node/spot_meta/init.rs | 103 ++++++++++++++++++++++++++++++++++ src/node/spot_meta/mod.rs | 3 +- src/node/storage/tables.rs | 11 ++++ src/node/types/mod.rs | 3 + src/node/types/reth_compat.rs | 101 ++++++++++++++++++++++++--------- 7 files changed, 214 insertions(+), 28 deletions(-) create mode 100644 src/node/spot_meta/init.rs diff --git a/src/main.rs b/src/main.rs index 9c446f297..601549156 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,9 @@ use reth_hl::{ HlNode, cli::{Cli, HlNodeArgs}, rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt}, + spot_meta::init as spot_meta_init, storage::tables::Tables, + types::set_spot_metadata_db, }, }; use tracing::info; @@ -95,6 +97,16 @@ fn main() -> eyre::Result<()> { }) .apply(|mut builder| { builder.db_mut().create_tables_for::().expect("create tables"); + + let chain_id = builder.config().chain.inner.chain().id(); + let db = builder.db_mut().clone(); + + // Set database handle for on-demand persistence + set_spot_metadata_db(db.clone()); + + // Load spot metadata from database and initialize cache + spot_meta_init::load_spot_metadata_cache(&db, chain_id); + builder }) .launch() diff --git a/src/node/cli.rs b/src/node/cli.rs index 67995eb2f..b3da7a346 100644 --- a/src/node/cli.rs +++ b/src/node/cli.rs @@ -2,7 +2,7 @@ use crate::{ chainspec::{HlChainSpec, parser::HlChainSpecParser}, node::{ HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator, - storage::tables::Tables, + spot_meta::init as spot_meta_init, storage::tables::Tables, }, pseudo_peer::BlockSourceArgs, }; @@ -201,7 +201,12 @@ where let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain()); let db_path = data_dir.db(); init_db(db_path.clone(), env.db.database_args())?; - init_db_for::<_, Tables>(db_path, env.db.database_args())?; + init_db_for::<_, Tables>(db_path.clone(), env.db.database_args())?; + + // Initialize spot metadata in database + let chain_id = env.chain.chain().id(); + spot_meta_init::init_spot_metadata(db_path, env.db.database_args(), chain_id)?; + Ok(()) } diff --git a/src/node/spot_meta/init.rs b/src/node/spot_meta/init.rs new file mode 100644 index 000000000..195ec058b --- /dev/null +++ b/src/node/spot_meta/init.rs @@ -0,0 +1,103 @@ +use crate::node::{ + spot_meta::{SpotId, erc20_contract_to_spot_token}, + storage::tables::{self, SPOT_METADATA_KEY}, + types::reth_compat, +}; +use alloy_primitives::Address; +use reth_db::{ + DatabaseEnv, + cursor::DbCursorRO, +}; +use reth_db_api::{ + Database, + transaction::DbTx, +}; +use std::{collections::BTreeMap, sync::Arc}; +use tracing::info; + +/// Load spot metadata from database and initialize cache +pub fn load_spot_metadata_cache(db: &Arc, chain_id: u64) { + // Try to read from database + let data = match db.view(|tx| -> Result>, reth_db::DatabaseError> { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.map(|(_, data)| data.to_vec())) + }) { + Ok(Ok(data)) => data, + Ok(Err(e)) => { + info!( + "Failed to read spot metadata from database: {}. Will fetch on-demand from API.", + e + ); + return; + } + Err(e) => { + info!( + "Database view error while loading spot metadata: {}. Will fetch on-demand from API.", + e + ); + return; + } + }; + + // Check if data exists + let Some(data) = data else { + info!( + "No spot metadata found in database for chain {}. Run 'init-state' to populate, or it will be fetched on-demand from API.", + chain_id + ); + return; + }; + + // Deserialize metadata + let serializable_map = match rmp_serde::from_slice::>(&data) { + Ok(map) => map, + Err(e) => { + info!("Failed to deserialize spot metadata: {}. Will fetch on-demand from API.", e); + return; + } + }; + + // Convert and initialize cache + let metadata: BTreeMap = + serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect(); + + info!("Loaded spot metadata from database ({} entries)", metadata.len()); + reth_compat::initialize_spot_metadata_cache(metadata); +} + +/// Initialize spot metadata in database from API +pub fn init_spot_metadata( + db_path: impl AsRef, + db_args: reth_db::mdbx::DatabaseArguments, + chain_id: u64, +) -> eyre::Result<()> { + info!("Initializing spot metadata for chain {}", chain_id); + + let db = Arc::new(reth_db::open_db(db_path.as_ref(), db_args)?); + + // Check if spot metadata already exists + let exists = db.view(|tx| -> Result { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.is_some()) + })??; + + if exists { + info!("Spot metadata already exists in database"); + return Ok(()); + } + + // Fetch from API + let metadata = match erc20_contract_to_spot_token(chain_id) { + Ok(m) => m, + Err(e) => { + info!("Failed to fetch spot metadata from API: {}. Will be fetched on-demand.", e); + return Ok(()); + } + }; + + // Store to database + reth_compat::store_spot_metadata(&db, &metadata)?; + + info!("Successfully fetched and stored spot metadata for chain {}", chain_id); + Ok(()) +} diff --git a/src/node/spot_meta/mod.rs b/src/node/spot_meta/mod.rs index cebfee732..f0d2b67d3 100644 --- a/src/node/spot_meta/mod.rs +++ b/src/node/spot_meta/mod.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID}; +pub mod init; mod patch; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -25,7 +26,7 @@ pub struct SpotMeta { } #[derive(Debug, Clone)] -pub(crate) struct SpotId { +pub struct SpotId { pub index: u64, } diff --git a/src/node/storage/tables.rs b/src/node/storage/tables.rs index 8efa02c2e..2369bf7ad 100644 --- a/src/node/storage/tables.rs +++ b/src/node/storage/tables.rs @@ -2,10 +2,21 @@ use alloy_primitives::{BlockNumber, Bytes}; use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables}; use std::fmt; +/// Static key used for spot metadata, as the database is unique to each chain. +/// This may later serve as a versioning key to assist with future database migrations. +pub const SPOT_METADATA_KEY: u64 = 0; + tables! { /// Read precompile calls for each block. table BlockReadPrecompileCalls { type Key = BlockNumber; type Value = Bytes; } + + /// Spot metadata mapping (EVM address to spot token index). + /// Uses a constant key since the database is chain-specific. + table SpotMetadata { + type Key = u64; + type Value = Bytes; + } } diff --git a/src/node/types/mod.rs b/src/node/types/mod.rs index b25eac527..478f76d23 100644 --- a/src/node/types/mod.rs +++ b/src/node/types/mod.rs @@ -19,6 +19,9 @@ pub struct ReadPrecompileCalls(pub Vec); pub(crate) mod reth_compat; +// Re-export spot metadata functions +pub use reth_compat::{initialize_spot_metadata_cache, set_spot_metadata_db}; + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct HlExtras { pub read_precompile_calls: Option, diff --git a/src/node/types/reth_compat.rs b/src/node/types/reth_compat.rs index 016ef9371..aab96c7e1 100644 --- a/src/node/types/reth_compat.rs +++ b/src/node/types/reth_compat.rs @@ -1,11 +1,14 @@ //! Copy of reth codebase to preserve serialization compatibility +use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata}; use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; -use alloy_primitives::{Address, BlockHash, Signature, TxKind, U256}; +use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256}; +use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW}; +use reth_db_api::{Database, transaction::DbTxMut}; use reth_primitives::TransactionSigned as RethTxSigned; use serde::{Deserialize, Serialize}; use std::{ collections::BTreeMap, - sync::{Arc, LazyLock, RwLock}, + sync::{Arc, LazyLock, Mutex, RwLock}, }; use tracing::info; @@ -81,33 +84,81 @@ pub struct SealedBlock { pub body: BlockBody, } -fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { - static EVM_MAP: LazyLock>>> = - LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - { - let Transaction::Legacy(tx) = &transaction.tx else { - panic!("Unexpected transaction type"); - }; - let TxKind::Call(to) = tx.to else { - panic!("Unexpected contract creation"); - }; - let s = if tx.input.is_empty() { - U256::from(0x1) - } else { - loop { - if let Some(spot) = EVM_MAP.read().unwrap().get(&to) { - break spot.to_s(); - } +static SPOT_EVM_MAP: LazyLock>>> = + LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); - info!("Contract not found: {to:?} from spot mapping, fetching again..."); - *EVM_MAP.write().unwrap() = erc20_contract_to_spot_token(chain_id).unwrap(); - } - }; - let signature = Signature::new(U256::from(0x1), s, true); - TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +// Optional database handle for persisting on-demand fetches +static DB_HANDLE: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); + +/// Set the database handle for persisting spot metadata +pub fn set_spot_metadata_db(db: Arc) { + *DB_HANDLE.lock().unwrap() = Some(db); +} + +/// Initialize the spot metadata cache with data loaded from database. +/// This should be called during node initialization. +pub fn initialize_spot_metadata_cache(metadata: BTreeMap) { + *SPOT_EVM_MAP.write().unwrap() = metadata; +} + +/// Helper function to serialize and store spot metadata to database +pub fn store_spot_metadata( + db: &Arc, + metadata: &BTreeMap, +) -> Result<(), DatabaseError> { + db.update(|tx| { + let mut cursor = tx.cursor_write::()?; + + // Serialize to BTreeMap + let serializable_map: BTreeMap = + metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect(); + + cursor.upsert( + SPOT_METADATA_KEY, + &Bytes::from( + rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"), + ), + )?; + Ok(()) + })? +} + +/// Persist spot metadata to database if handle is available +fn persist_spot_metadata_to_db(metadata: &BTreeMap) { + if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() { + match store_spot_metadata(db, metadata) { + Ok(_) => info!("Persisted spot metadata to database"), + Err(e) => info!("Failed to persist spot metadata to database: {}", e), + } } } +fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { + let Transaction::Legacy(tx) = &transaction.tx else { + panic!("Unexpected transaction type"); + }; + let TxKind::Call(to) = tx.to else { + panic!("Unexpected contract creation"); + }; + let s = if tx.input.is_empty() { + U256::from(0x1) + } else { + loop { + if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) { + break spot.to_s(); + } + + // Cache miss - fetch from API, update cache, and persist to database + info!("Contract not found: {to:?} from spot mapping, fetching from API..."); + let metadata = erc20_contract_to_spot_token(chain_id).unwrap(); + *SPOT_EVM_MAP.write().unwrap() = metadata.clone(); + persist_spot_metadata_to_db(&metadata); + } + }; + let signature = Signature::new(U256::from(0x1), s, true); + TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) +} + impl SealedBlock { pub fn to_reth_block( &self,