15 Commits

Author SHA1 Message Date
524db5af01 Merge pull request #105 from hl-archive-node/feat/cache-spot-meta
feat: cache spot metadata in database to reduce API calls
2025-11-05 02:56:06 -05:00
98cc4ce30b feat: cache spot metadata in database to reduce API calls
Implements persistent caching of ERC20 contract address to spot token ID
mappings in the database to minimize API requests and improve performance.

Changes:
- Add SpotMetadata database table for persistent storage
- Implement load_spot_metadata_cache() to initialize cache on startup
- Add init_spot_metadata() for init-state command to pre-populate cache
- Extract store_spot_metadata() helper to DRY serialization logic
- Enable on-demand API fetches with automatic database persistence
- Integrate cache loading in main node startup flow

The cache falls back to on-demand API fetches if database is empty,
with automatic persistence of fetched data for future use.
2025-11-05 07:48:42 +00:00
95c653cf14 chore: fmt 2025-11-05 07:46:22 +00:00
cb73aa7dd4 Merge pull request #104 from hl-archive-node/chore/discovery-local-only
feat: Default to localhost-only network, add --allow-network-overrides
2025-11-05 02:44:54 -05:00
2a118cdacd feat: Default to localhost-only network, add --allow-network-overrides
Network now defaults to localhost-only (local discovery/listener, no DNS/NAT).
Use --allow-network-overrides flag to restore CLI-based network configuration.
2025-11-05 07:38:24 +00:00
ff272fcfb3 Merge pull request #103 from hl-archive-node/chore/tx-spec
fix: Adjust transaction parser
2025-11-05 02:08:52 -05:00
387acd1024 fix: Adjust transaction parser based on observation on all blocks in both networks
Tracked by #97
2025-11-05 07:00:41 +00:00
010d056aad Merge pull request #102 from hl-archive-node/fix/testnet-txs-tracking
fix: Fix testnet transaction types
2025-11-04 12:24:04 -05:00
821c63494e fix: Fix testnet transaction types 2025-11-04 17:23:31 +00:00
f915aba568 Merge pull request #100 from hl-archive-node/feat/deprecate-migrator
feat: Place migrator behind `CHECK_DB_MIGRATION` env
2025-11-01 06:23:01 -04:00
1fe03bfc41 feat: Place migrator behind CHECK_DB_MIGRATION env 2025-11-01 09:36:30 +00:00
893822e5b0 Merge pull request #98 from hl-archive-node/fix/testnet-system-tx 2025-10-26 03:39:06 -04:00
c2528ce223 fix: Support certain types of system tx 2025-10-26 06:42:14 +00:00
d46e808b8d Merge pull request #94 from hl-archive-node/fix/migrator-typo
fix(migrate): Fix wrong chunk ranges
2025-10-16 10:42:59 -04:00
497353fd2f fix(migrate): Fix wrong chunk ranges 2025-10-16 14:35:04 +00:00
20 changed files with 318 additions and 65 deletions

View File

@ -1,5 +1,5 @@
pub mod call_forwarder; pub mod call_forwarder;
pub mod hl_node_compliance; pub mod hl_node_compliance;
pub mod tx_forwarder;
pub mod subscribe_fixup; pub mod subscribe_fixup;
pub mod tx_forwarder;
mod utils; mod utils;

View File

@ -1,7 +1,10 @@
pub mod hl; pub mod hl;
pub mod parser; pub mod parser;
use crate::{hardforks::HlHardforks, node::primitives::{header::HlHeaderExtras, HlHeader}}; use crate::{
hardforks::HlHardforks,
node::primitives::{HlHeader, header::HlHeaderExtras},
};
use alloy_eips::eip7840::BlobParams; use alloy_eips::eip7840::BlobParams;
use alloy_genesis::Genesis; use alloy_genesis::Genesis;
use alloy_primitives::{Address, B256, U256}; use alloy_primitives::{Address, B256, U256};

View File

@ -18,7 +18,9 @@ use reth_hl::{
HlNode, HlNode,
cli::{Cli, HlNodeArgs}, cli::{Cli, HlNodeArgs},
rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt}, rpc::precompile::{HlBlockPrecompileApiServer, HlBlockPrecompileExt},
spot_meta::init as spot_meta_init,
storage::tables::Tables, storage::tables::Tables,
types::set_spot_metadata_db,
}, },
}; };
use tracing::info; use tracing::info;
@ -39,8 +41,11 @@ fn main() -> eyre::Result<()> {
ext: HlNodeArgs| async move { ext: HlNodeArgs| async move {
let default_upstream_rpc_url = builder.config().chain.official_rpc_url(); let default_upstream_rpc_url = builder.config().chain.official_rpc_url();
let (node, engine_handle_tx) = let (node, engine_handle_tx) = HlNode::new(
HlNode::new(ext.block_source_args.parse().await?, ext.debug_cutoff_height); ext.block_source_args.parse().await?,
ext.debug_cutoff_height,
ext.allow_network_overrides,
);
let NodeHandle { node, node_exit_future: exit_future } = builder let NodeHandle { node, node_exit_future: exit_future } = builder
.node(node) .node(node)
.extend_rpc_modules(move |mut ctx| { .extend_rpc_modules(move |mut ctx| {
@ -92,6 +97,16 @@ fn main() -> eyre::Result<()> {
}) })
.apply(|mut builder| { .apply(|mut builder| {
builder.db_mut().create_tables_for::<Tables>().expect("create tables"); builder.db_mut().create_tables_for::<Tables>().expect("create tables");
let chain_id = builder.config().chain.inner.chain().id();
let db = builder.db_mut().clone();
// Set database handle for on-demand persistence
set_spot_metadata_db(db.clone());
// Load spot metadata from database and initialize cache
spot_meta_init::load_spot_metadata_cache(&db, chain_id);
builder builder
}) })
.launch() .launch()

View File

@ -2,7 +2,7 @@ use crate::{
chainspec::{HlChainSpec, parser::HlChainSpecParser}, chainspec::{HlChainSpec, parser::HlChainSpecParser},
node::{ node::{
HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator, HlNode, consensus::HlConsensus, evm::config::HlEvmConfig, migrate::Migrator,
storage::tables::Tables, spot_meta::init as spot_meta_init, storage::tables::Tables,
}, },
pseudo_peer::BlockSourceArgs, pseudo_peer::BlockSourceArgs,
}; };
@ -82,6 +82,13 @@ pub struct HlNodeArgs {
/// * Refers to the Merkle trie used for eth_getProof and state root, not actual state values. /// * Refers to the Merkle trie used for eth_getProof and state root, not actual state values.
#[arg(long, env = "EXPERIMENTAL_ETH_GET_PROOF")] #[arg(long, env = "EXPERIMENTAL_ETH_GET_PROOF")]
pub experimental_eth_get_proof: bool, pub experimental_eth_get_proof: bool,
/// Allow network configuration overrides from CLI.
///
/// When enabled, network settings (discovery_addr, listener_addr, dns_discovery, nat)
/// will be taken from CLI arguments instead of being hardcoded to localhost-only defaults.
#[arg(long, env = "ALLOW_NETWORK_OVERRIDES")]
pub allow_network_overrides: bool,
} }
/// The main reth_hl cli interface. /// The main reth_hl cli interface.
@ -145,8 +152,12 @@ where
match self.command { match self.command {
Commands::Node(command) => runner.run_command_until_exit(|ctx| { Commands::Node(command) => runner.run_command_until_exit(|ctx| {
// NOTE: This is for one time migration around Oct 10 upgrade:
// It's not necessary anymore, an environment variable gate is added here.
if std::env::var("CHECK_DB_MIGRATION").is_ok() {
Self::migrate_db(&command.chain, &command.datadir, &command.db) Self::migrate_db(&command.chain, &command.datadir, &command.db)
.expect("Failed to migrate database"); .expect("Failed to migrate database");
}
command.execute(ctx, FnLauncher::new::<C, Ext>(launcher)) command.execute(ctx, FnLauncher::new::<C, Ext>(launcher))
}), }),
Commands::Init(command) => { Commands::Init(command) => {
@ -190,7 +201,12 @@ where
let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain()); let data_dir = env.datadir.clone().resolve_datadir(env.chain.chain());
let db_path = data_dir.db(); let db_path = data_dir.db();
init_db(db_path.clone(), env.db.database_args())?; init_db(db_path.clone(), env.db.database_args())?;
init_db_for::<_, Tables>(db_path, env.db.database_args())?; init_db_for::<_, Tables>(db_path.clone(), env.db.database_args())?;
// Initialize spot metadata in database
let chain_id = env.chain.chain().id();
spot_meta_init::init_spot_metadata(db_path, env.db.database_args(), chain_id)?;
Ok(()) Ok(())
} }

View File

@ -1,4 +1,8 @@
use crate::{hardforks::HlHardforks, node::{primitives::HlHeader, HlNode}, HlBlock, HlBlockBody, HlPrimitives}; use crate::{
HlBlock, HlBlockBody, HlPrimitives,
hardforks::HlHardforks,
node::{HlNode, primitives::HlHeader},
};
use reth::{ use reth::{
api::{FullNodeTypes, NodeTypes}, api::{FullNodeTypes, NodeTypes},
beacon_consensus::EthBeaconConsensus, beacon_consensus::EthBeaconConsensus,

View File

@ -1,5 +1,6 @@
use crate::{ use crate::{
node::evm::config::{HlBlockExecutorFactory, HlEvmConfig}, HlBlock, HlHeader HlBlock, HlHeader,
node::evm::config::{HlBlockExecutorFactory, HlEvmConfig},
}; };
use reth_evm::{ use reth_evm::{
block::BlockExecutionError, block::BlockExecutionError,

View File

@ -270,8 +270,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> {
let mut first = true; let mut first = true;
for (block_range, _tx_ranges) in all_static_files { for (block_range, _tx_ranges) in all_static_files {
let migration_needed = self.using_old_header(block_range.start())? let migration_needed = self.using_old_header(block_range.start())? ||
|| self.using_old_header(block_range.end())?; self.using_old_header(block_range.end())?;
if !migration_needed { if !migration_needed {
// Create a placeholder symlink // Create a placeholder symlink
self.create_placeholder(block_range)?; self.create_placeholder(block_range)?;
@ -346,7 +346,7 @@ fn migrate_single_static_file<N: HlNodeType>(
// block_ranges into chunks of 50000 blocks // block_ranges into chunks of 50000 blocks
const CHUNK_SIZE: u64 = 50000; const CHUNK_SIZE: u64 = 50000;
for chunk in (0..=block_range.end()).step_by(CHUNK_SIZE as usize) { for chunk in (block_range.start()..=block_range.end()).step_by(CHUNK_SIZE as usize) {
let end = std::cmp::min(chunk + CHUNK_SIZE - 1, block_range.end()); let end = std::cmp::min(chunk + CHUNK_SIZE - 1, block_range.end());
let block_range = chunk..=end; let block_range = chunk..=end;
let headers = old_headers_range(sf_in, block_range.clone())?; let headers = old_headers_range(sf_in, block_range.clone())?;

View File

@ -51,12 +51,14 @@ pub struct HlNode {
engine_handle_rx: Arc<Mutex<Option<oneshot::Receiver<ConsensusEngineHandle<HlPayloadTypes>>>>>, engine_handle_rx: Arc<Mutex<Option<oneshot::Receiver<ConsensusEngineHandle<HlPayloadTypes>>>>>,
block_source_config: BlockSourceConfig, block_source_config: BlockSourceConfig,
debug_cutoff_height: Option<u64>, debug_cutoff_height: Option<u64>,
allow_network_overrides: bool,
} }
impl HlNode { impl HlNode {
pub fn new( pub fn new(
block_source_config: BlockSourceConfig, block_source_config: BlockSourceConfig,
debug_cutoff_height: Option<u64>, debug_cutoff_height: Option<u64>,
allow_network_overrides: bool,
) -> (Self, oneshot::Sender<ConsensusEngineHandle<HlPayloadTypes>>) { ) -> (Self, oneshot::Sender<ConsensusEngineHandle<HlPayloadTypes>>) {
let (tx, rx) = oneshot::channel(); let (tx, rx) = oneshot::channel();
( (
@ -64,6 +66,7 @@ impl HlNode {
engine_handle_rx: Arc::new(Mutex::new(Some(rx))), engine_handle_rx: Arc::new(Mutex::new(Some(rx))),
block_source_config, block_source_config,
debug_cutoff_height, debug_cutoff_height,
allow_network_overrides,
}, },
tx, tx,
) )
@ -95,6 +98,7 @@ impl HlNode {
engine_handle_rx: self.engine_handle_rx.clone(), engine_handle_rx: self.engine_handle_rx.clone(),
block_source_config: self.block_source_config.clone(), block_source_config: self.block_source_config.clone(),
debug_cutoff_height: self.debug_cutoff_height, debug_cutoff_height: self.debug_cutoff_height,
allow_network_overrides: self.allow_network_overrides,
}) })
.consensus(HlConsensusBuilder::default()) .consensus(HlConsensusBuilder::default())
} }

View File

@ -179,7 +179,7 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::{chainspec::hl::hl_mainnet, HlHeader}; use crate::{HlHeader, chainspec::hl::hl_mainnet};
use super::*; use super::*;
use alloy_primitives::{B256, U128}; use alloy_primitives::{B256, U128};

View File

@ -25,7 +25,10 @@ use reth_network::{NetworkConfig, NetworkHandle, NetworkManager};
use reth_network_api::PeersInfo; use reth_network_api::PeersInfo;
use reth_provider::StageCheckpointReader; use reth_provider::StageCheckpointReader;
use reth_stages_types::StageId; use reth_stages_types::StageId;
use std::sync::Arc; use std::{
net::{Ipv4Addr, SocketAddr},
sync::Arc,
};
use tokio::sync::{Mutex, mpsc, oneshot}; use tokio::sync::{Mutex, mpsc, oneshot};
use tracing::info; use tracing::info;
@ -144,6 +147,8 @@ pub struct HlNetworkBuilder {
pub(crate) block_source_config: BlockSourceConfig, pub(crate) block_source_config: BlockSourceConfig,
pub(crate) debug_cutoff_height: Option<u64>, pub(crate) debug_cutoff_height: Option<u64>,
pub(crate) allow_network_overrides: bool,
} }
impl HlNetworkBuilder { impl HlNetworkBuilder {
@ -174,15 +179,24 @@ impl HlNetworkBuilder {
ImportService::new(consensus, handle, from_network, to_network).await.unwrap(); ImportService::new(consensus, handle, from_network, to_network).await.unwrap();
}); });
Ok(ctx.build_network_config( let mut config_builder = ctx.network_config_builder()?;
ctx.network_config_builder()?
// Only apply localhost-only network settings if network overrides are NOT allowed
if !self.allow_network_overrides {
config_builder = config_builder
.discovery_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
.listener_addr(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
.disable_dns_discovery() .disable_dns_discovery()
.disable_nat() .disable_nat();
}
config_builder = config_builder
.boot_nodes(boot_nodes()) .boot_nodes(boot_nodes())
.set_head(ctx.head()) .set_head(ctx.head())
.with_pow() .with_pow()
.block_import(Box::new(HlBlockImport::new(handle))), .block_import(Box::new(HlBlockImport::new(handle)));
))
Ok(ctx.build_network_config(config_builder))
} }
} }

View File

@ -3,8 +3,13 @@ use alloy_primitives::Address;
use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize}; use reth_primitives_traits::{BlockBody as BlockBodyTrait, InMemorySize};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::node::types::{ReadPrecompileCall, ReadPrecompileCalls}; use crate::{
use crate::{HlHeader, node::primitives::TransactionSigned}; HlHeader,
node::{
primitives::TransactionSigned,
types::{ReadPrecompileCall, ReadPrecompileCalls},
},
};
/// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars /// Block body for HL. It is equivalent to Ethereum [`BlockBody`] but additionally stores sidecars
/// for blob transactions. /// for blob transactions.
@ -33,13 +38,11 @@ pub type BlockBody = alloy_consensus::BlockBody<TransactionSigned, HlHeader>;
impl InMemorySize for HlBlockBody { impl InMemorySize for HlBlockBody {
fn size(&self) -> usize { fn size(&self) -> usize {
self.inner.size() self.inner.size() +
+ self self.sidecars
.sidecars
.as_ref() .as_ref()
.map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>()) .map_or(0, |s| s.capacity() * core::mem::size_of::<BlobTransactionSidecar>()) +
+ self self.read_precompile_calls
.read_precompile_calls
.as_ref() .as_ref()
.map_or(0, |s| s.0.capacity() * core::mem::size_of::<ReadPrecompileCall>()) .map_or(0, |s| s.0.capacity() * core::mem::size_of::<ReadPrecompileCall>())
} }

View File

@ -45,7 +45,11 @@ pub struct HlHeaderExtras {
} }
impl HlHeader { impl HlHeader {
pub(crate) fn from_ethereum_header(header: Header, receipts: &[EthereumReceipt], system_tx_count: u64) -> HlHeader { pub(crate) fn from_ethereum_header(
header: Header,
receipts: &[EthereumReceipt],
system_tx_count: u64,
) -> HlHeader {
let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs)); let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| &r.logs));
HlHeader { HlHeader {
inner: header, inner: header,
@ -183,8 +187,9 @@ impl reth_codecs::Compact for HlHeader {
// because Compact trait requires the Bytes field to be placed at the end of the struct. // because Compact trait requires the Bytes field to be placed at the end of the struct.
// Bytes::from_compact just reads all trailing data as the Bytes field. // Bytes::from_compact just reads all trailing data as the Bytes field.
// //
// Hence we need to use other form of serialization, since extra headers are not Compact-compatible. // Hence we need to use other form of serialization, since extra headers are not
// We just treat all header fields as rmp-serialized one `Bytes` field. // Compact-compatible. We just treat all header fields as rmp-serialized one `Bytes`
// field.
let result: Bytes = rmp_serde::to_vec(&self).unwrap().into(); let result: Bytes = rmp_serde::to_vec(&self).unwrap().into();
result.to_compact(buf) result.to_compact(buf)
} }

View File

@ -1,6 +1,6 @@
#![allow(clippy::owned_cow)] #![allow(clippy::owned_cow)]
use super::{HlBlock, HlBlockBody, TransactionSigned}; use super::{HlBlock, HlBlockBody, TransactionSigned};
use crate::{node::types::ReadPrecompileCalls, HlHeader}; use crate::{HlHeader, node::types::ReadPrecompileCalls};
use alloy_consensus::{BlobTransactionSidecar, BlockBody}; use alloy_consensus::{BlobTransactionSidecar, BlockBody};
use alloy_eips::eip4895::Withdrawals; use alloy_eips::eip4895::Withdrawals;
use alloy_primitives::Address; use alloy_primitives::Address;

View File

@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize};
use std::borrow::Cow; use std::borrow::Cow;
use super::{HlBlock, HlBlockBody}; use super::{HlBlock, HlBlockBody};
use crate::{node::{primitives::BlockBody, types::ReadPrecompileCalls}, HlHeader}; use crate::{
HlHeader,
node::{primitives::BlockBody, types::ReadPrecompileCalls},
};
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct HlBlockBodyBincode<'a> { pub struct HlBlockBodyBincode<'a> {

103
src/node/spot_meta/init.rs Normal file
View File

@ -0,0 +1,103 @@
use crate::node::{
spot_meta::{SpotId, erc20_contract_to_spot_token},
storage::tables::{self, SPOT_METADATA_KEY},
types::reth_compat,
};
use alloy_primitives::Address;
use reth_db::{
DatabaseEnv,
cursor::DbCursorRO,
};
use reth_db_api::{
Database,
transaction::DbTx,
};
use std::{collections::BTreeMap, sync::Arc};
use tracing::info;
/// Load spot metadata from database and initialize cache
pub fn load_spot_metadata_cache(db: &Arc<DatabaseEnv>, chain_id: u64) {
// Try to read from database
let data = match db.view(|tx| -> Result<Option<Vec<u8>>, reth_db::DatabaseError> {
let mut cursor = tx.cursor_read::<tables::SpotMetadata>()?;
Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.map(|(_, data)| data.to_vec()))
}) {
Ok(Ok(data)) => data,
Ok(Err(e)) => {
info!(
"Failed to read spot metadata from database: {}. Will fetch on-demand from API.",
e
);
return;
}
Err(e) => {
info!(
"Database view error while loading spot metadata: {}. Will fetch on-demand from API.",
e
);
return;
}
};
// Check if data exists
let Some(data) = data else {
info!(
"No spot metadata found in database for chain {}. Run 'init-state' to populate, or it will be fetched on-demand from API.",
chain_id
);
return;
};
// Deserialize metadata
let serializable_map = match rmp_serde::from_slice::<BTreeMap<Address, u64>>(&data) {
Ok(map) => map,
Err(e) => {
info!("Failed to deserialize spot metadata: {}. Will fetch on-demand from API.", e);
return;
}
};
// Convert and initialize cache
let metadata: BTreeMap<Address, SpotId> =
serializable_map.into_iter().map(|(addr, index)| (addr, SpotId { index })).collect();
info!("Loaded spot metadata from database ({} entries)", metadata.len());
reth_compat::initialize_spot_metadata_cache(metadata);
}
/// Initialize spot metadata in database from API
pub fn init_spot_metadata(
db_path: impl AsRef<std::path::Path>,
db_args: reth_db::mdbx::DatabaseArguments,
chain_id: u64,
) -> eyre::Result<()> {
info!("Initializing spot metadata for chain {}", chain_id);
let db = Arc::new(reth_db::open_db(db_path.as_ref(), db_args)?);
// Check if spot metadata already exists
let exists = db.view(|tx| -> Result<bool, reth_db::DatabaseError> {
let mut cursor = tx.cursor_read::<tables::SpotMetadata>()?;
Ok(cursor.seek_exact(SPOT_METADATA_KEY)?.is_some())
})??;
if exists {
info!("Spot metadata already exists in database");
return Ok(());
}
// Fetch from API
let metadata = match erc20_contract_to_spot_token(chain_id) {
Ok(m) => m,
Err(e) => {
info!("Failed to fetch spot metadata from API: {}. Will be fetched on-demand.", e);
return Ok(());
}
};
// Store to database
reth_compat::store_spot_metadata(&db, &metadata)?;
info!("Successfully fetched and stored spot metadata for chain {}", chain_id);
Ok(())
}

View File

@ -5,6 +5,7 @@ use std::collections::BTreeMap;
use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID}; use crate::chainspec::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID};
pub mod init;
mod patch; mod patch;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@ -25,7 +26,7 @@ pub struct SpotMeta {
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(crate) struct SpotId { pub struct SpotId {
pub index: u64, pub index: u64,
} }

View File

@ -2,10 +2,21 @@ use alloy_primitives::{BlockNumber, Bytes};
use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables}; use reth_db::{TableSet, TableType, TableViewer, table::TableInfo, tables};
use std::fmt; use std::fmt;
/// Static key used for spot metadata, as the database is unique to each chain.
/// This may later serve as a versioning key to assist with future database migrations.
pub const SPOT_METADATA_KEY: u64 = 0;
tables! { tables! {
/// Read precompile calls for each block. /// Read precompile calls for each block.
table BlockReadPrecompileCalls { table BlockReadPrecompileCalls {
type Key = BlockNumber; type Key = BlockNumber;
type Value = Bytes; type Value = Bytes;
} }
/// Spot metadata mapping (EVM address to spot token index).
/// Uses a constant key since the database is chain-specific.
table SpotMetadata {
type Key = u64;
type Value = Bytes;
}
} }

View File

@ -19,6 +19,9 @@ pub struct ReadPrecompileCalls(pub Vec<ReadPrecompileCall>);
pub(crate) mod reth_compat; pub(crate) mod reth_compat;
// Re-export spot metadata functions
pub use reth_compat::{initialize_spot_metadata_cache, set_spot_metadata_db};
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct HlExtras { pub struct HlExtras {
pub read_precompile_calls: Option<ReadPrecompileCalls>, pub read_precompile_calls: Option<ReadPrecompileCalls>,
@ -127,6 +130,19 @@ pub struct SystemTx {
pub receipt: Option<LegacyReceipt>, pub receipt: Option<LegacyReceipt>,
} }
impl SystemTx {
pub fn gas_limit(&self) -> u64 {
use reth_compat::Transaction;
match &self.tx {
Transaction::Legacy(tx) => tx.gas_limit,
Transaction::Eip2930(tx) => tx.gas_limit,
Transaction::Eip1559(tx) => tx.gas_limit,
Transaction::Eip4844(tx) => tx.gas_limit,
Transaction::Eip7702(tx) => tx.gas_limit,
}
}
}
#[derive( #[derive(
Debug, Debug,
Clone, Clone,

View File

@ -1,11 +1,14 @@
//! Copy of reth codebase to preserve serialization compatibility //! Copy of reth codebase to preserve serialization compatibility
use crate::node::storage::tables::{SPOT_METADATA_KEY, SpotMetadata};
use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; use alloy_consensus::{Header, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy};
use alloy_primitives::{Address, BlockHash, Signature, TxKind, U256}; use alloy_primitives::{Address, BlockHash, Bytes, Signature, TxKind, U256};
use reth_db::{DatabaseEnv, DatabaseError, cursor::DbCursorRW};
use reth_db_api::{Database, transaction::DbTxMut};
use reth_primitives::TransactionSigned as RethTxSigned; use reth_primitives::TransactionSigned as RethTxSigned;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::{ use std::{
collections::BTreeMap, collections::BTreeMap,
sync::{Arc, LazyLock, RwLock}, sync::{Arc, LazyLock, Mutex, RwLock},
}; };
use tracing::info; use tracing::info;
@ -81,10 +84,56 @@ pub struct SealedBlock {
pub body: BlockBody, pub body: BlockBody,
} }
fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned { static SPOT_EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
static EVM_MAP: LazyLock<Arc<RwLock<BTreeMap<Address, SpotId>>>> =
LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new()))); LazyLock::new(|| Arc::new(RwLock::new(BTreeMap::new())));
{
// Optional database handle for persisting on-demand fetches
static DB_HANDLE: LazyLock<Mutex<Option<Arc<DatabaseEnv>>>> = LazyLock::new(|| Mutex::new(None));
/// Set the database handle for persisting spot metadata
pub fn set_spot_metadata_db(db: Arc<DatabaseEnv>) {
*DB_HANDLE.lock().unwrap() = Some(db);
}
/// Initialize the spot metadata cache with data loaded from database.
/// This should be called during node initialization.
pub fn initialize_spot_metadata_cache(metadata: BTreeMap<Address, SpotId>) {
*SPOT_EVM_MAP.write().unwrap() = metadata;
}
/// Helper function to serialize and store spot metadata to database
pub fn store_spot_metadata(
db: &Arc<DatabaseEnv>,
metadata: &BTreeMap<Address, SpotId>,
) -> Result<(), DatabaseError> {
db.update(|tx| {
let mut cursor = tx.cursor_write::<SpotMetadata>()?;
// Serialize to BTreeMap<Address, u64>
let serializable_map: BTreeMap<Address, u64> =
metadata.iter().map(|(addr, spot)| (*addr, spot.index)).collect();
cursor.upsert(
SPOT_METADATA_KEY,
&Bytes::from(
rmp_serde::to_vec(&serializable_map).expect("Failed to serialize spot metadata"),
),
)?;
Ok(())
})?
}
/// Persist spot metadata to database if handle is available
fn persist_spot_metadata_to_db(metadata: &BTreeMap<Address, SpotId>) {
if let Some(db) = DB_HANDLE.lock().unwrap().as_ref() {
match store_spot_metadata(db, metadata) {
Ok(_) => info!("Persisted spot metadata to database"),
Err(e) => info!("Failed to persist spot metadata to database: {}", e),
}
}
}
fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSigned {
let Transaction::Legacy(tx) = &transaction.tx else { let Transaction::Legacy(tx) = &transaction.tx else {
panic!("Unexpected transaction type"); panic!("Unexpected transaction type");
}; };
@ -95,28 +144,33 @@ fn system_tx_to_reth_transaction(transaction: &SystemTx, chain_id: u64) -> TxSig
U256::from(0x1) U256::from(0x1)
} else { } else {
loop { loop {
if let Some(spot) = EVM_MAP.read().unwrap().get(&to) { if let Some(spot) = SPOT_EVM_MAP.read().unwrap().get(&to) {
break spot.to_s(); break spot.to_s();
} }
info!("Contract not found: {to:?} from spot mapping, fetching again..."); // Cache miss - fetch from API, update cache, and persist to database
*EVM_MAP.write().unwrap() = erc20_contract_to_spot_token(chain_id).unwrap(); info!("Contract not found: {to:?} from spot mapping, fetching from API...");
let metadata = erc20_contract_to_spot_token(chain_id).unwrap();
*SPOT_EVM_MAP.write().unwrap() = metadata.clone();
persist_spot_metadata_to_db(&metadata);
} }
}; };
let signature = Signature::new(U256::from(0x1), s, true); let signature = Signature::new(U256::from(0x1), s, true);
TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature))) TxSigned::Default(RethTxSigned::Legacy(Signed::new_unhashed(tx.clone(), signature)))
} }
}
impl SealedBlock { impl SealedBlock {
pub fn to_reth_block( pub fn to_reth_block(
&self, &self,
read_precompile_calls: ReadPrecompileCalls, read_precompile_calls: ReadPrecompileCalls,
highest_precompile_address: Option<Address>, highest_precompile_address: Option<Address>,
system_txs: Vec<super::SystemTx>, mut system_txs: Vec<super::SystemTx>,
receipts: Vec<LegacyReceipt>, receipts: Vec<LegacyReceipt>,
chain_id: u64, chain_id: u64,
) -> HlBlock { ) -> HlBlock {
// NOTE: These types of transactions are tracked at #97.
system_txs.retain(|tx| tx.receipt.is_some());
let mut merged_txs = vec![]; let mut merged_txs = vec![];
merged_txs.extend(system_txs.iter().map(|tx| system_tx_to_reth_transaction(tx, chain_id))); merged_txs.extend(system_txs.iter().map(|tx| system_tx_to_reth_transaction(tx, chain_id)));
merged_txs.extend(self.body.transactions.iter().map(|tx| tx.to_reth_transaction())); merged_txs.extend(self.body.transactions.iter().map(|tx| tx.to_reth_transaction()));

View File

@ -82,8 +82,8 @@ impl BlockPoller {
.ok_or(eyre::eyre!("Failed to find latest block number"))?; .ok_or(eyre::eyre!("Failed to find latest block number"))?;
loop { loop {
if let Some(debug_cutoff_height) = debug_cutoff_height if let Some(debug_cutoff_height) = debug_cutoff_height &&
&& next_block_number > debug_cutoff_height next_block_number > debug_cutoff_height
{ {
next_block_number = debug_cutoff_height; next_block_number = debug_cutoff_height;
} }