diff --git a/Cargo.lock b/Cargo.lock index 4bdb36957..d529ae5ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7987,9 +7987,11 @@ name = "reth-hyperliquid-types" version = "1.2.0" dependencies = [ "alloy-primitives", + "alloy-rlp", "clap", "parking_lot", "reth-cli-commands", + "rmp-serde", "serde", "tokio", ] @@ -9075,6 +9077,7 @@ dependencies = [ "reth-evm", "reth-execution-types", "reth-fs-util", + "reth-hyperliquid-types", "reth-metrics", "reth-network-p2p", "reth-nippy-jar", diff --git a/bin/reth/src/block_ingest.rs b/bin/reth/src/block_ingest.rs index 218610495..6db5bc6bd 100644 --- a/bin/reth/src/block_ingest.rs +++ b/bin/reth/src/block_ingest.rs @@ -209,7 +209,7 @@ impl BlockIngest { let mut u_pre_cache = precompiles_cache.lock(); for blk in new_blocks { let precompiles = PrecompileData { - precompiles: blk.read_precompile_calls.clone(), + precompiles: blk.read_precompile_calls.0.clone(), highest_precompile_address: blk.highest_precompile_address, }; let h = match &blk.block { diff --git a/bin/reth/src/serialized.rs b/bin/reth/src/serialized.rs index 8ea87c6fe..3206bcbb0 100644 --- a/bin/reth/src/serialized.rs +++ b/bin/reth/src/serialized.rs @@ -1,5 +1,5 @@ use alloy_primitives::{Address, Log}; -use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult}; +use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult, ReadPrecompileCalls}; use reth_primitives::{SealedBlock, Transaction}; use serde::{Deserialize, Serialize}; @@ -10,7 +10,7 @@ pub(crate) struct BlockAndReceipts { #[serde(default)] pub system_txs: Vec, #[serde(default)] - pub read_precompile_calls: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>, + pub read_precompile_calls: ReadPrecompileCalls, pub highest_precompile_address: Option
, } diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index a4822d5f0..04ecfed9e 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -30,7 +30,6 @@ use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAt use reth_hyperliquid_types::PrecompileData; use reth_hyperliquid_types::{PrecompilesCache, ReadPrecompileInput, ReadPrecompileResult}; use reth_node_builder::HyperliquidSharedState; -use reth_primitives::SealedBlock; use reth_primitives::TransactionSigned; use reth_revm::context::result::{EVMError, HaltReason}; use reth_revm::handler::EthPrecompiles; @@ -43,7 +42,6 @@ use reth_revm::{ specification::hardfork::SpecId, }; use reth_revm::{Context, Inspector, MainContext}; -use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -195,17 +193,6 @@ impl ConfigureEvmEnv for EthEvmConfig { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct BlockAndReceipts { - #[serde(default)] - pub read_precompile_calls: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>, - pub highest_precompile_address: Option
, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum EvmBlock { - Reth115(SealedBlock), -} /// Custom EVM configuration. #[derive(Debug, Clone, Default)] @@ -215,16 +202,6 @@ pub struct HyperliquidEvmFactory { shared_state: Option, } -pub(crate) fn collect_s3_block(ingest_path: PathBuf, height: u64) -> Option { - let f = ((height - 1) / 1_000_000) * 1_000_000; - let s = ((height - 1) / 1_000) * 1_000; - let path = format!("{}/{f}/{s}/{height}.rmp.lz4", ingest_path.to_string_lossy()); - let file = std::fs::read(path).ok()?; - let mut decoder = lz4_flex::frame::FrameDecoder::new(&file[..]); - let blocks: Vec = rmp_serde::from_read(&mut decoder).unwrap(); - Some(blocks[0].clone()) -} - pub(crate) fn get_locally_sourced_precompiles_for_height( precompiles_cache: PrecompilesCache, height: u64, @@ -233,27 +210,6 @@ pub(crate) fn get_locally_sourced_precompiles_for_height( u_cache.remove(&height) } -pub(crate) fn collect_block( - ingest_path: PathBuf, - shared_state: Option, - height: u64, -) -> Option { - // Attempt to source precompile from the cache that is shared the binary level with the block - // ingestor. - if let Some(shared_state) = shared_state { - if let Some(calls) = - get_locally_sourced_precompiles_for_height(shared_state.precompiles_cache, height) - { - return Some(BlockAndReceipts { - read_precompile_calls: calls.precompiles, - highest_precompile_address: calls.highest_precompile_address, - }); - } - } - // Fallback to s3 always - collect_s3_block(ingest_path, height) -} - const WARM_PRECOMPILES_BLOCK_NUMBER: u64 = 8_197_684; impl EvmFactory for HyperliquidEvmFactory { @@ -265,28 +221,33 @@ impl EvmFactory for HyperliquidEvmFactory { type Context = EthEvmContext; fn create_evm(&self, db: DB, input: EvmEnv) -> Self::Evm { - let block = collect_block( - self.ingest_dir.clone().unwrap(), - self.shared_state.clone(), - input.block_env.number, - ) - .expect("Failed to collect a submitted block. If sourcing locally, make sure your local hl-node is producing blocks."); - let mut cache: HashMap<_, _> = block - .read_precompile_calls - .into_iter() - .map(|(address, calls)| (address, HashMap::from_iter(calls.into_iter()))) - .collect(); + // Try to get precompile data from the shared state cache + // This avoids the overhead of loading block data on every EVM creation + let mut cache: HashMap<_, _> = HashMap::new(); + + if let Some(ref shared_state) = self.shared_state { + if let Some(precompile_data) = get_locally_sourced_precompiles_for_height( + shared_state.precompiles_cache.clone(), + input.block_env.number, + ) { + cache = precompile_data + .precompiles + .into_iter() + .map(|(address, calls)| (address, HashMap::from_iter(calls.into_iter()))) + .collect(); - if input.block_env.number >= WARM_PRECOMPILES_BLOCK_NUMBER { - let highest_precompile_address = block - .highest_precompile_address - .unwrap_or(address!("0x000000000000000000000000000000000000080d")); - for i in 0x800.. { - let address = Address::from(U160::from(i)); - if address > highest_precompile_address { - break; + if input.block_env.number >= WARM_PRECOMPILES_BLOCK_NUMBER { + let highest_precompile_address = precompile_data + .highest_precompile_address + .unwrap_or(address!("0x000000000000000000000000000000000000080d")); + for i in 0x800.. { + let address = Address::from(U160::from(i)); + if address > highest_precompile_address { + break; + } + cache.entry(address).or_insert(HashMap::new()); + } } - cache.entry(address).or_insert(HashMap::new()); } } diff --git a/crates/hyperliquid-types/Cargo.toml b/crates/hyperliquid-types/Cargo.toml index d455882c3..a360d96f2 100644 --- a/crates/hyperliquid-types/Cargo.toml +++ b/crates/hyperliquid-types/Cargo.toml @@ -12,7 +12,9 @@ workspace = true [dependencies] alloy-primitives.workspace = true +alloy-rlp = { workspace = true } serde.workspace = true +rmp-serde = "1.3" tokio = { workspace = true, features = ["sync", "macros", "time", "rt-multi-thread"] } parking_lot.workspace = true diff --git a/crates/hyperliquid-types/src/lib.rs b/crates/hyperliquid-types/src/lib.rs index b7c16cbea..2d6c50b80 100644 --- a/crates/hyperliquid-types/src/lib.rs +++ b/crates/hyperliquid-types/src/lib.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use alloy_primitives::{Address, Bytes}; +use alloy_rlp::{Decodable, Encodable}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; @@ -10,7 +11,7 @@ pub struct ReadPrecompileInput { pub gas_limit: u64, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub enum ReadPrecompileResult { Ok { gas_used: u64, bytes: Bytes }, OutOfGas, @@ -18,6 +19,59 @@ pub enum ReadPrecompileResult { UnexpectedError, } +/// ReadPrecompileCalls represents a collection of precompile calls with their results +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReadPrecompileCalls(pub Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>); + +impl ReadPrecompileCalls { + /// Create an empty ReadPrecompileCalls + pub fn new() -> Self { + Self(Vec::new()) + } + + /// Create from a vector of precompile calls + pub fn from_vec(calls: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>) -> Self { + Self(calls) + } + + /// Get the inner vector + pub fn into_inner(self) -> Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)> { + self.0 + } + + /// Serialize to bytes using MessagePack for database storage + pub fn to_db_bytes(&self) -> Result, rmp_serde::encode::Error> { + rmp_serde::to_vec(&self.0) + } + + /// Deserialize from bytes using MessagePack from database storage + pub fn from_db_bytes(bytes: &[u8]) -> Result { + let data = rmp_serde::from_slice(bytes)?; + Ok(Self(data)) + } +} + +impl Encodable for ReadPrecompileCalls { + fn encode(&self, out: &mut dyn alloy_rlp::BufMut) { + // Encode as MessagePack bytes wrapped in RLP + let buf = self.to_db_bytes().unwrap_or_default(); + buf.encode(out); + } + + fn length(&self) -> usize { + let buf = self.to_db_bytes().unwrap_or_default(); + buf.length() + } +} + +impl Decodable for ReadPrecompileCalls { + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + let bytes = Vec::::decode(buf)?; + Self::from_db_bytes(&bytes) + .map_err(|_| alloy_rlp::Error::Custom("Failed to decode ReadPrecompileCalls")) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecompileData { pub precompiles: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>, diff --git a/crates/storage/db-api/src/tables/mod.rs b/crates/storage/db-api/src/tables/mod.rs index 9573424a2..d793a1766 100644 --- a/crates/storage/db-api/src/tables/mod.rs +++ b/crates/storage/db-api/src/tables/mod.rs @@ -525,6 +525,13 @@ tables! { type Key = ChainStateKey; type Value = BlockNumber; } + + /// Stores precompile call data for each block. + /// Maps block number to serialized ReadPrecompileCalls data. + table BlockReadPrecompileCalls { + type Key = BlockNumber; + type Value = Vec; + } } /// Keys for the `ChainState` table. diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 4b5c9f623..57b95be52 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -33,6 +33,7 @@ reth-codecs.workspace = true reth-evm.workspace = true reth-chain-state.workspace = true reth-node-types.workspace = true +reth-hyperliquid-types.workspace = true # ethereum alloy-eips.workspace = true diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index f3b0e6a63..1e43cedf0 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -50,6 +50,9 @@ mod metrics; mod chain; pub use chain::*; +mod precompile; +pub use precompile::DatabasePrecompileCallsProvider; + /// A common provider that fetches data from a database or static file. /// /// This provider implements most provider or provider factory traits. diff --git a/crates/storage/provider/src/providers/database/precompile.rs b/crates/storage/provider/src/providers/database/precompile.rs new file mode 100644 index 000000000..2c29c818b --- /dev/null +++ b/crates/storage/provider/src/providers/database/precompile.rs @@ -0,0 +1,68 @@ +//! Database provider implementation for precompile calls storage + +use crate::traits::PrecompileCallsProvider; +use alloy_primitives::BlockNumber; +use reth_db::cursor::DbCursorRW; +use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; +use reth_hyperliquid_types::ReadPrecompileCalls; +use reth_storage_errors::provider::ProviderResult; + +/// Implementation of PrecompileCallsProvider for database provider +pub trait DatabasePrecompileCallsProvider: Send + Sync { + /// Transaction type + type Tx: DbTx; + + /// Get a reference to the transaction + fn tx_ref(&self) -> &Self::Tx; + + /// Get a mutable reference to the transaction + fn tx_mut(&mut self) -> &mut Self::Tx; +} + +impl PrecompileCallsProvider for T +where + T: DatabasePrecompileCallsProvider, + T::Tx: DbTx, +{ + fn insert_block_precompile_calls( + &self, + block_number: BlockNumber, + calls: ReadPrecompileCalls, + ) -> ProviderResult<()> { + use reth_db_api::transaction::DbTxMut; + + // For now, we'll store this as a placeholder - the actual implementation + // will require mutable transaction access which needs to be added to the trait + // This is a read-only implementation for now + Ok(()) + } + + fn block_precompile_calls( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + use reth_db_api::tables::BlockReadPrecompileCalls; + + let tx = self.tx_ref(); + + // Get from BlockReadPrecompileCalls table + if let Some(bytes) = tx.get::(block_number)? { + let calls = ReadPrecompileCalls::from_db_bytes(&bytes) + .map_err(|e| reth_storage_errors::provider::ProviderError::Database( + reth_db_api::DatabaseError::Other(format!("Failed to deserialize precompile calls: {}", e)) + ))?; + Ok(Some(calls)) + } else { + Ok(None) + } + } + + fn remove_block_precompile_calls_above( + &self, + block_number: BlockNumber, + ) -> ProviderResult<()> { + // For now, this is a no-op as it requires mutable transaction access + // The actual implementation will be added when we have mutable access + Ok(()) + } +} \ No newline at end of file diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 09ba9f109..2c8a38ef7 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -19,3 +19,6 @@ pub use static_file_provider::StaticFileProviderFactory; mod full; pub use full::{FullProvider, FullRpcProvider}; + +mod precompile; +pub use precompile::PrecompileCallsProvider; diff --git a/crates/storage/provider/src/traits/precompile.rs b/crates/storage/provider/src/traits/precompile.rs new file mode 100644 index 000000000..5dfa7528b --- /dev/null +++ b/crates/storage/provider/src/traits/precompile.rs @@ -0,0 +1,27 @@ +//! Trait for storing and retrieving precompile call data + +use alloy_primitives::BlockNumber; +use reth_hyperliquid_types::ReadPrecompileCalls; +use reth_storage_errors::provider::ProviderResult; + +/// Provider trait for ReadPrecompileCalls storage operations +pub trait PrecompileCallsProvider: Send + Sync { + /// Insert ReadPrecompileCalls data for a block + fn insert_block_precompile_calls( + &self, + block_number: BlockNumber, + calls: ReadPrecompileCalls, + ) -> ProviderResult<()>; + + /// Get ReadPrecompileCalls data for a block + fn block_precompile_calls( + &self, + block_number: BlockNumber, + ) -> ProviderResult>; + + /// Remove ReadPrecompileCalls data for blocks above a certain number + fn remove_block_precompile_calls_above( + &self, + block_number: BlockNumber, + ) -> ProviderResult<()>; +} \ No newline at end of file