From 20fce881679d86fbb78e9bc437a72f59626eb5ea Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:14:19 +0000 Subject: [PATCH 1/9] fix: Use correct types for system tx receipts --- bin/reth/src/serialized.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/reth/src/serialized.rs b/bin/reth/src/serialized.rs index 25101e63e..d351a377c 100644 --- a/bin/reth/src/serialized.rs +++ b/bin/reth/src/serialized.rs @@ -1,6 +1,5 @@ use alloy_consensus::{TxEip1559, TxEip2930, TxLegacy}; -use alloy_rpc_types::Log; -use reth_primitives::{SealedBlock, Transaction}; +use reth_primitives::{Log, SealedBlock, Transaction}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] From a01133737d0ebf044467dcec710b90b00995dfa1 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:13:42 +0000 Subject: [PATCH 2/9] chore: Move dependencies to workspace --- Cargo.toml | 3 +++ bin/reth/Cargo.toml | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b0ab50f66..fd5fa8cb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -622,6 +622,9 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] } # See: https://github.com/eira-fransham/crunchy/issues/13 crunchy = "=0.2.2" +lz4_flex = "0.11.3" +rmp-serde = "1.3.0" + [patch.crates-io] alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "beb6832" } alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "beb6832" } diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 22bbdeaf6..6523f70f7 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -90,8 +90,6 @@ backon.workspace = true similar-asserts.workspace = true parking_lot.workspace = true -lz4_flex = "0.11.3" -rmp-serde = "1.3.0" serde = { workspace = true, features = ["derive"] } reth-e2e-test-utils.workspace = true once_cell.workspace = true @@ -100,6 +98,10 @@ jsonrpsee.workspace = true jsonrpsee-core.workspace = true reth-rpc-layer.workspace = true +lz4_flex.workspace = true +rmp-serde.workspace = true +reqwest.workspace = true + [dev-dependencies] tempfile.workspace = true From a5e51afb361cc55f0c3d671f79e948d1e9184f11 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:14:03 +0000 Subject: [PATCH 3/9] feat: Support erc20 token transfer --- bin/reth/src/block_ingest.rs | 93 ++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/block_ingest.rs b/bin/reth/src/block_ingest.rs index 5b736c59b..6152c103a 100644 --- a/bin/reth/src/block_ingest.rs +++ b/bin/reth/src/block_ingest.rs @@ -1,7 +1,8 @@ use std::path::PathBuf; use std::sync::Arc; -use alloy_consensus::{BlockBody, BlockHeader}; +use alloy_consensus::{BlockBody, BlockHeader, Transaction}; +use alloy_primitives::TxKind; use alloy_primitives::{Address, PrimitiveSignature, B256, U256}; use alloy_rpc_types::engine::{ ExecutionPayloadEnvelopeV3, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, @@ -19,12 +20,54 @@ use reth_provider::{BlockHashReader, StageCheckpointReader}; use reth_rpc_api::EngineApiClient; use reth_rpc_layer::AuthClientService; use reth_stages::StageId; -use tracing::debug; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info}; +use crate::serialized::TypedTransaction; use crate::serialized::{self, BlockInner}; pub(crate) struct BlockIngest(pub PathBuf); +#[derive(Debug, Clone, Serialize, Deserialize)] +struct EvmContract { + pub address: Address, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SpotToken { + pub index: u64, + #[serde(rename = "evmContract")] + pub evm_contract: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SpotMeta { + tokens: Vec, +} + +async fn fetch_spot_meta(is_testnet: bool) -> Result> { + let url = if is_testnet { + "https://api.hyperliquid-testnet.xyz" + } else { + "https://api.hyperliquid.xyz" + }; + let url = format!("{}/info", url); + // post body: {"type": "spotMeta"} + let client = reqwest::Client::new(); + let response = client.post(url).json(&serde_json::json!({"type": "spotMeta"})).send().await?; + Ok(response.json().await?) +} + +fn to_evm_map(meta: &SpotMeta) -> std::collections::HashMap { + let mut map = std::collections::HashMap::new(); + for token in &meta.tokens { + if let Some(evm_contract) = &token.evm_contract { + map.insert(evm_contract.address, token.index); + } + } + map +} + async fn submit_payload( engine_api_client: &HttpClient>, payload: EthBuiltPayload, @@ -92,6 +135,7 @@ impl BlockIngest { std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis(); let engine_api = node.auth_server_handle().http_client(); + let mut evm_map = to_evm_map(&fetch_spot_meta(node.chain_spec().chain_id() == 998).await?); loop { let Some(original_block) = self.collect_block(height) else { @@ -107,14 +151,47 @@ impl BlockIngest { { let BlockBody { transactions, ommers, withdrawals } = std::mem::take(block.body_mut()); - let signature = PrimitiveSignature::new( - // from anvil - U256::from(0x1), - U256::from(0x1), - true, - ); let mut system_txs = vec![]; for transaction in original_block.system_txs { + let s = match &transaction.tx { + TypedTransaction::Legacy(tx) => match tx.input().len() { + 0 => U256::from(0x1), + _ => { + let TxKind::Call(to) = tx.to else { + panic!("Unexpected contract creation"); + }; + loop { + match evm_map.get(&to).cloned() { + Some(s) => { + break { + let mut addr = [0u8; 32]; + addr[12] = 0x20; + addr[24..32].copy_from_slice(s.to_be_bytes().as_ref()); + U256::from_be_bytes(addr) + } + } + None => { + info!("Contract not found: {:?}, fetching again...", to); + evm_map = to_evm_map( + &fetch_spot_meta( + node.chain_spec().chain_id() == 998, + ) + .await?, + ); + continue; + } + } + } + } + }, + _ => unreachable!(), + }; + let signature = PrimitiveSignature::new( + // from anvil + U256::from(0x1), + s, + true, + ); let typed_transaction = transaction.tx.to_reth(); let tx = TransactionSigned::new( typed_transaction, From 7fe7c065075af076416167e33420933c3345e0b0 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:14:45 +0000 Subject: [PATCH 4/9] perf: Skip state root calculation --- crates/engine/tree/src/tree/root.rs | 33 +++++++++----- crates/stages/stages/src/stages/merkle.rs | 29 ------------ crates/storage/db-common/src/init.rs | 54 ----------------------- 3 files changed, 22 insertions(+), 94 deletions(-) diff --git a/crates/engine/tree/src/tree/root.rs b/crates/engine/tree/src/tree/root.rs index fceaab9c2..0e59b8fbd 100644 --- a/crates/engine/tree/src/tree/root.rs +++ b/crates/engine/tree/src/tree/root.rs @@ -195,6 +195,7 @@ pub enum StateRootMessage { } /// Message about completion of proof calculation for a specific state update +#[allow(dead_code)] #[derive(Debug)] pub struct ProofCalculated { /// The index of this proof in the sequence of state updates @@ -255,7 +256,7 @@ impl ProofSequencer { // return early if we don't have the next expected proof if !self.pending_proofs.contains_key(&self.next_to_deliver) { - return Vec::new() + return Vec::new(); } let mut consecutive_proofs = Vec::with_capacity(self.pending_proofs.len()); @@ -390,7 +391,7 @@ where sequence_number: input.proof_sequence_number, state: input.hashed_state_update, }); - return + return; } if self.inflight >= self.max_concurrent { @@ -480,10 +481,13 @@ where #[metrics(scope = "tree.root")] struct StateRootTaskMetrics { /// Histogram of proof calculation durations. + #[allow(unused)] pub proof_calculation_duration_histogram: Histogram, /// Histogram of proof calculation account targets. + #[allow(unused)] pub proof_calculation_account_targets_histogram: Histogram, /// Histogram of proof calculation storage targets. + #[allow(unused)] pub proof_calculation_storage_targets_histogram: Histogram, /// Histogram of sparse trie update durations. @@ -492,10 +496,13 @@ struct StateRootTaskMetrics { pub sparse_trie_final_update_duration_histogram: Histogram, /// Histogram of state updates received. + #[allow(unused)] pub state_updates_received_histogram: Histogram, /// Histogram of proofs processed. + #[allow(unused)] pub proofs_processed_histogram: Histogram, /// Histogram of state root update iterations. + #[allow(unused)] pub state_root_iterations_histogram: Histogram, /// Histogram of the number of updated state nodes. @@ -531,6 +538,7 @@ pub struct StateRootTask { /// Task configuration. config: StateRootConfig, /// Receiver for state root related messages. + #[allow(unused)] rx: Receiver, /// Sender for state root related messages. tx: Sender, @@ -539,6 +547,7 @@ pub struct StateRootTask { /// Proof sequencing handler. proof_sequencer: ProofSequencer, /// Reference to the shared thread pool for parallel proof generation. + #[allow(unused)] thread_pool: Arc, /// Manages calculation of multiproofs. multiproof_manager: MultiproofManager, @@ -591,12 +600,6 @@ where /// Spawns the state root task and returns a handle to await its result. pub fn spawn(self) -> StateRootHandle { - let sparse_trie_tx = Self::spawn_sparse_trie( - self.thread_pool.clone(), - self.config.clone(), - self.metrics.clone(), - self.tx.clone(), - ); let (tx, rx) = mpsc::sync_channel(1); std::thread::Builder::new() .name("State Root Task".to_string()) @@ -605,8 +608,11 @@ where self.observe_config(); - let result = self.run(sparse_trie_tx); - let _ = tx.send(result); + let _ = tx.send(Ok(StateRootComputeOutcome { + state_root: (B256::default(), Default::default()), + total_time: Duration::default(), + time_from_last_update: Duration::default(), + })); }) .expect("failed to spawn state root thread"); @@ -614,6 +620,7 @@ where } /// Logs and records in metrics the state root config parameters. + #[allow(unused)] fn observe_config(&self) { let nodes_sorted_account_nodes = self.config.nodes_sorted.account_nodes.len(); let nodes_sorted_removed_nodes = self.config.nodes_sorted.removed_nodes.len(); @@ -659,6 +666,7 @@ where } /// Spawn long running sparse trie task that forwards the final result upon completion. + #[allow(unused)] fn spawn_sparse_trie( thread_pool: Arc, config: StateRootConfig, @@ -682,6 +690,7 @@ where } /// Handles request for proof prefetch. + #[allow(unused)] fn on_prefetch_proof(&mut self, targets: MultiProofTargets) { let proof_targets = self.get_prefetch_proof_targets(targets); extend_multi_proof_targets_ref(&mut self.fetched_proof_targets, &proof_targets); @@ -697,6 +706,7 @@ where } /// Calls `get_proof_targets` with existing proof targets for prefetching. + #[allow(unused)] fn get_prefetch_proof_targets(&self, mut targets: MultiProofTargets) -> MultiProofTargets { // Here we want to filter out any targets that are already fetched // @@ -726,7 +736,7 @@ where let Some(fetched_storage) = self.fetched_proof_targets.get(hashed_address) else { // this means the account has not been fetched yet, so we must fetch everything // associated with this account - continue + continue; }; let prev_target_storage_len = target_storage.len(); @@ -749,6 +759,7 @@ where /// Handles state updates. /// /// Returns proof targets derived from the state update. + #[allow(unused)] fn on_state_update( &mut self, source: StateChangeSource, diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index 4880a5ee9..c8222150e 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -279,8 +279,6 @@ where // Reset the checkpoint self.save_execution_checkpoint(provider, None)?; - validate_state_root(trie_root, SealedHeader::seal_slow(target_block), to_block)?; - Ok(ExecOutput { checkpoint: StageCheckpoint::new(to_block) .with_entities_stage_checkpoint(entities_checkpoint), @@ -327,13 +325,6 @@ where let (block_root, updates) = StateRoot::incremental_root_with_updates(tx, range) .map_err(|e| StageError::Fatal(Box::new(e)))?; - // Validate the calculated state root - let target = provider - .header_by_number(input.unwind_to)? - .ok_or_else(|| ProviderError::HeaderNotFound(input.unwind_to.into()))?; - - validate_state_root(block_root, SealedHeader::seal_slow(target), input.unwind_to)?; - // Validation passed, apply unwind changes to the database. provider.write_trie_updates(&updates)?; @@ -344,26 +335,6 @@ where } } -/// Check that the computed state root matches the root in the expected header. -#[inline] -fn validate_state_root( - got: B256, - expected: SealedHeader, - target_block: BlockNumber, -) -> Result<(), StageError> { - if got == expected.state_root() { - Ok(()) - } else { - error!(target: "sync::stages::merkle", ?target_block, ?got, ?expected, "Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); - Err(StageError::Block { - error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff( - GotExpected { got, expected: expected.state_root() }.into(), - )), - block: Box::new(expected.block_with_parent()), - }) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 3027e12e0..8d728090d 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -547,60 +547,6 @@ where Ok(()) } -/// Computes the state root (from scratch) based on the accounts and storages present in the -/// database. -fn compute_state_root(provider: &Provider) -> eyre::Result -where - Provider: DBProvider + TrieWriter, -{ - trace!(target: "reth::cli", "Computing state root"); - - let tx = provider.tx_ref(); - let mut intermediate_state: Option = None; - let mut total_flushed_updates = 0; - - loop { - match StateRootComputer::from_tx(tx) - .with_intermediate_state(intermediate_state) - .root_with_progress()? - { - StateRootProgress::Progress(state, _, updates) => { - let updated_len = provider.write_trie_updates(&updates)?; - total_flushed_updates += updated_len; - - trace!(target: "reth::cli", - last_account_key = %state.last_account_key, - updated_len, - total_flushed_updates, - "Flushing trie updates" - ); - - intermediate_state = Some(*state); - - if total_flushed_updates % SOFT_LIMIT_COUNT_FLUSHED_UPDATES == 0 { - info!(target: "reth::cli", - total_flushed_updates, - "Flushing trie updates" - ); - } - } - StateRootProgress::Complete(root, _, updates) => { - let updated_len = provider.write_trie_updates(&updates)?; - total_flushed_updates += updated_len; - - trace!(target: "reth::cli", - %root, - updated_len, - total_flushed_updates, - "State root has been computed" - ); - - return Ok(root) - } - } - } -} - /// Type to deserialize state root from state dump file. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] struct StateRoot { From 9a6a1a4cc18b32ba0c606f331bc1eda4a80267ee Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:47:59 +0000 Subject: [PATCH 5/9] feat: Support erc20 system tx from address --- crates/ethereum/evm/src/execute.rs | 4 +-- crates/ethereum/primitives/src/transaction.rs | 6 ++-- crates/payload/validator/src/lib.rs | 6 ++-- crates/primitives-traits/src/block/body.rs | 4 +-- .../src/transaction/signed.rs | 30 ++++++++++++------- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 90a59afcc..250422032 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -23,7 +23,7 @@ use reth_execution_types::BlockExecutionResult; use reth_primitives::{ EthPrimitives, Receipt, Recovered, RecoveredBlock, SealedBlock, TransactionSigned, }; -use reth_primitives_traits::{transaction::signed::HL_SYSTEM_TX_FROM_ADDR, NodePrimitives}; +use reth_primitives_traits::{transaction::signed::is_impersonated_tx, NodePrimitives}; use reth_revm::{context_interface::result::ResultAndState, db::State, DatabaseCommit}; /// Factory for [`EthExecutionStrategy`]. @@ -191,7 +191,7 @@ where } let hash = tx.hash(); - let is_system_transaction = tx.signer() == HL_SYSTEM_TX_FROM_ADDR; + let is_system_transaction = is_impersonated_tx(tx.signature(), tx.gas_price()).is_some(); // Execute transaction. let result_and_state = diff --git a/crates/ethereum/primitives/src/transaction.rs b/crates/ethereum/primitives/src/transaction.rs index 9196c6669..bbd71f9de 100644 --- a/crates/ethereum/primitives/src/transaction.rs +++ b/crates/ethereum/primitives/src/transaction.rs @@ -21,7 +21,7 @@ use reth_primitives_traits::{ sync::OnceLock, transaction::{ error::TransactionConversionError, - signed::{is_impersonated_tx, RecoveryError, HL_SYSTEM_TX_FROM_ADDR}, + signed::{is_impersonated_tx, RecoveryError}, }, InMemorySize, SignedTransaction, }; @@ -836,8 +836,8 @@ impl SignedTransaction for TransactionSigned { fn recover_signer(&self) -> Result { let signature = self.signature(); - if is_impersonated_tx(signature, self.gas_price()) { - return Ok(HL_SYSTEM_TX_FROM_ADDR); + if let Some(address) = is_impersonated_tx(signature, self.gas_price()) { + return Ok(address); } let signature_hash = self.signature_hash(); recover_signer(&self.signature, signature_hash) diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index 878f1cf21..ae77c25b1 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -15,7 +15,7 @@ pub mod shanghai; use alloy_rpc_types_engine::{ExecutionData, PayloadError}; use reth_chainspec::EthereumHardforks; use reth_primitives::SealedBlock; -use reth_primitives_traits::transaction::signed::HL_SYSTEM_TX_FROM_ADDR; +use reth_primitives_traits::transaction::signed::is_impersonated_tx; use reth_primitives_traits::{Block, SignedTransaction}; use std::sync::Arc; @@ -94,9 +94,7 @@ impl ExecutionPayloadValidator { let (normal, system) = transactions.into_iter().partition(|tx| { let tx = T::decode_2718(&mut tx.iter().as_slice()); match tx { - Ok(tx) => { - !matches!(tx.recover_signer(), Ok(address) if HL_SYSTEM_TX_FROM_ADDR == address) - } + Ok(tx) => is_impersonated_tx(tx.signature(), tx.gas_price()).is_none(), Err(_) => true, } }); diff --git a/crates/primitives-traits/src/block/body.rs b/crates/primitives-traits/src/block/body.rs index 75131b28c..21af49458 100644 --- a/crates/primitives-traits/src/block/body.rs +++ b/crates/primitives-traits/src/block/body.rs @@ -1,7 +1,7 @@ //! Block body abstraction. use crate::{ - transaction::signed::{RecoveryError, HL_SYSTEM_TX_FROM_ADDR}, + transaction::signed::{is_impersonated_tx, RecoveryError}, BlockHeader, FullSignedTx, InMemorySize, MaybeSerde, MaybeSerdeBincodeCompat, SignedTransaction, }; @@ -85,7 +85,7 @@ pub trait BlockBody: let transactions: Vec = self .transactions() .into_iter() - .filter(|tx| !matches!(tx.recover_signer(), Ok(address) if HL_SYSTEM_TX_FROM_ADDR == address)) + .filter(|&tx| is_impersonated_tx(tx.signature(), tx.gas_price()).is_none()) .cloned() .collect::>(); alloy_consensus::proofs::calculate_transaction_root(transactions.as_slice()) diff --git a/crates/primitives-traits/src/transaction/signed.rs b/crates/primitives-traits/src/transaction/signed.rs index 1399db331..41ae8f7d3 100644 --- a/crates/primitives-traits/src/transaction/signed.rs +++ b/crates/primitives-traits/src/transaction/signed.rs @@ -10,9 +10,10 @@ use alloy_consensus::{ SignableTransaction, Transaction, }; use alloy_eips::eip2718::{Decodable2718, Encodable2718}; -use alloy_primitives::{keccak256, Address, PrimitiveSignature as Signature, TxHash, B256}; +use alloy_primitives::{keccak256, Address, PrimitiveSignature as Signature, TxHash, B256, U160}; use core::hash::Hash; use revm_primitives::{address, U256}; +use std::ops::Add; /// Helper trait that unifies all behaviour required by block to support full node operations. pub trait FullSignedTx: SignedTransaction + MaybeCompact + MaybeSerdeBincodeCompat {} @@ -23,11 +24,20 @@ pub const HL_SYSTEM_TX_FROM_ADDR: Address = address!("22222222222222222222222222 /// Check if the transaction is impersonated. /// Signature part is introduced in block_ingest, while the gas_price is trait of hyperliquid system transactions. -pub fn is_impersonated_tx(signature: &Signature, gas_price: Option) -> bool { - signature.r() == U256::from(1) - && signature.s() == U256::from(1) - && signature.v() == true - && gas_price == Some(0u128) +pub fn is_impersonated_tx(signature: &Signature, gas_price: Option) -> Option
{ + if signature.r() == U256::from(1) && signature.v() == true && gas_price == Some(0u128) { + if signature.s() == U256::from(1) { + Some(HL_SYSTEM_TX_FROM_ADDR) + } else { + let s = signature.s().reduce_mod(U256::from(U160::MAX).add(U256::from(1))); + let s = U160::from(s); + let s: [u8; 20] = s.to_be_bytes(); + let s = Address::from_slice(&s); + Some(s) + } + } else { + None + } } /// A signed transaction. @@ -89,8 +99,8 @@ pub trait SignedTransaction: /// Returns `None` if the transaction's signature is invalid, see also /// `reth_primitives::transaction::recover_signer_unchecked`. fn recover_signer_unchecked(&self) -> Result { - if is_impersonated_tx(self.signature(), self.gas_price()) { - return Ok(HL_SYSTEM_TX_FROM_ADDR); + if let Some(address) = is_impersonated_tx(self.signature(), self.gas_price()) { + return Ok(address); } self.recover_signer_unchecked_with_buf(&mut Vec::new()).map_err(|_| RecoveryError) } @@ -183,8 +193,8 @@ impl SignedTransaction for PooledTransaction { buf: &mut Vec, ) -> Result { let signature = self.signature(); - if is_impersonated_tx(signature, self.gas_price()) { - return Ok(HL_SYSTEM_TX_FROM_ADDR); + if let Some(address) = is_impersonated_tx(signature, self.gas_price()) { + return Ok(address); } match self { Self::Legacy(tx) => tx.tx().encode_for_signing(buf), From 451d5bb3870199b422f986d16052b02795e4354d Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:48:26 +0000 Subject: [PATCH 6/9] feat: Support testnet genesis --- crates/ethereum/cli/src/chainspec.rs | 3 +- crates/ethereum/cli/src/hl_testnet.rs | 113 ++++++++++++++++++++++++++ crates/ethereum/cli/src/lib.rs | 2 + 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 crates/ethereum/cli/src/hl_testnet.rs diff --git a/crates/ethereum/cli/src/chainspec.rs b/crates/ethereum/cli/src/chainspec.rs index 419ff71cb..24151fe2f 100644 --- a/crates/ethereum/cli/src/chainspec.rs +++ b/crates/ethereum/cli/src/chainspec.rs @@ -8,7 +8,7 @@ use reth_primitives::{Header, SealedHeader}; use std::sync::Arc; /// Chains supported by reth. First value should be used as the default. -pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "sepolia", "holesky", "dev"]; +pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "testnet", "sepolia", "holesky", "dev"]; static GENESIS_HASH: B256 = b256!("d8fcc13b6a195b88b7b2da3722ff6cad767b13a8c1e9ffb1c73aa9d216d895f0"); @@ -92,6 +92,7 @@ pub static HL_MAINNET: Lazy> = Lazy::new(|| { pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { Ok(match s { "mainnet" => HL_MAINNET.clone(), + "testnet" => Arc::new(super::hl_testnet::load_hl_testnet()), "sepolia" => SEPOLIA.clone(), "holesky" => HOLESKY.clone(), "dev" => DEV.clone(), diff --git a/crates/ethereum/cli/src/hl_testnet.rs b/crates/ethereum/cli/src/hl_testnet.rs new file mode 100644 index 000000000..4f52ad5b1 --- /dev/null +++ b/crates/ethereum/cli/src/hl_testnet.rs @@ -0,0 +1,113 @@ +use alloy_consensus::Header; +use alloy_genesis::{ChainConfig, Genesis}; +use alloy_primitives::U256; +use alloy_rlp::Decodable; +use reqwest::blocking::get; +use reth_chainspec::{ChainSpec, DEV_HARDFORKS}; +use reth_primitives::SealedHeader; +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{Read, Write}; + +pub(crate) fn load_hl_testnet() -> ChainSpec { + const TESTNET_GENESIS_URL: &str = "https://raw.githubusercontent.com/sprites0/hl-testnet-genesis/main/19386700.rlp"; + + fn download_testnet_genesis() -> Result<&'static str, Box> { + let path = "/tmp/hl_testnet.rmp.lz4"; + println!("Downloading testnet genesis"); + let mut response = get(TESTNET_GENESIS_URL)?; + if let Some(length) = response.content_length() { + // Check if the file exists + if let Ok(metadata) = std::fs::metadata(path) { + if metadata.len() == length { + println!("Already downloaded"); + return Ok(path); + } + } + } + let mut file = File::create(path)?; + let mut downloaded = 0; + let total_size = response.content_length().unwrap_or(0); + let mut buffer = vec![0; 0x100000]; + + loop { + let size = response.read(buffer.as_mut_slice())?; + if size == 0 { + break; + } + file.write_all(&buffer[..size])?; + downloaded += size as u64; + println!( + "Downloaded {} of {} bytes ({}%)", + downloaded, + total_size, + (downloaded as f64 / total_size as f64 * 100.0).round() + ); + } + Ok(path) + } + + let path = download_testnet_genesis().expect("Failed to download testnet genesis"); + let mut file = File::open(path).expect("Failed to open testnet genesis"); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).expect("Failed to read testnet genesis"); + let mut header = Header::decode(&mut &buffer[..]).expect("Failed to decode testnet genesis"); + + let config = ChainConfig { + chain_id: 998, + homestead_block: Some(0), + dao_fork_block: Some(0), + dao_fork_support: false, + eip150_block: Some(0), + eip155_block: Some(0), + eip158_block: Some(0), + byzantium_block: Some(0), + constantinople_block: Some(0), + petersburg_block: Some(0), + istanbul_block: Some(0), + muir_glacier_block: Some(0), + berlin_block: Some(0), + london_block: Some(0), + arrow_glacier_block: Some(0), + gray_glacier_block: Some(0), + merge_netsplit_block: Some(0), + shanghai_time: Some(0), + cancun_time: Some(0), + prague_time: Some(0), + osaka_time: Some(0), + terminal_total_difficulty: Some(U256::ZERO), + terminal_total_difficulty_passed: true, + ethash: None, + clique: None, + parlia: None, + extra_fields: Default::default(), + deposit_contract_address: None, + blob_schedule: Default::default(), + }; + header.number = 0; + let genesis_header = SealedHeader::new(header.clone(), header.hash_slow()); + let genesis = Genesis { + config, + nonce: header.nonce.into(), + timestamp: header.timestamp, + extra_data: header.extra_data, + gas_limit: header.gas_limit, + difficulty: header.difficulty, + mix_hash: header.mix_hash, + coinbase: header.beneficiary, + alloc: BTreeMap::default(), + base_fee_per_gas: header.base_fee_per_gas.map(|x| x.into()), + excess_blob_gas: header.excess_blob_gas, + blob_gas_used: header.blob_gas_used, + number: None, + }; + + ChainSpec { + chain: alloy_chains::Chain::from_id(998), + genesis: genesis.into(), + genesis_header, + hardforks: DEV_HARDFORKS.clone(), + prune_delete_limit: 10000, + ..Default::default() + } +} diff --git a/crates/ethereum/cli/src/lib.rs b/crates/ethereum/cli/src/lib.rs index 7296e7c2d..dcdfcdb03 100644 --- a/crates/ethereum/cli/src/lib.rs +++ b/crates/ethereum/cli/src/lib.rs @@ -11,6 +11,8 @@ /// Chain specification parser. pub mod chainspec; +mod hl_testnet; + #[cfg(test)] mod test { use clap::Parser; From a7a4eee4a76f2b778686a315b1d0491060baaf6c Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:49:02 +0000 Subject: [PATCH 7/9] feat: Support precompile record/replay --- crates/ethereum/evm/src/lib.rs | 172 ++++++++++++++++++++++++++++++++- 1 file changed, 167 insertions(+), 5 deletions(-) diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index ece44b1d7..b5829c1f0 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -19,18 +19,35 @@ extern crate alloc; use alloc::sync::Arc; use alloy_consensus::{BlockHeader, Header}; +use alloy_evm::eth::EthEvmContext; pub use alloy_evm::EthEvm; -use alloy_evm::EthEvmFactory; -use alloy_primitives::U256; +use alloy_primitives::bytes::BufMut; +use alloy_primitives::hex::{FromHex, ToHexExt}; +use alloy_primitives::{Address, Bytes, U256}; use core::{convert::Infallible, fmt::Debug}; use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET}; -use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, NextBlockEnvAttributes}; +use reth_evm::Database; +use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAttributes}; use reth_primitives::TransactionSigned; +use reth_revm::context::result::{EVMError, HaltReason}; +use reth_revm::context::{Block, Cfg, ContextTr}; +use reth_revm::handler::{EthPrecompiles, PrecompileProvider}; +use reth_revm::inspector::NoOpInspector; +use reth_revm::interpreter::interpreter::EthInterpreter; +use reth_revm::interpreter::{Gas, InstructionResult, InterpreterResult}; +use reth_revm::precompile::{ + PrecompileError, PrecompileErrors, PrecompileFn, PrecompileOutput, PrecompileResult, + Precompiles, +}; use reth_revm::{ context::{BlockEnv, CfgEnv, TxEnv}, context_interface::block::BlobExcessGasAndPrice, specification::hardfork::SpecId, }; +use reth_revm::{revm, Context, Inspector, MainBuilder, MainContext}; +use sha2::Digest; +use std::io::Write; +use std::sync::OnceLock; mod config; use alloy_eips::eip1559::INITIAL_BASE_FEE; @@ -49,7 +66,7 @@ pub mod eip6110; #[derive(Debug, Clone)] pub struct EthEvmConfig { chain_spec: Arc, - evm_factory: EthEvmFactory, + evm_factory: HyperliquidEvmFactory, } impl EthEvmConfig { @@ -164,8 +181,153 @@ impl ConfigureEvmEnv for EthEvmConfig { } } +/// A custom precompile that contains static precompiles. +#[allow(missing_debug_implementations)] +#[derive(Clone)] +pub struct L1ReadPrecompiles { + precompiles: EthPrecompiles, + warm_addresses: Vec
, +} + +impl L1ReadPrecompiles { + fn new() -> Self { + let mut this = Self { precompiles: EthPrecompiles::default(), warm_addresses: vec![] }; + this.update_warm_addresses(false); + this + } + + fn update_warm_addresses(&mut self, precompile_enabled: bool) { + self.warm_addresses = if !precompile_enabled { + self.precompiles.warm_addresses().collect() + } else { + self.precompiles + .warm_addresses() + .chain((0..=9).into_iter().map(|x| { + let mut addr = [0u8; 20]; + addr[18] = 0x8; + addr[19] = x; + Address::from_slice(&addr) + })) + .collect() + } + } +} + +impl PrecompileProvider for L1ReadPrecompiles { + type Context = CTX; + type Output = InterpreterResult; + + fn set_spec(&mut self, spec: <::Cfg as Cfg>::Spec) { + self.precompiles.set_spec(spec); + // TODO: How to pass block number and chain id? + self.update_warm_addresses(false); + } + + fn run( + &mut self, + context: &mut Self::Context, + address: &Address, + bytes: &Bytes, + gas_limit: u64, + ) -> Result, revm::precompile::PrecompileErrors> { + if address[..18] == [0u8; 18] { + let maybe_precompile_index = u16::from_be_bytes([address[18], address[19]]); + let precompile_base = + std::env::var("PRECOMPILE_BASE").unwrap_or("/tmp/precompiles".to_string()); + if 0x800 <= maybe_precompile_index && maybe_precompile_index <= 0x809 { + let block_number = context.block().number(); + let input = vec![]; + let mut writer = input.writer(); + writer.write(&address.as_slice()).unwrap(); + writer.write(bytes).unwrap(); + writer.flush().unwrap(); + let hash = sha2::Sha256::digest(writer.get_ref()); + let file = + format!("{}/{}/{}.json", precompile_base, block_number, hash.encode_hex()); + let (output, gas) = match load_result(file) { + Ok(Some(value)) => value, + Ok(None) => { + return Ok(Some(InterpreterResult { + result: InstructionResult::Return, + gas: Gas::new(gas_limit), + output: Bytes::new(), + })) + } + Err(value) => return Err(value), + }; + return Ok(Some(InterpreterResult { + result: InstructionResult::Return, + gas: Gas::new(gas_limit - gas), + output, + })); + } + } + self.precompiles.run(context, address, bytes, gas_limit) + } + + fn contains(&self, address: &Address) -> bool { + self.precompiles.contains(address) + } + + fn warm_addresses(&self) -> Box + '_> { + Box::new(self.warm_addresses.iter().cloned()) + } +} + +fn load_result(file: String) -> Result, PrecompileErrors> { + let Ok(file) = std::fs::File::open(file) else { + return Ok(None); + }; + let reader = std::io::BufReader::new(file); + let json: serde_json::Value = serde_json::from_reader(reader).unwrap(); + let object = json.as_object().unwrap().clone(); + let success = object.get("success").unwrap().as_bool().unwrap(); + if !success { + return Err(PrecompileErrors::Error(PrecompileError::other("Invalid input"))); + } + let output = + Bytes::from_hex(object.get("output").unwrap().as_str().unwrap().to_owned()).unwrap(); + let gas = object.get("gas").unwrap_or(&serde_json::json!(0)).as_u64().unwrap_or_default(); + println!("output: {}, gas: {}", output.encode_hex(), gas); + Ok(Some((output, gas))) +} + +/// Custom EVM configuration. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct HyperliquidEvmFactory; + +impl EvmFactory for HyperliquidEvmFactory { + type Evm, EthInterpreter>> = + EthEvm>>; + type Tx = TxEnv; + type Error = EVMError; + type HaltReason = HaltReason; + type Context = EthEvmContext; + + fn create_evm(&self, db: DB, input: EvmEnv) -> Self::Evm { + let evm = Context::mainnet() + .with_db(db) + .with_cfg(input.cfg_env) + .with_block(input.block_env) + .build_mainnet_with_inspector(NoOpInspector {}) + .with_precompiles(L1ReadPrecompiles::new()); + + EthEvm::new(evm, false) + } + + fn create_evm_with_inspector, EthInterpreter>>( + &self, + db: DB, + input: EvmEnv, + inspector: I, + ) -> Self::Evm { + EthEvm::new(self.create_evm(db, input).into_inner().with_inspector(inspector), true) + } +} + impl ConfigureEvm for EthEvmConfig { - type EvmFactory = EthEvmFactory; + type EvmFactory = HyperliquidEvmFactory; fn evm_factory(&self) -> &Self::EvmFactory { &self.evm_factory From c76ae4e1fb80af5f83287190e9d139d2303d50e3 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:49:28 +0000 Subject: [PATCH 8/9] chore: Update dependencies --- Cargo.lock | 10 ++++++++++ crates/ethereum/cli/Cargo.toml | 8 ++++++++ crates/ethereum/evm/Cargo.toml | 3 +++ 3 files changed, 21 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 1cbcfa7a6..24793a311 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6663,6 +6663,7 @@ dependencies = [ "lz4_flex", "once_cell", "parking_lot", + "reqwest", "reth-basic-payload-builder", "reth-chainspec", "reth-cli", @@ -7631,14 +7632,22 @@ name = "reth-ethereum-cli" version = "1.2.0" dependencies = [ "alloy-chains", + "alloy-consensus", + "alloy-genesis", "alloy-primitives", + "alloy-rlp", "clap", "eyre", + "lz4_flex", "once_cell", + "reqwest", "reth-chainspec", "reth-cli", "reth-cli-commands", "reth-primitives", + "revm", + "rmp-serde", + "serde", "serde_json", ] @@ -7804,6 +7813,7 @@ dependencies = [ "reth-testing-utils", "secp256k1 0.30.0", "serde_json", + "sha2 0.10.8", ] [[package]] diff --git a/crates/ethereum/cli/Cargo.toml b/crates/ethereum/cli/Cargo.toml index 72f1fbc9b..70a1c4648 100644 --- a/crates/ethereum/cli/Cargo.toml +++ b/crates/ethereum/cli/Cargo.toml @@ -21,7 +21,15 @@ eyre.workspace = true once_cell.workspace = true alloy-chains.workspace = true alloy-primitives.workspace = true +alloy-genesis.workspace = true +alloy-consensus.workspace = true +alloy-rlp.workspace = true serde_json.workspace = true +lz4_flex.workspace = true +revm = { workspace = true, features = ["serde"] } +serde.workspace = true +rmp-serde.workspace = true +reqwest = { workspace = true } [dev-dependencies] clap.workspace = true diff --git a/crates/ethereum/evm/Cargo.toml b/crates/ethereum/evm/Cargo.toml index 0838a59f9..4ef4f6282 100644 --- a/crates/ethereum/evm/Cargo.toml +++ b/crates/ethereum/evm/Cargo.toml @@ -29,6 +29,9 @@ alloy-evm.workspace = true alloy-sol-types.workspace = true alloy-consensus.workspace = true +sha2.workspace = true +serde_json.workspace = true + [dev-dependencies] reth-testing-utils.workspace = true reth-evm = { workspace = true, features = ["test-utils"] } From bbed5bc27fc8f01e134d77917f1960f8425c94e8 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:49:47 +0000 Subject: [PATCH 9/9] perf: Skip state root calculation --- crates/engine/tree/src/tree/root.rs | 9 ++------- crates/storage/db-common/src/init.rs | 26 -------------------------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/crates/engine/tree/src/tree/root.rs b/crates/engine/tree/src/tree/root.rs index 0e59b8fbd..e4be54dfa 100644 --- a/crates/engine/tree/src/tree/root.rs +++ b/crates/engine/tree/src/tree/root.rs @@ -587,14 +587,9 @@ where /// Returns a state hook to be used to send state updates to this task. pub fn state_hook(&self) -> impl OnStateHook { - let state_hook = self.state_hook_sender(); + let _state_hook = self.state_hook_sender(); - move |source: StateChangeSource, state: &EvmState| { - if let Err(error) = - state_hook.send(StateRootMessage::StateUpdate(source, state.clone())) - { - error!(target: "engine::root", ?error, "Failed to send state update"); - } + move |_source: StateChangeSource, _state: &EvmState| { } } diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 8d728090d..58a8f960e 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -19,8 +19,6 @@ use reth_provider::{ StateWriter, StaticFileProviderFactory, StorageLocation, TrieWriter, }; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_trie::{IntermediateStateRootState, StateRoot as StateRootComputer, StateRootProgress}; -use reth_trie_db::DatabaseStateRoot; use serde::{Deserialize, Serialize}; use std::io::BufRead; use tracing::{debug, error, info, trace}; @@ -39,9 +37,6 @@ pub const DEFAULT_SOFT_LIMIT_BYTE_LEN_ACCOUNTS_CHUNK: usize = 1_000_000_000; // account) pub const AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP: usize = 285_228; -/// Soft limit for the number of flushed updates after which to log progress summary. -const SOFT_LIMIT_COUNT_FLUSHED_UPDATES: usize = 1_000_000; - /// Storage initialization error type. #[derive(Debug, thiserror::Error, Clone)] pub enum InitStorageError { @@ -415,27 +410,6 @@ where // write state to db dump_state(collector, provider_rw, block)?; - // compute and compare state root. this advances the stage checkpoints. - let computed_state_root = compute_state_root(provider_rw)?; - if computed_state_root == expected_state_root { - info!(target: "reth::cli", - ?computed_state_root, - "Computed state root matches state root in state dump" - ); - } else { - error!(target: "reth::cli", - ?computed_state_root, - ?expected_state_root, - "Computed state root does not match state root in state dump" - ); - - return Err(InitStorageError::StateRootMismatch(GotExpected { - got: computed_state_root, - expected: expected_state_root, - }) - .into()) - } - // insert sync stages for stages that require state for stage in StageId::STATE_REQUIRED { provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(block))?;