Local block sync (#7)

* update: logs * update: more logging * update: rename local ingest dir args * update: fix build * update: directory path * update: logs * update: log ts * update: fetch last block * update: time formatting * update: handle seconds * update: lmore logs * fix: provided args * update: logs * fix: build * update: indefinite wiat * update: run the right loop * update: remove offset * update: scan impl * update: log exists * update: collect s3 blocks * update: change the file * update: logs * fix: deserialization * fix: build * update: remove block * update: add logs * update: logs * update: logs * update: dates * update: ignore older blocks * update: hook up to sync * fix: build * fix: build * update: logs * update: logs * update: start height cond * update: height * update: loggy * update: cond * update: cond * update: cond * update: logs * update: fix height issues * update: logs * only collect s3 * update: log block * update: log both blocks * update; return s3 block * update: use local block * update: blocks * update: remove logs * update: logs * update: remove warns and logs * update: collection log * update: logs * update: logs * update: scan through heights when registering evm * update: add local ingest dir to other evm factory * fix: build * update: add cli cmd * update: remove additional arg * update: change where local ingest dir comes from * fix: receipts * update: deser format * update: fix build * update: logs * update: logs * update: logs * update: logs * update: share precompiles with engine * update: insert compiles * update: change sync dir * update: logs * update: logs * update: logs * update: fix build * update: pipe builder context through * update: untracked * update: pass through context * fix: build * fix: build * update: logs * update: logs * update: logs * update: fix cache passthrough * update: remove logs * update: logs * update: hour rollover * update: zero out hour * update: hour sync * update: cleanup code and speedup sync * update: speedup sync * update: remove logs * update: speed up sync * update: speed up sync * update: ingest in reverse * fix: iter rev * update: break line loop early * update: remove break * update: iteration speed * update: fix build * update: slow down tail ival * update: logs * update: skip last line * update: remove log * update: height * update: logs * update: return logs * update: disable attempt logs * update: tail interval * update: cleanup logs * update: add iter skip * update: fix build * update: skip -1 * fix: skip * fix: build * update: build * fix: build * update: logs * update: log idx * update: skip after enumerate * update: cleanup * update: more cleanup * update: refactor BuilderSharedState to HyperliquidSharedState * update: more cleanup * update: cleanup and refactor collect_local_block * update: error msg * update: readme * update: typo * update: file log * fix: typo build * update: debug log
2025-12-06 10:59:55 +00:00 · 2025-06-26 04:15:58 +10:00
parent 138377fc62
commit 28f6c1e6be
21 changed files with 375 additions and 47 deletions
--- a/crates/ethereum/consensus/src/lib.rs
+++ b/crates/ethereum/consensus/src/lib.rs
@ -225,6 +225,7 @@ where
 #[cfg(test)]
 mod tests {
    use super::*;
+    use alloy_eips::eip1559::GAS_LIMIT_BOUND_DIVISOR;
    use alloy_primitives::B256;
    use reth_chainspec::{ChainSpec, ChainSpecBuilder};
    use reth_primitives_traits::proofs;
--- a/crates/ethereum/evm/Cargo.toml
+++ b/crates/ethereum/evm/Cargo.toml
@ -18,6 +18,7 @@ reth-ethereum-forks.workspace = true
 reth-revm.workspace = true
 reth-evm.workspace = true
 reth-primitives.workspace = true
+reth-node-builder.workspace = true

 # Ethereum
 reth-primitives-traits.workspace = true
--- a/crates/ethereum/evm/src/lib.rs
+++ b/crates/ethereum/evm/src/lib.rs
@ -21,24 +21,21 @@ use alloc::sync::Arc;
 use alloy_consensus::{BlockHeader, Header};
 use alloy_evm::eth::EthEvmContext;
 pub use alloy_evm::EthEvm;
-use alloy_primitives::bytes::BufMut;
-use alloy_primitives::hex::{FromHex, ToHexExt};
-use alloy_primitives::{Address, B256};
-use alloy_primitives::{Bytes, U256};
+use alloy_primitives::Address;
+use alloy_primitives::U256;
 use core::{convert::Infallible, fmt::Debug};
 use parking_lot::RwLock;
 use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
 use reth_evm::Database;
 use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAttributes};
-use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
+use reth_hyperliquid_types::{PrecompilesCache, ReadPrecompileInput, ReadPrecompileResult};
+use reth_node_builder::HyperliquidSharedState;
+use reth_primitives::SealedBlock;
 use reth_primitives::TransactionSigned;
-use reth_primitives::{SealedBlock, Transaction};
 use reth_revm::context::result::{EVMError, HaltReason};
-use reth_revm::context::Cfg;
 use reth_revm::handler::EthPrecompiles;
 use reth_revm::inspector::NoOpInspector;
 use reth_revm::interpreter::interpreter::EthInterpreter;
-use reth_revm::precompile::{PrecompileError, PrecompileErrors, Precompiles};
 use reth_revm::MainBuilder;
 use reth_revm::{
    context::{BlockEnv, CfgEnv, TxEnv},
@ -48,7 +45,6 @@ use reth_revm::{
 use reth_revm::{Context, Inspector, MainContext};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use std::io::Write;
 use std::path::PathBuf;

 mod config;
@ -72,12 +68,13 @@ pub struct EthEvmConfig {
    chain_spec: Arc<ChainSpec>,
    evm_factory: HyperliquidEvmFactory,
    ingest_dir: Option<PathBuf>,
+    shared_state: Option<HyperliquidSharedState>,
 }

 impl EthEvmConfig {
    /// Creates a new Ethereum EVM configuration with the given chain spec.
    pub fn new(chain_spec: Arc<ChainSpec>) -> Self {
-        Self { chain_spec, ingest_dir: None, evm_factory: Default::default() }
+        Self { chain_spec, ingest_dir: None, evm_factory: Default::default(), shared_state: None }
    }

    pub fn with_ingest_dir(mut self, ingest_dir: PathBuf) -> Self {
@ -86,6 +83,12 @@ impl EthEvmConfig {
        self
    }

+    pub fn with_shared_state(mut self, shared_state: Option<HyperliquidSharedState>) -> Self {
+        self.shared_state = shared_state.clone();
+        self.evm_factory.shared_state = shared_state;
+        self
+    }
+
    /// Creates a new Ethereum EVM configuration for the ethereum mainnet.
    pub fn mainnet() -> Self {
        Self::new(MAINNET.clone())
@ -208,9 +211,10 @@ pub(crate) enum EvmBlock {
 #[non_exhaustive]
 pub struct HyperliquidEvmFactory {
    ingest_dir: Option<PathBuf>,
+    shared_state: Option<HyperliquidSharedState>,
 }

-pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
+pub(crate) fn collect_s3_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
    let f = ((height - 1) / 1_000_000) * 1_000_000;
    let s = ((height - 1) / 1_000) * 1_000;
    let path = format!("{}/{f}/{s}/{height}.rmp.lz4", ingest_path.to_string_lossy());
@ -225,6 +229,32 @@ pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAn
    }
 }

+pub(crate) fn get_locally_sourced_precompiles_for_height(
+    precompiles_cache: PrecompilesCache,
+    height: u64,
+) -> Option<Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>> {
+    let mut u_cache = precompiles_cache.lock();
+    u_cache.remove(&height)
+}
+
+pub(crate) fn collect_block(
+    ingest_path: PathBuf,
+    shared_state: Option<HyperliquidSharedState>,
+    height: u64,
+) -> Option<BlockAndReceipts> {
+    // Attempt to source precompile from the cache that is shared the binary level with the block
+    // ingestor.
+    if let Some(shared_state) = shared_state {
+        if let Some(calls) =
+            get_locally_sourced_precompiles_for_height(shared_state.precompiles_cache, height)
+        {
+            return Some(BlockAndReceipts { read_precompile_calls: calls });
+        }
+    }
+    // Fallback to s3 always
+    collect_s3_block(ingest_path, height)
+}
+
 impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
    type Evm<DB: Database, I: Inspector<EthEvmContext<DB>, EthInterpreter>> =
        EthEvm<DB, I, ReplayPrecompile<EthEvmContext<DB>>>;
@ -234,9 +264,14 @@ impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
    type Context<DB: Database> = EthEvmContext<DB>;

    fn create_evm<DB: Database>(&self, db: DB, input: EvmEnv) -> Self::Evm<DB, NoOpInspector> {
-        let cache = collect_block(self.ingest_dir.clone().unwrap(), input.block_env.number)
-            .unwrap()
-            .read_precompile_calls;
+        let block = collect_block(
+            self.ingest_dir.clone().unwrap(),
+            self.shared_state.clone(),
+            input.block_env.number,
+        )
+        .expect("Failed to collect a submitted block. If sourcing locally, make sure your local hl-node is producing blocks.");
+        let cache = block.read_precompile_calls;
+
        let evm = Context::mainnet()
            .with_db(db)
            .with_cfg(input.cfg_env)
--- a/crates/ethereum/node/src/node.rs
+++ b/crates/ethereum/node/src/node.rs
@ -248,7 +248,9 @@ where
        ctx: &BuilderContext<Node>,
    ) -> eyre::Result<(Self::EVM, Self::Executor)> {
        let chain_spec = ctx.chain_spec();
-        let evm_config = EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir());
+        let evm_config = EthEvmConfig::new(ctx.chain_spec())
+            .with_ingest_dir(ctx.ingest_dir())
+            .with_shared_state(ctx.shared_state());
        let strategy_factory = EthExecutionStrategyFactory::new(chain_spec, evm_config.clone());
        let executor = BasicBlockExecutorProvider::new(strategy_factory);

--- a/crates/ethereum/node/src/payload.rs
+++ b/crates/ethereum/node/src/payload.rs
@ -74,6 +74,12 @@ where
        ctx: &BuilderContext<Node>,
        pool: Pool,
    ) -> eyre::Result<Self::PayloadBuilder> {
-        self.build(EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir()), ctx, pool)
+        self.build(
+            EthEvmConfig::new(ctx.chain_spec())
+                .with_ingest_dir(ctx.ingest_dir())
+                .with_shared_state(ctx.shared_state()),
+            ctx,
+            pool,
+        )
    }
 }