Local block sync (#7)

* update: logs * update: more logging * update: rename local ingest dir args * update: fix build * update: directory path * update: logs * update: log ts * update: fetch last block * update: time formatting * update: handle seconds * update: lmore logs * fix: provided args * update: logs * fix: build * update: indefinite wiat * update: run the right loop * update: remove offset * update: scan impl * update: log exists * update: collect s3 blocks * update: change the file * update: logs * fix: deserialization * fix: build * update: remove block * update: add logs * update: logs * update: logs * update: dates * update: ignore older blocks * update: hook up to sync * fix: build * fix: build * update: logs * update: logs * update: start height cond * update: height * update: loggy * update: cond * update: cond * update: cond * update: logs * update: fix height issues * update: logs * only collect s3 * update: log block * update: log both blocks * update; return s3 block * update: use local block * update: blocks * update: remove logs * update: logs * update: remove warns and logs * update: collection log * update: logs * update: logs * update: scan through heights when registering evm * update: add local ingest dir to other evm factory * fix: build * update: add cli cmd * update: remove additional arg * update: change where local ingest dir comes from * fix: receipts * update: deser format * update: fix build * update: logs * update: logs * update: logs * update: logs * update: share precompiles with engine * update: insert compiles * update: change sync dir * update: logs * update: logs * update: logs * update: fix build * update: pipe builder context through * update: untracked * update: pass through context * fix: build * fix: build * update: logs * update: logs * update: logs * update: fix cache passthrough * update: remove logs * update: logs * update: hour rollover * update: zero out hour * update: hour sync * update: cleanup code and speedup sync * update: speedup sync * update: remove logs * update: speed up sync * update: speed up sync * update: ingest in reverse * fix: iter rev * update: break line loop early * update: remove break * update: iteration speed * update: fix build * update: slow down tail ival * update: logs * update: skip last line * update: remove log * update: height * update: logs * update: return logs * update: disable attempt logs * update: tail interval * update: cleanup logs * update: add iter skip * update: fix build * update: skip -1 * fix: skip * fix: build * update: build * fix: build * update: logs * update: log idx * update: skip after enumerate * update: cleanup * update: more cleanup * update: refactor BuilderSharedState to HyperliquidSharedState * update: more cleanup * update: cleanup and refactor collect_local_block * update: error msg * update: readme * update: typo * update: file log * fix: typo build * update: debug log
2025-12-06 10:59:55 +00:00 · 2025-06-26 04:15:58 +10:00
parent 138377fc62
commit 28f6c1e6be
21 changed files with 375 additions and 47 deletions
--- a/crates/node/builder/Cargo.toml
+++ b/crates/node/builder/Cargo.toml
@ -55,6 +55,7 @@ reth-tokio-util.workspace = true
 reth-tracing.workspace = true
 reth-transaction-pool.workspace = true
 reth-basic-payload-builder.workspace = true
+reth-hyperliquid-types.workspace = true

 ## ethereum
 alloy-consensus.workspace = true
--- a/crates/node/builder/src/builder/mod.rs
+++ b/crates/node/builder/src/builder/mod.rs
@ -16,6 +16,7 @@ use reth_cli_util::get_secret_key;
 use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
 use reth_engine_tree::tree::TreeConfig;
 use reth_exex::ExExContext;
+use reth_hyperliquid_types::PrecompilesCache;
 use reth_network::{
    transactions::TransactionsManagerConfig, NetworkBuilder, NetworkConfig, NetworkConfigBuilder,
    NetworkHandle, NetworkManager, NetworkPrimitives,
@ -474,6 +475,14 @@ where
        Self { builder: self.builder.on_rpc_started(hook), task_executor: self.task_executor }
    }

+    /// Add precompiles cache <hyperliquid>
+    pub fn add_precompiles_cache(self, precompile_cache: PrecompilesCache) -> Self {
+        Self {
+            builder: self.builder.add_precompiles_cache(precompile_cache),
+            task_executor: self.task_executor,
+        }
+    }
+
    /// Sets the hook that is run to configure the rpc modules.
    pub fn extend_rpc_modules<F>(self, hook: F) -> Self
    where
@ -587,6 +596,8 @@ pub struct BuilderContext<Node: FullNodeTypes> {
    pub(crate) executor: TaskExecutor,
    /// Config container
    pub(crate) config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
+    /// Shared state
+    pub(crate) shared_state: Option<HyperliquidSharedState>,
 }

 impl<Node: FullNodeTypes> BuilderContext<Node> {
@ -596,8 +607,9 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
        provider: Node::Provider,
        executor: TaskExecutor,
        config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
+        shared_state: Option<HyperliquidSharedState>,
    ) -> Self {
-        Self { head, provider, executor, config_container }
+        Self { head, provider, executor, config_container, shared_state }
    }

    /// Returns the configured provider to interact with the blockchain.
@ -754,6 +766,14 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
    pub fn ingest_dir(&self) -> PathBuf {
        self.config().ingest_dir.clone().expect("ingest dir not set")
    }
+
+    pub fn local_ingest_dir(&self) -> PathBuf {
+        self.config().local_ingest_dir.clone().expect("local ingest dir not set")
+    }
+
+    pub fn shared_state(&self) -> Option<HyperliquidSharedState> {
+        self.shared_state.clone()
+    }
 }

 impl<Node: FullNodeTypes<Types: NodeTypes<ChainSpec: Hardforks>>> BuilderContext<Node> {
--- a/crates/node/builder/src/builder/states.rs
+++ b/crates/node/builder/src/builder/states.rs
@ -13,6 +13,7 @@ use crate::{
    AddOns, FullNode,
 };
 use reth_exex::ExExContext;
+use reth_hyperliquid_types::PrecompilesCache;
 use reth_node_api::{FullNodeComponents, FullNodeTypes, NodeAddOns, NodeTypes};
 use reth_node_core::node_config::NodeConfig;
 use reth_tasks::TaskExecutor;
@ -47,6 +48,7 @@ impl<T: FullNodeTypes> NodeBuilderWithTypes<T> {
            adapter,
            components_builder,
            add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons: () },
+            shared_state: None,
        }
    }
 }
@ -146,6 +148,11 @@ impl<T: FullNodeTypes, C: NodeComponents<T>> Clone for NodeAdapter<T, C> {
    }
 }

+#[derive(Debug, Clone)]
+pub struct HyperliquidSharedState {
+    pub precompiles_cache: PrecompilesCache,
+}
+
 /// A fully type configured node builder.
 ///
 /// Supports adding additional addons to the node.
@ -162,6 +169,8 @@ pub struct NodeBuilderWithComponents<
    pub components_builder: CB,
    /// Additional node extensions.
    pub add_ons: AddOns<NodeAdapter<T, CB::Components>, AO>,
+    /// Shared state
+    pub shared_state: Option<HyperliquidSharedState>,
 }

 impl<T, CB> NodeBuilderWithComponents<T, CB, ()>
@ -182,6 +191,7 @@ where
            adapter,
            components_builder,
            add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons },
+            shared_state: None,
        }
    }
 }
@ -294,4 +304,10 @@ where
            add_ons
        })
    }
+
+    /// Add state
+    pub fn add_precompiles_cache(mut self, precompiles_cache: PrecompilesCache) -> Self {
+        self.shared_state = Some(HyperliquidSharedState { precompiles_cache });
+        self
+    }
 }
--- a/crates/node/builder/src/launch/common.rs
+++ b/crates/node/builder/src/launch/common.rs
@ -5,7 +5,7 @@ use std::{sync::Arc, thread::available_parallelism};
 use crate::{
    components::{NodeComponents, NodeComponentsBuilder},
    hooks::OnComponentInitializedHook,
-    BuilderContext, NodeAdapter,
+    BuilderContext, HyperliquidSharedState, NodeAdapter,
 };
 use alloy_primitives::{BlockNumber, B256};
 use eyre::{Context, OptionExt};
@ -258,6 +258,7 @@ impl<L, R> LaunchContextWith<Attached<L, R>> {
        &mut self.attachment.right
    }
 }
+
 impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpec>, R>> {
    /// Adjust certain settings in the config to make sure they are set correctly
    ///
@ -655,6 +656,7 @@ where
        on_component_initialized: Box<
            dyn OnComponentInitializedHook<NodeAdapter<T, CB::Components>>,
        >,
+        shared_state: Option<HyperliquidSharedState>,
    ) -> eyre::Result<
        LaunchContextWith<
            Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, WithComponents<T, CB>>,
@ -671,6 +673,7 @@ where
            self.blockchain_db().clone(),
            self.task_executor().clone(),
            self.configs().clone(),
+            shared_state,
        );

        debug!(target: "reth::cli", "creating components");
@ -790,15 +793,15 @@ where
    /// This checks for OP-Mainnet and ensures we have all the necessary data to progress (past
    /// bedrock height)
    fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> {
-        if self.chain_spec().is_optimism() &&
-            !self.is_dev() &&
-            self.chain_id() == Chain::optimism_mainnet()
+        if self.chain_spec().is_optimism()
+            && !self.is_dev()
+            && self.chain_id() == Chain::optimism_mainnet()
        {
            let latest = self.blockchain_db().last_block_number()?;
            // bedrock height
            if latest < 105235063 {
                error!("Op-mainnet has been launched without importing the pre-Bedrock state. The chain can't progress without this. See also https://reth.rs/run/sync-op-mainnet.html?minimal-bootstrap-recommended");
-                return Err(ProviderError::BestBlockNotFound)
+                return Err(ProviderError::BestBlockNotFound);
            }
        }

@ -880,7 +883,7 @@ where
        &self,
    ) -> eyre::Result<Box<dyn InvalidBlockHook<<T::Types as NodeTypes>::Primitives>>> {
        let Some(ref hook) = self.node_config().debug.invalid_block_hook else {
-            return Ok(Box::new(NoopInvalidBlockHook::default()))
+            return Ok(Box::new(NoopInvalidBlockHook::default()));
        };
        let healthy_node_rpc_client = self.get_healthy_node_client()?;

--- a/crates/node/builder/src/launch/engine.rs
+++ b/crates/node/builder/src/launch/engine.rs
@ -93,7 +93,9 @@ where
            components_builder,
            add_ons: AddOns { hooks, exexs: installed_exex, add_ons },
            config,
+            shared_state,
        } = target;
+
        let NodeHooks { on_component_initialized, on_node_started, .. } = hooks;

        // setup the launch context
@ -126,7 +128,7 @@ where
            .with_blockchain_db::<T, _>(move |provider_factory| {
                Ok(BlockchainProvider::new(provider_factory)?)
            })?
-            .with_components(components_builder, on_component_initialized).await?;
+            .with_components(components_builder, on_component_initialized, shared_state.clone()).await?;

        // spawn exexs
        let exex_manager_handle = ExExLauncher::new(