jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Local block sync (#7)

Browse Source 
* update: logs

* update: more logging

* update: rename local ingest dir args

* update: fix build

* update: directory path

* update: logs

* update: log ts

* update: fetch last block

* update: time formatting

* update: handle seconds

* update: lmore logs

* fix: provided args

* update: logs

* fix: build

* update: indefinite wiat

* update: run the right loop

* update: remove offset

* update: scan impl

* update: log exists

* update: collect s3 blocks

* update: change the file

* update: logs

* fix: deserialization

* fix: build

* update: remove block

* update: add logs

* update: logs

* update: logs

* update: dates

* update: ignore older blocks

* update: hook up to sync

* fix: build

* fix: build

* update: logs

* update: logs

* update: start height cond

* update: height

* update: loggy

* update: cond

* update: cond

* update: cond

* update: logs

* update: fix height issues

* update: logs

* only collect s3

* update: log block

* update: log both blocks

* update; return s3 block

* update: use local block

* update: blocks

* update: remove logs

* update: logs

* update: remove warns and logs

* update: collection log

* update: logs

* update: logs

* update: scan through heights when registering evm

* update: add local ingest dir to other evm factory

* fix: build

* update: add cli cmd

* update: remove additional arg

* update: change where local ingest dir comes from

* fix: receipts

* update: deser format

* update: fix build

* update: logs

* update: logs

* update: logs

* update: logs

* update: share precompiles with engine

* update: insert compiles

* update: change sync dir

* update: logs

* update: logs

* update: logs

* update: fix build

* update: pipe builder context through

* update: untracked

* update: pass through context

* fix: build

* fix: build

* update: logs

* update: logs

* update: logs

* update: fix cache passthrough

* update: remove logs

* update: logs

* update: hour rollover

* update: zero out hour

* update: hour sync

* update: cleanup code and speedup sync

* update: speedup sync

* update: remove logs

* update: speed up sync

* update: speed up sync

* update: ingest in reverse

* fix: iter rev

* update: break line loop early

* update: remove break

* update: iteration speed

* update: fix build

* update: slow down tail ival

* update: logs

* update: skip last line

* update: remove log

* update: height

* update: logs

* update: return logs

* update: disable attempt logs

* update: tail interval

* update: cleanup logs

* update: add iter skip

* update: fix build

* update: skip -1

* fix: skip

* fix: build

* update: build

* fix: build

* update: logs

* update: log idx

* update: skip after enumerate

* update: cleanup

* update: more cleanup

* update: refactor BuilderSharedState to HyperliquidSharedState

* update: more cleanup

* update: cleanup and refactor collect_local_block

* update: error msg

* update: readme

* update: typo

* update: file log

* fix: typo build

* update: debug log
This commit is contained in:
arb00r
2025-06-26 04:15:58 +10:00
committed by GitHub
parent 138377fc62
commit 28f6c1e6be
21 changed files with 375 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/cli/commands/src/node.rs
View File

@ -118,6 +118,10 @@ pub struct NodeCommand<
/// EVM blocks base directory
#[arg(long, default_value = "/tmp/evm-blocks")]
pub ingest_dir: PathBuf,
/// Local EVM blocks base directory
#[arg(long, default_value = "/tmp/evm-blocks-and-receipts")]
pub local_ingest_dir: PathBuf,
}
impl<C: ChainSpecParser> NodeCommand<C> {
@ -170,6 +174,7 @@ impl<
ext,
engine,
ingest_dir,
local_ingest_dir,
} = self;
// set up node config
@ -189,6 +194,7 @@ impl<
pruning,
engine,
ingest_dir: Some(ingest_dir),
local_ingest_dir: Some(local_ingest_dir),
};
let data_dir = node_config.datadir();

1
crates/ethereum/consensus/src/lib.rs
View File

@ -225,6 +225,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
use alloy_eips::eip1559::GAS_LIMIT_BOUND_DIVISOR;
use alloy_primitives::B256;
use reth_chainspec::{ChainSpec, ChainSpecBuilder};
use reth_primitives_traits::proofs;

1
crates/ethereum/evm/Cargo.toml
View File

@ -18,6 +18,7 @@ reth-ethereum-forks.workspace = true
reth-revm.workspace = true
reth-evm.workspace = true
reth-primitives.workspace = true
reth-node-builder.workspace = true
# Ethereum
reth-primitives-traits.workspace = true

63
crates/ethereum/evm/src/lib.rs
View File

@ -21,24 +21,21 @@ use alloc::sync::Arc;
use alloy_consensus::{BlockHeader, Header};
use alloy_evm::eth::EthEvmContext;
pub use alloy_evm::EthEvm;
use alloy_primitives::bytes::BufMut;
use alloy_primitives::hex::{FromHex, ToHexExt};
use alloy_primitives::{Address, B256};
use alloy_primitives::{Bytes, U256};
use alloy_primitives::Address;
use alloy_primitives::U256;
use core::{convert::Infallible, fmt::Debug};
use parking_lot::RwLock;
use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
use reth_evm::Database;
use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAttributes};
use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
use reth_hyperliquid_types::{PrecompilesCache, ReadPrecompileInput, ReadPrecompileResult};
use reth_node_builder::HyperliquidSharedState;
use reth_primitives::SealedBlock;
use reth_primitives::TransactionSigned;
use reth_primitives::{SealedBlock, Transaction};
use reth_revm::context::result::{EVMError, HaltReason};
use reth_revm::context::Cfg;
use reth_revm::handler::EthPrecompiles;
use reth_revm::inspector::NoOpInspector;
use reth_revm::interpreter::interpreter::EthInterpreter;
use reth_revm::precompile::{PrecompileError, PrecompileErrors, Precompiles};
use reth_revm::MainBuilder;
use reth_revm::{
context::{BlockEnv, CfgEnv, TxEnv},
@ -48,7 +45,6 @@ use reth_revm::{
use reth_revm::{Context, Inspector, MainContext};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Write;
use std::path::PathBuf;
mod config;
@ -72,12 +68,13 @@ pub struct EthEvmConfig {
chain_spec: Arc<ChainSpec>,
evm_factory: HyperliquidEvmFactory,
ingest_dir: Option<PathBuf>,
shared_state: Option<HyperliquidSharedState>,
}
impl EthEvmConfig {
/// Creates a new Ethereum EVM configuration with the given chain spec.
pub fn new(chain_spec: Arc<ChainSpec>) -> Self {
Self { chain_spec, ingest_dir: None, evm_factory: Default::default() }
Self { chain_spec, ingest_dir: None, evm_factory: Default::default(), shared_state: None }
}
pub fn with_ingest_dir(mut self, ingest_dir: PathBuf) -> Self {
@ -86,6 +83,12 @@ impl EthEvmConfig {
self
}
pub fn with_shared_state(mut self, shared_state: Option<HyperliquidSharedState>) -> Self {
self.shared_state = shared_state.clone();
self.evm_factory.shared_state = shared_state;
self
}
/// Creates a new Ethereum EVM configuration for the ethereum mainnet.
pub fn mainnet() -> Self {
Self::new(MAINNET.clone())
@ -208,9 +211,10 @@ pub(crate) enum EvmBlock {
#[non_exhaustive]
pub struct HyperliquidEvmFactory {
ingest_dir: Option<PathBuf>,
shared_state: Option<HyperliquidSharedState>,
}
pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
pub(crate) fn collect_s3_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
let f = ((height - 1) / 1_000_000) * 1_000_000;
let s = ((height - 1) / 1_000) * 1_000;
let path = format!("{}/{f}/{s}/{height}.rmp.lz4", ingest_path.to_string_lossy());
@ -225,6 +229,32 @@ pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAn
}
}
pub(crate) fn get_locally_sourced_precompiles_for_height(
precompiles_cache: PrecompilesCache,
height: u64,
) -> Option<Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>> {
let mut u_cache = precompiles_cache.lock();
u_cache.remove(&height)
}
pub(crate) fn collect_block(
ingest_path: PathBuf,
shared_state: Option<HyperliquidSharedState>,
height: u64,
) -> Option<BlockAndReceipts> {
// Attempt to source precompile from the cache that is shared the binary level with the block
// ingestor.
if let Some(shared_state) = shared_state {
if let Some(calls) =
get_locally_sourced_precompiles_for_height(shared_state.precompiles_cache, height)
{
return Some(BlockAndReceipts { read_precompile_calls: calls });
}
}
// Fallback to s3 always
collect_s3_block(ingest_path, height)
}
impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
type Evm<DB: Database, I: Inspector<EthEvmContext<DB>, EthInterpreter>> =
EthEvm<DB, I, ReplayPrecompile<EthEvmContext<DB>>>;
@ -234,9 +264,14 @@ impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
type Context<DB: Database> = EthEvmContext<DB>;
fn create_evm<DB: Database>(&self, db: DB, input: EvmEnv) -> Self::Evm<DB, NoOpInspector> {
let cache = collect_block(self.ingest_dir.clone().unwrap(), input.block_env.number)
.unwrap()
.read_precompile_calls;
let block = collect_block(
self.ingest_dir.clone().unwrap(),
self.shared_state.clone(),
input.block_env.number,
)
.expect("Failed to collect a submitted block. If sourcing locally, make sure your local hl-node is producing blocks.");
let cache = block.read_precompile_calls;
let evm = Context::mainnet()
.with_db(db)
.with_cfg(input.cfg_env)

4
crates/ethereum/node/src/node.rs
View File

@ -248,7 +248,9 @@ where
ctx: &BuilderContext<Node>,
) -> eyre::Result<(Self::EVM, Self::Executor)> {
let chain_spec = ctx.chain_spec();
let evm_config = EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir());
let evm_config = EthEvmConfig::new(ctx.chain_spec())
.with_ingest_dir(ctx.ingest_dir())
.with_shared_state(ctx.shared_state());
let strategy_factory = EthExecutionStrategyFactory::new(chain_spec, evm_config.clone());
let executor = BasicBlockExecutorProvider::new(strategy_factory);

8
crates/ethereum/node/src/payload.rs
View File

@ -74,6 +74,12 @@ where
ctx: &BuilderContext<Node>,
pool: Pool,
) -> eyre::Result<Self::PayloadBuilder> {
self.build(EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir()), ctx, pool)
self.build(
EthEvmConfig::new(ctx.chain_spec())
.with_ingest_dir(ctx.ingest_dir())
.with_shared_state(ctx.shared_state()),
ctx,
pool,
)
}
}

2
crates/hyperliquid-types/Cargo.toml
View File

@ -13,6 +13,8 @@ workspace = true
[dependencies]
alloy-primitives.workspace = true
serde.workspace = true
tokio = { workspace = true, features = ["sync", "macros", "time", "rt-multi-thread"] }
parking_lot.workspace = true
[dev-dependencies]
clap.workspace = true

8
crates/hyperliquid-types/src/lib.rs
View File

@ -1,4 +1,7 @@
use alloy_primitives::Bytes;
use std::{collections::BTreeMap, sync::Arc};
use alloy_primitives::{Address, Bytes};
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)]
@ -14,3 +17,6 @@ pub enum ReadPrecompileResult {
Error,
UnexpectedError,
}
pub type PrecompilesCache =
Arc<Mutex<BTreeMap<u64, Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>>>>;

1
crates/node/builder/Cargo.toml
View File

@ -55,6 +55,7 @@ reth-tokio-util.workspace = true
reth-tracing.workspace = true
reth-transaction-pool.workspace = true
reth-basic-payload-builder.workspace = true
reth-hyperliquid-types.workspace = true
## ethereum
alloy-consensus.workspace = true

22
crates/node/builder/src/builder/mod.rs
View File

@ -16,6 +16,7 @@ use reth_cli_util::get_secret_key;
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
use reth_engine_tree::tree::TreeConfig;
use reth_exex::ExExContext;
use reth_hyperliquid_types::PrecompilesCache;
use reth_network::{
transactions::TransactionsManagerConfig, NetworkBuilder, NetworkConfig, NetworkConfigBuilder,
NetworkHandle, NetworkManager, NetworkPrimitives,
@ -474,6 +475,14 @@ where
Self { builder: self.builder.on_rpc_started(hook), task_executor: self.task_executor }
}
/// Add precompiles cache <hyperliquid>
pub fn add_precompiles_cache(self, precompile_cache: PrecompilesCache) -> Self {
Self {
builder: self.builder.add_precompiles_cache(precompile_cache),
task_executor: self.task_executor,
}
}
/// Sets the hook that is run to configure the rpc modules.
pub fn extend_rpc_modules<F>(self, hook: F) -> Self
where
@ -587,6 +596,8 @@ pub struct BuilderContext<Node: FullNodeTypes> {
pub(crate) executor: TaskExecutor,
/// Config container
pub(crate) config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
/// Shared state
pub(crate) shared_state: Option<HyperliquidSharedState>,
}
impl<Node: FullNodeTypes> BuilderContext<Node> {
@ -596,8 +607,9 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
provider: Node::Provider,
executor: TaskExecutor,
config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
shared_state: Option<HyperliquidSharedState>,
) -> Self {
Self { head, provider, executor, config_container }
Self { head, provider, executor, config_container, shared_state }
}
/// Returns the configured provider to interact with the blockchain.
@ -754,6 +766,14 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
pub fn ingest_dir(&self) -> PathBuf {
self.config().ingest_dir.clone().expect("ingest dir not set")
}
pub fn local_ingest_dir(&self) -> PathBuf {
self.config().local_ingest_dir.clone().expect("local ingest dir not set")
}
pub fn shared_state(&self) -> Option<HyperliquidSharedState> {
self.shared_state.clone()
}
}
impl<Node: FullNodeTypes<Types: NodeTypes<ChainSpec: Hardforks>>> BuilderContext<Node> {

16
crates/node/builder/src/builder/states.rs
View File

@ -13,6 +13,7 @@ use crate::{
AddOns, FullNode,
};
use reth_exex::ExExContext;
use reth_hyperliquid_types::PrecompilesCache;
use reth_node_api::{FullNodeComponents, FullNodeTypes, NodeAddOns, NodeTypes};
use reth_node_core::node_config::NodeConfig;
use reth_tasks::TaskExecutor;
@ -47,6 +48,7 @@ impl<T: FullNodeTypes> NodeBuilderWithTypes<T> {
adapter,
components_builder,
add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons: () },
shared_state: None,
}
}
}
@ -146,6 +148,11 @@ impl<T: FullNodeTypes, C: NodeComponents<T>> Clone for NodeAdapter<T, C> {
}
}
#[derive(Debug, Clone)]
pub struct HyperliquidSharedState {
pub precompiles_cache: PrecompilesCache,
}
/// A fully type configured node builder.
///
/// Supports adding additional addons to the node.
@ -162,6 +169,8 @@ pub struct NodeBuilderWithComponents<
pub components_builder: CB,
/// Additional node extensions.
pub add_ons: AddOns<NodeAdapter<T, CB::Components>, AO>,
/// Shared state
pub shared_state: Option<HyperliquidSharedState>,
}
impl<T, CB> NodeBuilderWithComponents<T, CB, ()>
@ -182,6 +191,7 @@ where
adapter,
components_builder,
add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons },
shared_state: None,
}
}
}
@ -294,4 +304,10 @@ where
add_ons
})
}
/// Add state
pub fn add_precompiles_cache(mut self, precompiles_cache: PrecompilesCache) -> Self {
self.shared_state = Some(HyperliquidSharedState { precompiles_cache });
self
}
}

15
crates/node/builder/src/launch/common.rs
View File

@ -5,7 +5,7 @@ use std::{sync::Arc, thread::available_parallelism};
use crate::{
components::{NodeComponents, NodeComponentsBuilder},
hooks::OnComponentInitializedHook,
BuilderContext, NodeAdapter,
BuilderContext, HyperliquidSharedState, NodeAdapter,
};
use alloy_primitives::{BlockNumber, B256};
use eyre::{Context, OptionExt};
@ -258,6 +258,7 @@ impl<L, R> LaunchContextWith<Attached<L, R>> {
&mut self.attachment.right
}
}
impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpec>, R>> {
/// Adjust certain settings in the config to make sure they are set correctly
///
@ -655,6 +656,7 @@ where
on_component_initialized: Box<
dyn OnComponentInitializedHook<NodeAdapter<T, CB::Components>>,
>,
shared_state: Option<HyperliquidSharedState>,
) -> eyre::Result<
LaunchContextWith<
Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, WithComponents<T, CB>>,
@ -671,6 +673,7 @@ where
self.blockchain_db().clone(),
self.task_executor().clone(),
self.configs().clone(),
shared_state,
);
debug!(target: "reth::cli", "creating components");
@ -790,15 +793,15 @@ where
/// This checks for OP-Mainnet and ensures we have all the necessary data to progress (past
/// bedrock height)
fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> {
if self.chain_spec().is_optimism() &&
!self.is_dev() &&
self.chain_id() == Chain::optimism_mainnet()
if self.chain_spec().is_optimism()
&& !self.is_dev()
&& self.chain_id() == Chain::optimism_mainnet()
{
let latest = self.blockchain_db().last_block_number()?;
// bedrock height
if latest < 105235063 {
error!("Op-mainnet has been launched without importing the pre-Bedrock state. The chain can't progress without this. See also https://reth.rs/run/sync-op-mainnet.html?minimal-bootstrap-recommended");
return Err(ProviderError::BestBlockNotFound)
return Err(ProviderError::BestBlockNotFound);
}
}
@ -880,7 +883,7 @@ where
&self,
) -> eyre::Result<Box<dyn InvalidBlockHook<<T::Types as NodeTypes>::Primitives>>> {
let Some(ref hook) = self.node_config().debug.invalid_block_hook else {
return Ok(Box::new(NoopInvalidBlockHook::default()))
return Ok(Box::new(NoopInvalidBlockHook::default()));
};
let healthy_node_rpc_client = self.get_healthy_node_client()?;

4
crates/node/builder/src/launch/engine.rs
View File

@ -93,7 +93,9 @@ where
components_builder,
add_ons: AddOns { hooks, exexs: installed_exex, add_ons },
config,
shared_state,
} = target;
let NodeHooks { on_component_initialized, on_node_started, .. } = hooks;
// setup the launch context
@ -126,7 +128,7 @@ where
.with_blockchain_db::<T, _>(move |provider_factory| {
Ok(BlockchainProvider::new(provider_factory)?)
})?
.with_components(components_builder, on_component_initialized).await?;
.with_components(components_builder, on_component_initialized, shared_state.clone()).await?;
// spawn exexs
let exex_manager_handle = ExExLauncher::new(

10
crates/node/core/src/node_config.rs
View File

@ -148,6 +148,9 @@ pub struct NodeConfig<ChainSpec> {
/// The ingest directory for the node.
pub ingest_dir: Option<PathBuf>,
/// The local ingest directory for the node.
pub local_ingest_dir: Option<PathBuf>,
}
impl NodeConfig<ChainSpec> {
@ -178,6 +181,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
datadir: DatadirArgs::default(),
engine: EngineArgs::default(),
ingest_dir: None,
local_ingest_dir: None,
}
}
@ -363,7 +367,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
// try to look up the header in the database
if let Some(header) = header {
info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database");
return Ok(header.number())
return Ok(header.number());
}
Ok(self.fetch_tip_from_network(client, tip.into()).await.number())
@ -386,7 +390,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
match get_single_header(&client, tip).await {
Ok(tip_header) => {
info!(target: "reth::cli", ?tip, "Successfully fetched tip");
return tip_header
return tip_header;
}
Err(error) => {
fetch_failures += 1;
@ -470,6 +474,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
pruning: self.pruning,
engine: self.engine,
ingest_dir: self.ingest_dir,
local_ingest_dir: self.local_ingest_dir,
}
}
}
@ -498,6 +503,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
datadir: self.datadir.clone(),
engine: self.engine.clone(),
ingest_dir: self.ingest_dir.clone(),
local_ingest_dir: self.local_ingest_dir.clone(),
}
}
}