fix: read precompile draft (dirty)

This commit is contained in:
sprites0
2025-04-17 07:17:07 +00:00
parent aa8cffd78a
commit 6047b3d123
16 changed files with 245 additions and 138 deletions

17
Cargo.lock generated
View File

@ -6686,6 +6686,7 @@ dependencies = [
"reth-execution-types",
"reth-exex",
"reth-fs-util",
"reth-hyperliquid-types",
"reth-network",
"reth-network-api",
"reth-network-p2p",
@ -7394,6 +7395,7 @@ dependencies = [
"reth-chain-state",
"reth-errors",
"reth-execution-types",
"reth-hyperliquid-types",
"reth-payload-builder-primitives",
"reth-payload-primitives",
"reth-primitives",
@ -7803,15 +7805,20 @@ dependencies = [
"alloy-genesis",
"alloy-primitives",
"alloy-sol-types",
"lz4_flex",
"parking_lot",
"reth-chainspec",
"reth-ethereum-forks",
"reth-evm",
"reth-execution-types",
"reth-hyperliquid-types",
"reth-primitives",
"reth-primitives-traits",
"reth-revm",
"reth-testing-utils",
"rmp-serde",
"secp256k1 0.30.0",
"serde",
"serde_json",
"sha2 0.10.8",
]
@ -7954,6 +7961,16 @@ dependencies = [
"thiserror 2.0.11",
]
[[package]]
name = "reth-hyperliquid-types"
version = "1.2.0"
dependencies = [
"alloy-primitives",
"clap",
"reth-cli-commands",
"serde",
]
[[package]]
name = "reth-invalid-block-hooks"
version = "1.2.0"

View File

@ -129,6 +129,7 @@ members = [
"crates/trie/parallel/",
"crates/trie/sparse",
"crates/trie/trie",
"crates/hyperliquid-types",
"examples/beacon-api-sidecar-fetcher/",
"examples/beacon-api-sse/",
"examples/bsc-p2p",
@ -622,6 +623,8 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
# See: https://github.com/eira-fransham/crunchy/issues/13
crunchy = "=0.2.2"
# hyperliquid
reth-hyperliquid-types = { path = "crates/hyperliquid-types" }
lz4_flex = "0.11.3"
rmp-serde = "1.3.0"

View File

@ -64,6 +64,7 @@ reth-node-events.workspace = true
reth-node-metrics.workspace = true
reth-consensus.workspace = true
reth-prune.workspace = true
reth-hyperliquid-types.workspace = true
# crypto
alloy-eips = { workspace = true, features = ["kzg"] }

View File

@ -8,8 +8,6 @@ mod forwarder;
mod serialized;
mod spot_meta;
use std::path::PathBuf;
use block_ingest::BlockIngest;
use clap::{Args, Parser};
use forwarder::EthForwarderApiServer;
@ -20,10 +18,6 @@ use tracing::info;
#[derive(Args, Debug, Clone)]
struct HyperliquidExtArgs {
/// EVM blocks base directory
#[arg(long, default_value = "/tmp/evm-blocks")]
pub ingest_dir: PathBuf,
/// Upstream RPC URL to forward incoming transactions.
#[arg(long, default_value = "https://rpc.hyperliquid.xyz/evm")]
pub upstream_rpc_url: String,
@ -38,12 +32,13 @@ fn main() {
}
if let Err(err) = Cli::<EthereumChainSpecParser, HyperliquidExtArgs>::parse().run(
|builder, ingest_args| async move {
|builder, ext_args| async move {
let ingest_dir = builder.config().ingest_dir.clone().expect("ingest dir not set");
info!(target: "reth::cli", "Launching node");
let handle = builder
.node(EthereumNode::default())
.extend_rpc_modules(move |ctx| {
let upstream_rpc_url = ingest_args.upstream_rpc_url.clone();
let upstream_rpc_url = ext_args.upstream_rpc_url.clone();
let rpc = forwarder::EthForwarderExt::new(upstream_rpc_url).into_rpc();
for method_name in rpc.method_names() {
ctx.modules.remove_method_from_configured(method_name);
@ -56,7 +51,6 @@ fn main() {
.launch()
.await?;
let ingest_dir = ingest_args.ingest_dir;
let ingest = BlockIngest(ingest_dir);
ingest.run(handle.node).await.unwrap();
handle.node_exit_future.await

View File

@ -1,16 +1,16 @@
use alloy_primitives::{Address, Bytes, Log};
use alloy_primitives::{Address, Log};
use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
use reth_primitives::{SealedBlock, Transaction};
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct BlockAndReceipts {
pub(crate) block: EvmBlock,
pub(crate) receipts: Vec<LegacyReceipt>,
pub block: EvmBlock,
pub receipts: Vec<LegacyReceipt>,
#[serde(default)]
pub(crate) system_txs: Vec<SystemTx>,
pub system_txs: Vec<SystemTx>,
#[serde(default)]
pub(crate) read_precompile_calls:
Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>,
pub read_precompile_calls: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -18,6 +18,7 @@ pub(crate) enum EvmBlock {
Reth115(SealedBlock),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct LegacyReceipt {
tx_type: LegacyTxType,
@ -37,20 +38,6 @@ enum LegacyTxType {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct SystemTx {
pub(crate) tx: Transaction,
pub(crate) receipt: Option<LegacyReceipt>,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)]
pub(crate) struct ReadPrecompileInput {
pub(crate) input: Bytes,
pub(crate) gas_limit: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) enum ReadPrecompileResult {
Ok { gas_used: u64, bytes: Bytes },
OutOfGas,
Error,
UnexpectedError,
pub tx: Transaction,
pub receipt: Option<LegacyReceipt>,
}

View File

@ -114,6 +114,10 @@ pub struct NodeCommand<
/// Additional cli arguments
#[command(flatten, next_help_heading = "Extension")]
pub ext: Ext,
/// EVM blocks base directory
#[arg(long, default_value = "/tmp/evm-blocks")]
pub ingest_dir: PathBuf,
}
impl<C: ChainSpecParser> NodeCommand<C> {
@ -165,6 +169,7 @@ impl<
pruning,
ext,
engine,
ingest_dir,
} = self;
// set up node config
@ -183,6 +188,7 @@ impl<
dev,
pruning,
engine,
ingest_dir: Some(ingest_dir),
};
let data_dir = node_config.datadir();

View File

@ -31,6 +31,9 @@ alloy-eips.workspace = true
tokio = { workspace = true, features = ["sync"] }
futures.workspace = true
# hyperevm
reth-hyperliquid-types.workspace = true
# misc
auto_impl.workspace = true
serde.workspace = true

View File

@ -31,6 +31,12 @@ alloy-consensus.workspace = true
sha2.workspace = true
serde_json.workspace = true
serde = { workspace = true, features = ["derive"] }
rmp-serde.workspace = true
lz4_flex.workspace = true
reth-hyperliquid-types.workspace = true
parking_lot.workspace = true
[dev-dependencies]
reth-testing-utils.workspace = true

View File

@ -23,31 +23,33 @@ use alloy_evm::eth::EthEvmContext;
pub use alloy_evm::EthEvm;
use alloy_primitives::bytes::BufMut;
use alloy_primitives::hex::{FromHex, ToHexExt};
use alloy_primitives::{Address, Bytes, U256};
use alloy_primitives::{Address, B256};
use alloy_primitives::{Bytes, U256};
use core::{convert::Infallible, fmt::Debug};
use parking_lot::RwLock;
use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
use reth_evm::Database;
use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAttributes};
use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
use reth_primitives::TransactionSigned;
use reth_primitives::{SealedBlock, Transaction};
use reth_revm::context::result::{EVMError, HaltReason};
use reth_revm::context::{Block, Cfg, ContextTr};
use reth_revm::handler::{EthPrecompiles, PrecompileProvider};
use reth_revm::context::Cfg;
use reth_revm::handler::EthPrecompiles;
use reth_revm::inspector::NoOpInspector;
use reth_revm::interpreter::interpreter::EthInterpreter;
use reth_revm::interpreter::{Gas, InstructionResult, InterpreterResult};
use reth_revm::precompile::{
PrecompileError, PrecompileErrors, PrecompileFn, PrecompileOutput, PrecompileResult,
Precompiles,
};
use reth_revm::precompile::{PrecompileError, PrecompileErrors, Precompiles};
use reth_revm::MainBuilder;
use reth_revm::{
context::{BlockEnv, CfgEnv, TxEnv},
context_interface::block::BlobExcessGasAndPrice,
specification::hardfork::SpecId,
};
use reth_revm::{revm, Context, Inspector, MainBuilder, MainContext};
use sha2::Digest;
use reth_revm::{Context, Inspector, MainContext};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Write;
use std::sync::OnceLock;
use std::path::PathBuf;
mod config;
mod fix;
@ -65,15 +67,23 @@ pub mod eip6110;
/// Ethereum-related EVM configuration.
#[derive(Debug, Clone)]
pub struct EthEvmConfig {
chain_spec: Arc<ChainSpec>,
evm_factory: HyperliquidEvmFactory,
ingest_dir: Option<PathBuf>,
}
impl EthEvmConfig {
/// Creates a new Ethereum EVM configuration with the given chain spec.
pub fn new(chain_spec: Arc<ChainSpec>) -> Self {
Self { chain_spec, evm_factory: Default::default() }
Self { chain_spec, ingest_dir: None, evm_factory: Default::default() }
}
pub fn with_ingest_dir(mut self, ingest_dir: PathBuf) -> Self {
self.ingest_dir = Some(ingest_dir.clone());
self.evm_factory.ingest_dir = Some(ingest_dir);
self
}
/// Creates a new Ethereum EVM configuration for the ethereum mainnet.
@ -182,97 +192,15 @@ impl ConfigureEvmEnv for EthEvmConfig {
}
}
/// A custom precompile that contains static precompiles.
#[allow(missing_debug_implementations)]
#[derive(Clone)]
pub struct L1ReadPrecompiles<CTX> {
precompiles: EthPrecompiles<CTX>,
warm_addresses: Vec<Address>,
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct BlockAndReceipts {
#[serde(default)]
pub read_precompile_calls: Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>,
}
impl<CTX: ContextTr> L1ReadPrecompiles<CTX> {
fn new() -> Self {
let mut this = Self { precompiles: EthPrecompiles::default(), warm_addresses: vec![] };
this.update_warm_addresses(false);
this
}
fn update_warm_addresses(&mut self, precompile_enabled: bool) {
self.warm_addresses = if !precompile_enabled {
self.precompiles.warm_addresses().collect()
} else {
self.precompiles
.warm_addresses()
.chain((0..=9).into_iter().map(|x| {
let mut addr = [0u8; 20];
addr[18] = 0x8;
addr[19] = x;
Address::from_slice(&addr)
}))
.collect()
}
}
}
impl<CTX: ContextTr> PrecompileProvider for L1ReadPrecompiles<CTX> {
type Context = CTX;
type Output = InterpreterResult;
fn set_spec(&mut self, spec: <<Self::Context as ContextTr>::Cfg as Cfg>::Spec) {
self.precompiles.set_spec(spec);
// TODO: How to pass block number and chain id?
self.update_warm_addresses(false);
}
fn run(
&mut self,
context: &mut Self::Context,
address: &Address,
bytes: &Bytes,
gas_limit: u64,
) -> Result<Option<Self::Output>, revm::precompile::PrecompileErrors> {
if address[..18] == [0u8; 18] {
let maybe_precompile_index = u16::from_be_bytes([address[18], address[19]]);
let precompile_base =
std::env::var("PRECOMPILE_BASE").unwrap_or("/tmp/precompiles".to_string());
if 0x800 <= maybe_precompile_index && maybe_precompile_index <= 0x809 {
let block_number = context.block().number();
let input = vec![];
let mut writer = input.writer();
writer.write(&address.as_slice()).unwrap();
writer.write(bytes).unwrap();
writer.flush().unwrap();
let hash = sha2::Sha256::digest(writer.get_ref());
let file =
format!("{}/{}/{}.json", precompile_base, block_number, hash.encode_hex());
let (output, gas) = match load_result(file) {
Ok(Some(value)) => value,
Ok(None) => {
return Ok(Some(InterpreterResult {
result: InstructionResult::Return,
gas: Gas::new(gas_limit),
output: Bytes::new(),
}))
}
Err(value) => return Err(value),
};
return Ok(Some(InterpreterResult {
result: InstructionResult::Return,
gas: Gas::new(gas_limit - gas),
output,
}));
}
}
self.precompiles.run(context, address, bytes, gas_limit)
}
fn contains(&self, address: &Address) -> bool {
self.precompiles.contains(address)
}
fn warm_addresses(&self) -> Box<impl Iterator<Item = Address> + '_> {
Box::new(self.warm_addresses.iter().cloned())
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) enum EvmBlock {
Reth115(SealedBlock),
}
fn load_result(file: String) -> Result<Option<(Bytes, u64)>, PrecompileErrors> {
@ -296,23 +224,51 @@ fn load_result(file: String) -> Result<Option<(Bytes, u64)>, PrecompileErrors> {
/// Custom EVM configuration.
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct HyperliquidEvmFactory;
pub struct HyperliquidEvmFactory {
ingest_dir: Option<PathBuf>,
}
pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
let f = ((height - 1) / 1_000_000) * 1_000_000;
let s = ((height - 1) / 1_000) * 1_000;
let path = format!("{}/{f}/{s}/{height}.rmp.lz4", ingest_path.to_string_lossy());
if std::path::Path::new(&path).exists() {
let file = std::fs::File::open(path).unwrap();
let file = std::io::BufReader::new(file);
let mut decoder = lz4_flex::frame::FrameDecoder::new(file);
let blocks: Vec<BlockAndReceipts> = rmp_serde::from_read(&mut decoder).unwrap();
Some(blocks[0].clone())
} else {
None
}
}
impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
type Evm<DB: Database, I: Inspector<EthEvmContext<DB>, EthInterpreter>> =
EthEvm<DB, I, L1ReadPrecompiles<EthEvmContext<DB>>>;
EthEvm<DB, I, ReplayPrecompile<EthEvmContext<DB>>>;
type Tx = TxEnv;
type Error<DBError: core::error::Error + Send + Sync + 'static> = EVMError<DBError>;
type HaltReason = HaltReason;
type Context<DB: Database> = EthEvmContext<DB>;
fn create_evm<DB: Database>(&self, db: DB, input: EvmEnv) -> Self::Evm<DB, NoOpInspector> {
let cache = collect_block(self.ingest_dir.clone().unwrap(), input.block_env.number)
.unwrap()
.read_precompile_calls;
let evm = Context::mainnet()
.with_db(db)
.with_cfg(input.cfg_env)
.with_block(input.block_env)
.build_mainnet_with_inspector(NoOpInspector {})
.with_precompiles(L1ReadPrecompiles::new());
.with_precompiles(ReplayPrecompile::new(
EthPrecompiles::default(),
Arc::new(RwLock::new(
cache
.into_iter()
.map(|(address, calls)| (address, HashMap::from_iter(calls.into_iter())))
.collect(),
)),
));
EthEvm::new(evm, false)
}
@ -509,3 +465,7 @@ mod tests {
assert_eq!(evm.tx, Default::default());
}
}
mod precompile_replay;
pub use precompile_replay::ReplayPrecompile;

View File

@ -0,0 +1,85 @@
use alloy_primitives::{Address, Bytes};
use parking_lot::RwLock;
use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
use reth_revm::{
context::{Cfg, ContextTr},
handler::{EthPrecompiles, PrecompileProvider},
interpreter::{Gas, InstructionResult, InterpreterResult},
precompile::{PrecompileError, PrecompileErrors},
};
use std::{collections::HashMap, sync::Arc};
/// Precompile that replays cached results.
#[derive(Clone)]
pub struct ReplayPrecompile<CTX: ContextTr> {
precompiles: EthPrecompiles<CTX>,
cache: Arc<RwLock<HashMap<Address, HashMap<ReadPrecompileInput, ReadPrecompileResult>>>>,
}
impl<CTX: ContextTr> std::fmt::Debug for ReplayPrecompile<CTX> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ReplayPrecompile").finish()
}
}
impl<CTX: ContextTr> ReplayPrecompile<CTX> {
/// Creates a new replay precompile with the given precompiles and cache.
pub fn new(
precompiles: EthPrecompiles<CTX>,
cache: Arc<RwLock<HashMap<Address, HashMap<ReadPrecompileInput, ReadPrecompileResult>>>>,
) -> Self {
Self { precompiles, cache }
}
}
impl<CTX: ContextTr> PrecompileProvider for ReplayPrecompile<CTX> {
type Context = CTX;
type Output = InterpreterResult;
fn set_spec(&mut self, spec: <<Self::Context as ContextTr>::Cfg as Cfg>::Spec) {
self.precompiles.set_spec(spec);
}
fn run(
&mut self,
context: &mut Self::Context,
address: &Address,
bytes: &Bytes,
gas_limit: u64,
) -> Result<Option<Self::Output>, PrecompileErrors> {
let cache = self.cache.read();
if let Some(precompile_calls) = cache.get(address) {
let input = ReadPrecompileInput { input: bytes.clone(), gas_limit };
let mut result = InterpreterResult {
result: InstructionResult::Return,
gas: Gas::new(gas_limit),
output: Bytes::new(),
};
return match *precompile_calls.get(&input).expect("missing precompile call") {
ReadPrecompileResult::Ok { gas_used, ref bytes } => {
let underflow = result.gas.record_cost(gas_used);
assert!(underflow, "Gas underflow is not possible");
result.output = bytes.clone();
Ok(Some(result))
}
ReadPrecompileResult::OutOfGas => Err(PrecompileError::OutOfGas.into()),
ReadPrecompileResult::Error => {
Err(PrecompileError::other("precompile failed").into())
}
ReadPrecompileResult::UnexpectedError => panic!("unexpected precompile error"),
};
}
// If no cached result, fall back to normal precompile execution
self.precompiles.run(context, address, bytes, gas_limit)
}
fn contains(&self, address: &Address) -> bool {
self.precompiles.contains(address)
}
fn warm_addresses(&self) -> Box<impl Iterator<Item = Address> + '_> {
Box::new(self.precompiles.warm_addresses())
}
}

View File

@ -248,7 +248,7 @@ where
ctx: &BuilderContext<Node>,
) -> eyre::Result<(Self::EVM, Self::Executor)> {
let chain_spec = ctx.chain_spec();
let evm_config = EthEvmConfig::new(ctx.chain_spec());
let evm_config = EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir());
let strategy_factory = EthExecutionStrategyFactory::new(chain_spec, evm_config.clone());
let executor = BasicBlockExecutorProvider::new(strategy_factory);

View File

@ -74,6 +74,6 @@ where
ctx: &BuilderContext<Node>,
pool: Pool,
) -> eyre::Result<Self::PayloadBuilder> {
self.build(EthEvmConfig::new(ctx.chain_spec()), ctx, pool)
self.build(EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir()), ctx, pool)
}
}

View File

@ -0,0 +1,19 @@
[package]
name = "reth-hyperliquid-types"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
alloy-primitives.workspace = true
serde.workspace = true
[dev-dependencies]
clap.workspace = true
reth-cli-commands.workspace = true

View File

@ -0,0 +1,16 @@
use alloy_primitives::Bytes;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)]
pub struct ReadPrecompileInput {
pub input: Bytes,
pub gas_limit: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ReadPrecompileResult {
Ok { gas_used: u64, bytes: Bytes },
OutOfGas,
Error,
UnexpectedError,
}

View File

@ -37,7 +37,7 @@ use reth_provider::{
use reth_tasks::TaskExecutor;
use reth_transaction_pool::{PoolConfig, PoolTransaction, TransactionPool};
use secp256k1::SecretKey;
use std::sync::Arc;
use std::{path::PathBuf, sync::Arc};
use tracing::{info, trace, warn};
pub mod add_ons;
@ -750,6 +750,10 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
{
network_builder.build(self.provider.clone())
}
pub fn ingest_dir(&self) -> PathBuf {
self.config().ingest_dir.clone().expect("ingest dir not set")
}
}
impl<Node: FullNodeTypes<Types: NodeTypes<ChainSpec: Hardforks>>> BuilderContext<Node> {

View File

@ -145,6 +145,9 @@ pub struct NodeConfig<ChainSpec> {
/// All engine related arguments
pub engine: EngineArgs,
/// The ingest directory for the node.
pub ingest_dir: Option<PathBuf>,
}
impl NodeConfig<ChainSpec> {
@ -174,6 +177,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
pruning: PruningArgs::default(),
datadir: DatadirArgs::default(),
engine: EngineArgs::default(),
ingest_dir: None,
}
}
@ -465,6 +469,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
dev: self.dev,
pruning: self.pruning,
engine: self.engine,
ingest_dir: self.ingest_dir,
}
}
}
@ -492,6 +497,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
pruning: self.pruning.clone(),
datadir: self.datadir.clone(),
engine: self.engine.clone(),
ingest_dir: self.ingest_dir.clone(),
}
}
}