mirror of
https://github.com/hl-archive-node/nanoreth.git
synced 2025-12-06 02:49:55 +00:00
Local block sync (#7)
* update: logs * update: more logging * update: rename local ingest dir args * update: fix build * update: directory path * update: logs * update: log ts * update: fetch last block * update: time formatting * update: handle seconds * update: lmore logs * fix: provided args * update: logs * fix: build * update: indefinite wiat * update: run the right loop * update: remove offset * update: scan impl * update: log exists * update: collect s3 blocks * update: change the file * update: logs * fix: deserialization * fix: build * update: remove block * update: add logs * update: logs * update: logs * update: dates * update: ignore older blocks * update: hook up to sync * fix: build * fix: build * update: logs * update: logs * update: start height cond * update: height * update: loggy * update: cond * update: cond * update: cond * update: logs * update: fix height issues * update: logs * only collect s3 * update: log block * update: log both blocks * update; return s3 block * update: use local block * update: blocks * update: remove logs * update: logs * update: remove warns and logs * update: collection log * update: logs * update: logs * update: scan through heights when registering evm * update: add local ingest dir to other evm factory * fix: build * update: add cli cmd * update: remove additional arg * update: change where local ingest dir comes from * fix: receipts * update: deser format * update: fix build * update: logs * update: logs * update: logs * update: logs * update: share precompiles with engine * update: insert compiles * update: change sync dir * update: logs * update: logs * update: logs * update: fix build * update: pipe builder context through * update: untracked * update: pass through context * fix: build * fix: build * update: logs * update: logs * update: logs * update: fix cache passthrough * update: remove logs * update: logs * update: hour rollover * update: zero out hour * update: hour sync * update: cleanup code and speedup sync * update: speedup sync * update: remove logs * update: speed up sync * update: speed up sync * update: ingest in reverse * fix: iter rev * update: break line loop early * update: remove break * update: iteration speed * update: fix build * update: slow down tail ival * update: logs * update: skip last line * update: remove log * update: height * update: logs * update: return logs * update: disable attempt logs * update: tail interval * update: cleanup logs * update: add iter skip * update: fix build * update: skip -1 * fix: skip * fix: build * update: build * fix: build * update: logs * update: log idx * update: skip after enumerate * update: cleanup * update: more cleanup * update: refactor BuilderSharedState to HyperliquidSharedState * update: more cleanup * update: cleanup and refactor collect_local_block * update: error msg * update: readme * update: typo * update: file log * fix: typo build * update: debug log
This commit is contained in:
arb00r
21
Cargo.lock
generated
21
Cargo.lock
generated
@ -2588,9 +2588,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.3.11"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
|
||||
checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
"serde",
|
||||
@ -6724,6 +6724,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"similar-asserts",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@ -7813,6 +7814,7 @@ dependencies = [
|
||||
"reth-evm",
|
||||
"reth-execution-types",
|
||||
"reth-hyperliquid-types",
|
||||
"reth-node-builder",
|
||||
"reth-primitives",
|
||||
"reth-primitives-traits",
|
||||
"reth-revm",
|
||||
@ -7968,8 +7970,10 @@ version = "1.2.0"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"clap",
|
||||
"parking_lot",
|
||||
"reth-cli-commands",
|
||||
"serde",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -8289,6 +8293,7 @@ dependencies = [
|
||||
"reth-evm",
|
||||
"reth-exex",
|
||||
"reth-fs-util",
|
||||
"reth-hyperliquid-types",
|
||||
"reth-invalid-block-hooks",
|
||||
"reth-network",
|
||||
"reth-network-api",
|
||||
@ -11427,9 +11432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.37"
|
||||
version = "0.3.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
|
||||
checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
@ -11445,15 +11450,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
|
||||
checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.19"
|
||||
version = "0.2.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
|
||||
checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
|
||||
@ -555,6 +555,9 @@ tokio = { version = "1.39", default-features = false }
|
||||
tokio-stream = "0.1.11"
|
||||
tokio-util = { version = "0.7.4", features = ["codec"] }
|
||||
|
||||
# time
|
||||
time = "0.3.41"
|
||||
|
||||
# async
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1.68"
|
||||
|
||||
23
README.md
23
README.md
@ -21,6 +21,29 @@ $ reth node --http --http.addr 0.0.0.0 --http.api eth,ots,net,web3 \
|
||||
--ws --ws.addr 0.0.0.0 --ws.origins '*' --ws.api eth,ots,net,web3 --ingest-dir ~/evm-blocks --ws.port 8545
|
||||
```
|
||||
|
||||
## How to run (mainnet) (with local block sync)
|
||||
|
||||
You can choose to source blocks from your local instance of hl-node instead of relying on an s3 replica.
|
||||
This will require you to first have a hl-node outputting blocks prior to running the initial s3 sync,
|
||||
the node will prioritise locally produced blocks with a fallback to s3.
|
||||
This method will allow you to reduce the need to rely on goofys.
|
||||
|
||||
It is recommended that you periodically sync evm-blocks from s3 so you have a fallback in case your hl-node fails, as hl-node
|
||||
will not backfill evm blocks.
|
||||
```sh
|
||||
# Run your local hl-node (make sure output file buffering is disabled)
|
||||
# Make sure evm blocks are being produced inside evm_block_and_receipts
|
||||
$ hl-node run-non-validator --replica-cmds-style recent-actions --serve-eth-rpc --disable-output-file-buffering
|
||||
|
||||
# Fetch EVM blocks (Initial sync)
|
||||
$ aws s3 sync s3://hl-mainnet-evm-blocks/ ~/evm-blocks --request-payer requester # one-time
|
||||
|
||||
# Run node (with local-ingest-dir arg)
|
||||
$ make install
|
||||
$ reth node --http --http.addr 0.0.0.0 --http.api eth,ots,net,web3 \
|
||||
--ws --ws.addr 0.0.0.0 --ws.origins '*' --ws.api eth,ots,net,web3 --ingest-dir ~/evm-blocks --local-ingest-dir <path-to-your-hl-node-evm-blocks-dir> --ws.port 8545
|
||||
```
|
||||
|
||||
## How to run (testnet)
|
||||
|
||||
Testnet is supported since block 21304281.
|
||||
|
||||
@ -84,6 +84,9 @@ serde_json.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "macros", "time", "rt-multi-thread"] }
|
||||
futures.workspace = true
|
||||
|
||||
# time
|
||||
time = { workspace = true }
|
||||
|
||||
# misc
|
||||
aquamarine.workspace = true
|
||||
eyre.workspace = true
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
use std::path::PathBuf;
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use alloy_consensus::{BlockBody, BlockHeader, Transaction};
|
||||
@ -10,22 +12,90 @@ use alloy_rpc_types::engine::{
|
||||
use jsonrpsee::http_client::{transport::HttpBackend, HttpClient};
|
||||
use reth::network::PeersHandleProvider;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_node_api::{FullNodeComponents, PayloadTypes};
|
||||
use reth_hyperliquid_types::PrecompilesCache;
|
||||
use reth_node_api::{Block, FullNodeComponents, PayloadTypes};
|
||||
use reth_node_builder::EngineTypes;
|
||||
use reth_node_builder::NodeTypesWithEngine;
|
||||
use reth_node_builder::{rpc::RethRpcAddOns, FullNode};
|
||||
use reth_payload_builder::{EthBuiltPayload, EthPayloadBuilderAttributes, PayloadId};
|
||||
use reth_primitives::{Transaction as TypedTransaction, TransactionSigned};
|
||||
use reth_provider::{BlockHashReader, StageCheckpointReader};
|
||||
use reth_provider::{BlockHashReader, BlockReader, StageCheckpointReader};
|
||||
use reth_rpc_api::EngineApiClient;
|
||||
use reth_rpc_layer::AuthClientService;
|
||||
use reth_stages::StageId;
|
||||
use serde::Deserialize;
|
||||
use time::{format_description, Duration, OffsetDateTime};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, info};
|
||||
|
||||
use crate::serialized::{BlockAndReceipts, EvmBlock};
|
||||
use crate::spot_meta::erc20_contract_to_spot_token;
|
||||
|
||||
pub(crate) struct BlockIngest(pub PathBuf);
|
||||
/// Poll interval when tailing an *open* hourly file.
|
||||
const TAIL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(25);
|
||||
/// Sub‑directory that contains day folders (inside `local_ingest_dir`).
|
||||
const HOURLY_SUBDIR: &str = "hourly";
|
||||
|
||||
pub(crate) struct BlockIngest {
|
||||
pub ingest_dir: PathBuf,
|
||||
pub local_ingest_dir: Option<PathBuf>,
|
||||
pub local_blocks_cache: Arc<Mutex<BTreeMap<u64, BlockAndReceipts>>>, // height → block
|
||||
pub precompiles_cache: PrecompilesCache,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LocalBlockAndReceipts(String, BlockAndReceipts);
|
||||
|
||||
struct ScanResult {
|
||||
next_expected_height: u64,
|
||||
new_blocks: Vec<BlockAndReceipts>,
|
||||
}
|
||||
|
||||
fn scan_hour_file(path: &Path, last_line: &mut usize, start_height: u64) -> ScanResult {
|
||||
// info!(
|
||||
// "Scanning hour block file @ {:?} for height [{:?}] | Last Line {:?}",
|
||||
// path, start_height, last_line
|
||||
// );
|
||||
let file = std::fs::File::open(path).expect("Failed to open hour file path");
|
||||
let reader = BufReader::new(file);
|
||||
|
||||
let mut new_blocks = Vec::<BlockAndReceipts>::new();
|
||||
let mut last_height = start_height;
|
||||
let lines: Vec<String> = reader.lines().collect::<Result<_, _>>().unwrap();
|
||||
let skip = if *last_line == 0 { 0 } else { (last_line.clone()) - 1 };
|
||||
|
||||
for (line_idx, line) in lines.iter().enumerate().skip(skip) {
|
||||
// Safety check ensuring efficiency
|
||||
if line_idx < *last_line {
|
||||
continue;
|
||||
}
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let LocalBlockAndReceipts(_block_timestamp, parsed_block): LocalBlockAndReceipts =
|
||||
serde_json::from_str(&line).expect("Failed to parse local block and receipts");
|
||||
|
||||
let height = match &parsed_block.block {
|
||||
EvmBlock::Reth115(b) => {
|
||||
let block_number = b.header().number() as u64;
|
||||
// Another check to ensure not returning an older block
|
||||
if block_number < start_height {
|
||||
continue;
|
||||
}
|
||||
block_number
|
||||
}
|
||||
};
|
||||
// println!("Iterating block height {:?} | Line {}", height, line_idx);
|
||||
if height >= start_height {
|
||||
last_height = last_height.max(height);
|
||||
new_blocks.push(parsed_block);
|
||||
*last_line = line_idx;
|
||||
}
|
||||
}
|
||||
|
||||
ScanResult { next_expected_height: last_height + 1, new_blocks }
|
||||
}
|
||||
|
||||
async fn submit_payload<Engine: PayloadTypes + EngineTypes>(
|
||||
engine_api_client: &HttpClient<AuthClientService<HttpBackend>>,
|
||||
@ -53,22 +123,118 @@ async fn submit_payload<Engine: PayloadTypes + EngineTypes>(
|
||||
Ok(submission.latest_valid_hash.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn datetime_from_timestamp(ts_sec: u64) -> OffsetDateTime {
|
||||
OffsetDateTime::from_unix_timestamp_nanos((ts_sec as i128) * 1_000 * 1_000_000)
|
||||
.expect("timestamp out of range")
|
||||
}
|
||||
|
||||
fn date_from_datetime(dt: OffsetDateTime) -> String {
|
||||
dt.format(&format_description::parse("[year][month][day]").unwrap()).unwrap()
|
||||
}
|
||||
|
||||
impl BlockIngest {
|
||||
pub(crate) fn collect_block(&self, height: u64) -> Option<BlockAndReceipts> {
|
||||
pub(crate) async fn collect_block(&self, height: u64) -> Option<BlockAndReceipts> {
|
||||
// info!("Attempting to collect block @ height [{height}]");
|
||||
|
||||
// Not a one liner (using .or) to include logs
|
||||
if let Some(block) = self.try_collect_local_block(height).await {
|
||||
info!("Returning locally synced block for @ Height [{height}]");
|
||||
return Some(block);
|
||||
} else {
|
||||
self.try_collect_s3_block(height)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn try_collect_s3_block(&self, height: u64) -> Option<BlockAndReceipts> {
|
||||
let f = ((height - 1) / 1_000_000) * 1_000_000;
|
||||
let s = ((height - 1) / 1_000) * 1_000;
|
||||
let path = format!("{}/{f}/{s}/{height}.rmp.lz4", self.0.to_string_lossy());
|
||||
let path = format!("{}/{f}/{s}/{height}.rmp.lz4", self.ingest_dir.to_string_lossy());
|
||||
if std::path::Path::new(&path).exists() {
|
||||
let file = std::fs::File::open(path).unwrap();
|
||||
let file = std::io::BufReader::new(file);
|
||||
let mut decoder = lz4_flex::frame::FrameDecoder::new(file);
|
||||
let blocks: Vec<BlockAndReceipts> = rmp_serde::from_read(&mut decoder).unwrap();
|
||||
info!("Returning s3 synced block for @ Height [{height}]");
|
||||
Some(blocks[0].clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_collect_local_block(&self, height: u64) -> Option<BlockAndReceipts> {
|
||||
let mut u_cache = self.local_blocks_cache.lock().await;
|
||||
u_cache.remove(&height)
|
||||
}
|
||||
|
||||
async fn start_local_ingest_loop(&self, current_head: u64, current_ts: u64) {
|
||||
let Some(root) = &self.local_ingest_dir else { return }; // nothing to do
|
||||
let root = root.to_owned();
|
||||
let cache = self.local_blocks_cache.clone();
|
||||
let precompiles_cache = self.precompiles_cache.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut next_height = current_head;
|
||||
let mut dt = datetime_from_timestamp(current_ts)
|
||||
.replace_minute(0)
|
||||
.unwrap()
|
||||
.replace_second(0)
|
||||
.unwrap()
|
||||
.replace_nanosecond(0)
|
||||
.unwrap();
|
||||
|
||||
let mut hour = dt.hour();
|
||||
let mut day_str = date_from_datetime(dt);
|
||||
let mut last_line = 0;
|
||||
|
||||
loop {
|
||||
let hour_file = root.join(HOURLY_SUBDIR).join(&day_str).join(format!("{hour}"));
|
||||
|
||||
if hour_file.exists() {
|
||||
let ScanResult { next_expected_height, new_blocks } =
|
||||
scan_hour_file(&hour_file, &mut last_line, next_height);
|
||||
if !new_blocks.is_empty() {
|
||||
let mut u_cache = cache.lock().await;
|
||||
let mut u_pre_cache = precompiles_cache.lock();
|
||||
for blk in new_blocks {
|
||||
let precompiles = blk.read_precompile_calls.clone();
|
||||
let h = match &blk.block {
|
||||
EvmBlock::Reth115(b) => {
|
||||
let block_number = b.header().number() as u64;
|
||||
block_number
|
||||
}
|
||||
};
|
||||
u_cache.insert(h, blk);
|
||||
u_pre_cache.insert(h, precompiles);
|
||||
}
|
||||
next_height = next_expected_height;
|
||||
}
|
||||
}
|
||||
|
||||
// Decide whether the *current* hour file is closed (past) or
|
||||
// still live. If it’s in the past by > 1 h, move to next hour;
|
||||
// otherwise, keep tailing the same file.
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
// println!("Date Current {:?}", dt);
|
||||
// println!("Now Current {:?}", now);
|
||||
|
||||
if dt + Duration::HOUR < now {
|
||||
dt += Duration::HOUR;
|
||||
hour = dt.hour();
|
||||
day_str = date_from_datetime(dt);
|
||||
last_line = 0;
|
||||
info!(
|
||||
"Moving to a new file. {:?}",
|
||||
root.join(HOURLY_SUBDIR).join(&day_str).join(format!("{hour}"))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
tokio::time::sleep(TAIL_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) async fn run<Node, Engine, AddOns>(
|
||||
&self,
|
||||
node: FullNode<Node, AddOns>,
|
||||
@ -96,9 +262,19 @@ impl BlockIngest {
|
||||
let engine_api = node.auth_server_handle().http_client();
|
||||
let mut evm_map = erc20_contract_to_spot_token(node.chain_spec().chain_id()).await?;
|
||||
|
||||
let current_block_timestamp: u64 = provider
|
||||
.block_by_number(head)
|
||||
.expect("Failed to fetch current block in db")
|
||||
.expect("Block does not exist")
|
||||
.into_header()
|
||||
.timestamp();
|
||||
|
||||
info!("Current height {height}, timestamp {current_block_timestamp}");
|
||||
self.start_local_ingest_loop(height, current_block_timestamp).await;
|
||||
|
||||
loop {
|
||||
let Some(original_block) = self.collect_block(height) else {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
let Some(original_block) = self.collect_block(height).await else {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
|
||||
continue;
|
||||
};
|
||||
let EvmBlock::Reth115(mut block) = original_block.block;
|
||||
@ -111,6 +287,7 @@ impl BlockIngest {
|
||||
let BlockBody { transactions, ommers, withdrawals } =
|
||||
std::mem::take(block.body_mut());
|
||||
let mut system_txs = vec![];
|
||||
|
||||
for transaction in original_block.system_txs {
|
||||
let TypedTransaction::Legacy(tx) = &transaction.tx else {
|
||||
panic!("Unexpected transaction type");
|
||||
@ -180,10 +357,12 @@ impl BlockIngest {
|
||||
PayloadStatusEnum::Valid,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let current_timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis();
|
||||
|
||||
if height % 100 == 0 || current_timestamp - previous_timestamp > 100 {
|
||||
EngineApiClient::<Engine>::fork_choice_updated_v2(
|
||||
&engine_api,
|
||||
|
||||
@ -9,12 +9,16 @@ mod serialized;
|
||||
mod spot_meta;
|
||||
mod tx_forwarder;
|
||||
|
||||
use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
|
||||
|
||||
use block_ingest::BlockIngest;
|
||||
use call_forwarder::CallForwarderApiServer;
|
||||
use clap::{Args, Parser};
|
||||
use reth::cli::Cli;
|
||||
use reth_ethereum_cli::chainspec::EthereumChainSpecParser;
|
||||
use reth_hyperliquid_types::PrecompilesCache;
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::info;
|
||||
use tx_forwarder::EthForwarderApiServer;
|
||||
|
||||
@ -37,12 +41,17 @@ fn main() {
|
||||
std::env::set_var("RUST_BACKTRACE", "1");
|
||||
}
|
||||
|
||||
let precompiles_cache = PrecompilesCache::new(parking_lot::Mutex::new(BTreeMap::new()));
|
||||
let local_blocks_cache = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
|
||||
if let Err(err) = Cli::<EthereumChainSpecParser, HyperliquidExtArgs>::parse().run(
|
||||
|builder, ext_args| async move {
|
||||
let ingest_dir = builder.config().ingest_dir.clone().expect("ingest dir not set");
|
||||
let local_ingest_dir = builder.config().local_ingest_dir.clone();
|
||||
info!(target: "reth::cli", "Launching node");
|
||||
let handle = builder
|
||||
.node(EthereumNode::default())
|
||||
.add_precompiles_cache(precompiles_cache.clone())
|
||||
.extend_rpc_modules(move |ctx| {
|
||||
let upstream_rpc_url = ext_args.upstream_rpc_url;
|
||||
ctx.modules.replace_configured(
|
||||
@ -62,7 +71,8 @@ fn main() {
|
||||
.launch()
|
||||
.await?;
|
||||
|
||||
let ingest = BlockIngest(ingest_dir);
|
||||
let ingest =
|
||||
BlockIngest { ingest_dir, local_ingest_dir, local_blocks_cache, precompiles_cache };
|
||||
ingest.run(handle.node).await.unwrap();
|
||||
handle.node_exit_future.await
|
||||
},
|
||||
|
||||
@ -118,6 +118,10 @@ pub struct NodeCommand<
|
||||
/// EVM blocks base directory
|
||||
#[arg(long, default_value = "/tmp/evm-blocks")]
|
||||
pub ingest_dir: PathBuf,
|
||||
|
||||
/// Local EVM blocks base directory
|
||||
#[arg(long, default_value = "/tmp/evm-blocks-and-receipts")]
|
||||
pub local_ingest_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> NodeCommand<C> {
|
||||
@ -170,6 +174,7 @@ impl<
|
||||
ext,
|
||||
engine,
|
||||
ingest_dir,
|
||||
local_ingest_dir,
|
||||
} = self;
|
||||
|
||||
// set up node config
|
||||
@ -189,6 +194,7 @@ impl<
|
||||
pruning,
|
||||
engine,
|
||||
ingest_dir: Some(ingest_dir),
|
||||
local_ingest_dir: Some(local_ingest_dir),
|
||||
};
|
||||
|
||||
let data_dir = node_config.datadir();
|
||||
|
||||
@ -225,6 +225,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::eip1559::GAS_LIMIT_BOUND_DIVISOR;
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder};
|
||||
use reth_primitives_traits::proofs;
|
||||
|
||||
@ -18,6 +18,7 @@ reth-ethereum-forks.workspace = true
|
||||
reth-revm.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-node-builder.workspace = true
|
||||
|
||||
# Ethereum
|
||||
reth-primitives-traits.workspace = true
|
||||
|
||||
@ -21,24 +21,21 @@ use alloc::sync::Arc;
|
||||
use alloy_consensus::{BlockHeader, Header};
|
||||
use alloy_evm::eth::EthEvmContext;
|
||||
pub use alloy_evm::EthEvm;
|
||||
use alloy_primitives::bytes::BufMut;
|
||||
use alloy_primitives::hex::{FromHex, ToHexExt};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_primitives::{Bytes, U256};
|
||||
use alloy_primitives::Address;
|
||||
use alloy_primitives::U256;
|
||||
use core::{convert::Infallible, fmt::Debug};
|
||||
use parking_lot::RwLock;
|
||||
use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
|
||||
use reth_evm::Database;
|
||||
use reth_evm::{ConfigureEvm, ConfigureEvmEnv, EvmEnv, EvmFactory, NextBlockEnvAttributes};
|
||||
use reth_hyperliquid_types::{ReadPrecompileInput, ReadPrecompileResult};
|
||||
use reth_hyperliquid_types::{PrecompilesCache, ReadPrecompileInput, ReadPrecompileResult};
|
||||
use reth_node_builder::HyperliquidSharedState;
|
||||
use reth_primitives::SealedBlock;
|
||||
use reth_primitives::TransactionSigned;
|
||||
use reth_primitives::{SealedBlock, Transaction};
|
||||
use reth_revm::context::result::{EVMError, HaltReason};
|
||||
use reth_revm::context::Cfg;
|
||||
use reth_revm::handler::EthPrecompiles;
|
||||
use reth_revm::inspector::NoOpInspector;
|
||||
use reth_revm::interpreter::interpreter::EthInterpreter;
|
||||
use reth_revm::precompile::{PrecompileError, PrecompileErrors, Precompiles};
|
||||
use reth_revm::MainBuilder;
|
||||
use reth_revm::{
|
||||
context::{BlockEnv, CfgEnv, TxEnv},
|
||||
@ -48,7 +45,6 @@ use reth_revm::{
|
||||
use reth_revm::{Context, Inspector, MainContext};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
mod config;
|
||||
@ -72,12 +68,13 @@ pub struct EthEvmConfig {
|
||||
chain_spec: Arc<ChainSpec>,
|
||||
evm_factory: HyperliquidEvmFactory,
|
||||
ingest_dir: Option<PathBuf>,
|
||||
shared_state: Option<HyperliquidSharedState>,
|
||||
}
|
||||
|
||||
impl EthEvmConfig {
|
||||
/// Creates a new Ethereum EVM configuration with the given chain spec.
|
||||
pub fn new(chain_spec: Arc<ChainSpec>) -> Self {
|
||||
Self { chain_spec, ingest_dir: None, evm_factory: Default::default() }
|
||||
Self { chain_spec, ingest_dir: None, evm_factory: Default::default(), shared_state: None }
|
||||
}
|
||||
|
||||
pub fn with_ingest_dir(mut self, ingest_dir: PathBuf) -> Self {
|
||||
@ -86,6 +83,12 @@ impl EthEvmConfig {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_shared_state(mut self, shared_state: Option<HyperliquidSharedState>) -> Self {
|
||||
self.shared_state = shared_state.clone();
|
||||
self.evm_factory.shared_state = shared_state;
|
||||
self
|
||||
}
|
||||
|
||||
/// Creates a new Ethereum EVM configuration for the ethereum mainnet.
|
||||
pub fn mainnet() -> Self {
|
||||
Self::new(MAINNET.clone())
|
||||
@ -208,9 +211,10 @@ pub(crate) enum EvmBlock {
|
||||
#[non_exhaustive]
|
||||
pub struct HyperliquidEvmFactory {
|
||||
ingest_dir: Option<PathBuf>,
|
||||
shared_state: Option<HyperliquidSharedState>,
|
||||
}
|
||||
|
||||
pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
|
||||
pub(crate) fn collect_s3_block(ingest_path: PathBuf, height: u64) -> Option<BlockAndReceipts> {
|
||||
let f = ((height - 1) / 1_000_000) * 1_000_000;
|
||||
let s = ((height - 1) / 1_000) * 1_000;
|
||||
let path = format!("{}/{f}/{s}/{height}.rmp.lz4", ingest_path.to_string_lossy());
|
||||
@ -225,6 +229,32 @@ pub(crate) fn collect_block(ingest_path: PathBuf, height: u64) -> Option<BlockAn
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn get_locally_sourced_precompiles_for_height(
|
||||
precompiles_cache: PrecompilesCache,
|
||||
height: u64,
|
||||
) -> Option<Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>> {
|
||||
let mut u_cache = precompiles_cache.lock();
|
||||
u_cache.remove(&height)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_block(
|
||||
ingest_path: PathBuf,
|
||||
shared_state: Option<HyperliquidSharedState>,
|
||||
height: u64,
|
||||
) -> Option<BlockAndReceipts> {
|
||||
// Attempt to source precompile from the cache that is shared the binary level with the block
|
||||
// ingestor.
|
||||
if let Some(shared_state) = shared_state {
|
||||
if let Some(calls) =
|
||||
get_locally_sourced_precompiles_for_height(shared_state.precompiles_cache, height)
|
||||
{
|
||||
return Some(BlockAndReceipts { read_precompile_calls: calls });
|
||||
}
|
||||
}
|
||||
// Fallback to s3 always
|
||||
collect_s3_block(ingest_path, height)
|
||||
}
|
||||
|
||||
impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
|
||||
type Evm<DB: Database, I: Inspector<EthEvmContext<DB>, EthInterpreter>> =
|
||||
EthEvm<DB, I, ReplayPrecompile<EthEvmContext<DB>>>;
|
||||
@ -234,9 +264,14 @@ impl EvmFactory<EvmEnv> for HyperliquidEvmFactory {
|
||||
type Context<DB: Database> = EthEvmContext<DB>;
|
||||
|
||||
fn create_evm<DB: Database>(&self, db: DB, input: EvmEnv) -> Self::Evm<DB, NoOpInspector> {
|
||||
let cache = collect_block(self.ingest_dir.clone().unwrap(), input.block_env.number)
|
||||
.unwrap()
|
||||
.read_precompile_calls;
|
||||
let block = collect_block(
|
||||
self.ingest_dir.clone().unwrap(),
|
||||
self.shared_state.clone(),
|
||||
input.block_env.number,
|
||||
)
|
||||
.expect("Failed to collect a submitted block. If sourcing locally, make sure your local hl-node is producing blocks.");
|
||||
let cache = block.read_precompile_calls;
|
||||
|
||||
let evm = Context::mainnet()
|
||||
.with_db(db)
|
||||
.with_cfg(input.cfg_env)
|
||||
|
||||
@ -248,7 +248,9 @@ where
|
||||
ctx: &BuilderContext<Node>,
|
||||
) -> eyre::Result<(Self::EVM, Self::Executor)> {
|
||||
let chain_spec = ctx.chain_spec();
|
||||
let evm_config = EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir());
|
||||
let evm_config = EthEvmConfig::new(ctx.chain_spec())
|
||||
.with_ingest_dir(ctx.ingest_dir())
|
||||
.with_shared_state(ctx.shared_state());
|
||||
let strategy_factory = EthExecutionStrategyFactory::new(chain_spec, evm_config.clone());
|
||||
let executor = BasicBlockExecutorProvider::new(strategy_factory);
|
||||
|
||||
|
||||
@ -74,6 +74,12 @@ where
|
||||
ctx: &BuilderContext<Node>,
|
||||
pool: Pool,
|
||||
) -> eyre::Result<Self::PayloadBuilder> {
|
||||
self.build(EthEvmConfig::new(ctx.chain_spec()).with_ingest_dir(ctx.ingest_dir()), ctx, pool)
|
||||
self.build(
|
||||
EthEvmConfig::new(ctx.chain_spec())
|
||||
.with_ingest_dir(ctx.ingest_dir())
|
||||
.with_shared_state(ctx.shared_state()),
|
||||
ctx,
|
||||
pool,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,6 +13,8 @@ workspace = true
|
||||
[dependencies]
|
||||
alloy-primitives.workspace = true
|
||||
serde.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "macros", "time", "rt-multi-thread"] }
|
||||
parking_lot.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
clap.workspace = true
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
use alloy_primitives::Bytes;
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use alloy_primitives::{Address, Bytes};
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)]
|
||||
@ -14,3 +17,6 @@ pub enum ReadPrecompileResult {
|
||||
Error,
|
||||
UnexpectedError,
|
||||
}
|
||||
|
||||
pub type PrecompilesCache =
|
||||
Arc<Mutex<BTreeMap<u64, Vec<(Address, Vec<(ReadPrecompileInput, ReadPrecompileResult)>)>>>>;
|
||||
|
||||
@ -55,6 +55,7 @@ reth-tokio-util.workspace = true
|
||||
reth-tracing.workspace = true
|
||||
reth-transaction-pool.workspace = true
|
||||
reth-basic-payload-builder.workspace = true
|
||||
reth-hyperliquid-types.workspace = true
|
||||
|
||||
## ethereum
|
||||
alloy-consensus.workspace = true
|
||||
|
||||
@ -16,6 +16,7 @@ use reth_cli_util::get_secret_key;
|
||||
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
|
||||
use reth_engine_tree::tree::TreeConfig;
|
||||
use reth_exex::ExExContext;
|
||||
use reth_hyperliquid_types::PrecompilesCache;
|
||||
use reth_network::{
|
||||
transactions::TransactionsManagerConfig, NetworkBuilder, NetworkConfig, NetworkConfigBuilder,
|
||||
NetworkHandle, NetworkManager, NetworkPrimitives,
|
||||
@ -474,6 +475,14 @@ where
|
||||
Self { builder: self.builder.on_rpc_started(hook), task_executor: self.task_executor }
|
||||
}
|
||||
|
||||
/// Add precompiles cache <hyperliquid>
|
||||
pub fn add_precompiles_cache(self, precompile_cache: PrecompilesCache) -> Self {
|
||||
Self {
|
||||
builder: self.builder.add_precompiles_cache(precompile_cache),
|
||||
task_executor: self.task_executor,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the hook that is run to configure the rpc modules.
|
||||
pub fn extend_rpc_modules<F>(self, hook: F) -> Self
|
||||
where
|
||||
@ -587,6 +596,8 @@ pub struct BuilderContext<Node: FullNodeTypes> {
|
||||
pub(crate) executor: TaskExecutor,
|
||||
/// Config container
|
||||
pub(crate) config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
|
||||
/// Shared state
|
||||
pub(crate) shared_state: Option<HyperliquidSharedState>,
|
||||
}
|
||||
|
||||
impl<Node: FullNodeTypes> BuilderContext<Node> {
|
||||
@ -596,8 +607,9 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
|
||||
provider: Node::Provider,
|
||||
executor: TaskExecutor,
|
||||
config_container: WithConfigs<<Node::Types as NodeTypes>::ChainSpec>,
|
||||
shared_state: Option<HyperliquidSharedState>,
|
||||
) -> Self {
|
||||
Self { head, provider, executor, config_container }
|
||||
Self { head, provider, executor, config_container, shared_state }
|
||||
}
|
||||
|
||||
/// Returns the configured provider to interact with the blockchain.
|
||||
@ -754,6 +766,14 @@ impl<Node: FullNodeTypes> BuilderContext<Node> {
|
||||
pub fn ingest_dir(&self) -> PathBuf {
|
||||
self.config().ingest_dir.clone().expect("ingest dir not set")
|
||||
}
|
||||
|
||||
pub fn local_ingest_dir(&self) -> PathBuf {
|
||||
self.config().local_ingest_dir.clone().expect("local ingest dir not set")
|
||||
}
|
||||
|
||||
pub fn shared_state(&self) -> Option<HyperliquidSharedState> {
|
||||
self.shared_state.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl<Node: FullNodeTypes<Types: NodeTypes<ChainSpec: Hardforks>>> BuilderContext<Node> {
|
||||
|
||||
@ -13,6 +13,7 @@ use crate::{
|
||||
AddOns, FullNode,
|
||||
};
|
||||
use reth_exex::ExExContext;
|
||||
use reth_hyperliquid_types::PrecompilesCache;
|
||||
use reth_node_api::{FullNodeComponents, FullNodeTypes, NodeAddOns, NodeTypes};
|
||||
use reth_node_core::node_config::NodeConfig;
|
||||
use reth_tasks::TaskExecutor;
|
||||
@ -47,6 +48,7 @@ impl<T: FullNodeTypes> NodeBuilderWithTypes<T> {
|
||||
adapter,
|
||||
components_builder,
|
||||
add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons: () },
|
||||
shared_state: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -146,6 +148,11 @@ impl<T: FullNodeTypes, C: NodeComponents<T>> Clone for NodeAdapter<T, C> {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HyperliquidSharedState {
|
||||
pub precompiles_cache: PrecompilesCache,
|
||||
}
|
||||
|
||||
/// A fully type configured node builder.
|
||||
///
|
||||
/// Supports adding additional addons to the node.
|
||||
@ -162,6 +169,8 @@ pub struct NodeBuilderWithComponents<
|
||||
pub components_builder: CB,
|
||||
/// Additional node extensions.
|
||||
pub add_ons: AddOns<NodeAdapter<T, CB::Components>, AO>,
|
||||
/// Shared state
|
||||
pub shared_state: Option<HyperliquidSharedState>,
|
||||
}
|
||||
|
||||
impl<T, CB> NodeBuilderWithComponents<T, CB, ()>
|
||||
@ -182,6 +191,7 @@ where
|
||||
adapter,
|
||||
components_builder,
|
||||
add_ons: AddOns { hooks: NodeHooks::default(), exexs: Vec::new(), add_ons },
|
||||
shared_state: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -294,4 +304,10 @@ where
|
||||
add_ons
|
||||
})
|
||||
}
|
||||
|
||||
/// Add state
|
||||
pub fn add_precompiles_cache(mut self, precompiles_cache: PrecompilesCache) -> Self {
|
||||
self.shared_state = Some(HyperliquidSharedState { precompiles_cache });
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@ use std::{sync::Arc, thread::available_parallelism};
|
||||
use crate::{
|
||||
components::{NodeComponents, NodeComponentsBuilder},
|
||||
hooks::OnComponentInitializedHook,
|
||||
BuilderContext, NodeAdapter,
|
||||
BuilderContext, HyperliquidSharedState, NodeAdapter,
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use eyre::{Context, OptionExt};
|
||||
@ -258,6 +258,7 @@ impl<L, R> LaunchContextWith<Attached<L, R>> {
|
||||
&mut self.attachment.right
|
||||
}
|
||||
}
|
||||
|
||||
impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpec>, R>> {
|
||||
/// Adjust certain settings in the config to make sure they are set correctly
|
||||
///
|
||||
@ -655,6 +656,7 @@ where
|
||||
on_component_initialized: Box<
|
||||
dyn OnComponentInitializedHook<NodeAdapter<T, CB::Components>>,
|
||||
>,
|
||||
shared_state: Option<HyperliquidSharedState>,
|
||||
) -> eyre::Result<
|
||||
LaunchContextWith<
|
||||
Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, WithComponents<T, CB>>,
|
||||
@ -671,6 +673,7 @@ where
|
||||
self.blockchain_db().clone(),
|
||||
self.task_executor().clone(),
|
||||
self.configs().clone(),
|
||||
shared_state,
|
||||
);
|
||||
|
||||
debug!(target: "reth::cli", "creating components");
|
||||
@ -790,15 +793,15 @@ where
|
||||
/// This checks for OP-Mainnet and ensures we have all the necessary data to progress (past
|
||||
/// bedrock height)
|
||||
fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> {
|
||||
if self.chain_spec().is_optimism() &&
|
||||
!self.is_dev() &&
|
||||
self.chain_id() == Chain::optimism_mainnet()
|
||||
if self.chain_spec().is_optimism()
|
||||
&& !self.is_dev()
|
||||
&& self.chain_id() == Chain::optimism_mainnet()
|
||||
{
|
||||
let latest = self.blockchain_db().last_block_number()?;
|
||||
// bedrock height
|
||||
if latest < 105235063 {
|
||||
error!("Op-mainnet has been launched without importing the pre-Bedrock state. The chain can't progress without this. See also https://reth.rs/run/sync-op-mainnet.html?minimal-bootstrap-recommended");
|
||||
return Err(ProviderError::BestBlockNotFound)
|
||||
return Err(ProviderError::BestBlockNotFound);
|
||||
}
|
||||
}
|
||||
|
||||
@ -880,7 +883,7 @@ where
|
||||
&self,
|
||||
) -> eyre::Result<Box<dyn InvalidBlockHook<<T::Types as NodeTypes>::Primitives>>> {
|
||||
let Some(ref hook) = self.node_config().debug.invalid_block_hook else {
|
||||
return Ok(Box::new(NoopInvalidBlockHook::default()))
|
||||
return Ok(Box::new(NoopInvalidBlockHook::default()));
|
||||
};
|
||||
let healthy_node_rpc_client = self.get_healthy_node_client()?;
|
||||
|
||||
|
||||
@ -93,7 +93,9 @@ where
|
||||
components_builder,
|
||||
add_ons: AddOns { hooks, exexs: installed_exex, add_ons },
|
||||
config,
|
||||
shared_state,
|
||||
} = target;
|
||||
|
||||
let NodeHooks { on_component_initialized, on_node_started, .. } = hooks;
|
||||
|
||||
// setup the launch context
|
||||
@ -126,7 +128,7 @@ where
|
||||
.with_blockchain_db::<T, _>(move |provider_factory| {
|
||||
Ok(BlockchainProvider::new(provider_factory)?)
|
||||
})?
|
||||
.with_components(components_builder, on_component_initialized).await?;
|
||||
.with_components(components_builder, on_component_initialized, shared_state.clone()).await?;
|
||||
|
||||
// spawn exexs
|
||||
let exex_manager_handle = ExExLauncher::new(
|
||||
|
||||
@ -148,6 +148,9 @@ pub struct NodeConfig<ChainSpec> {
|
||||
|
||||
/// The ingest directory for the node.
|
||||
pub ingest_dir: Option<PathBuf>,
|
||||
|
||||
/// The local ingest directory for the node.
|
||||
pub local_ingest_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl NodeConfig<ChainSpec> {
|
||||
@ -178,6 +181,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
datadir: DatadirArgs::default(),
|
||||
engine: EngineArgs::default(),
|
||||
ingest_dir: None,
|
||||
local_ingest_dir: None,
|
||||
}
|
||||
}
|
||||
|
||||
@ -363,7 +367,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
// try to look up the header in the database
|
||||
if let Some(header) = header {
|
||||
info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database");
|
||||
return Ok(header.number())
|
||||
return Ok(header.number());
|
||||
}
|
||||
|
||||
Ok(self.fetch_tip_from_network(client, tip.into()).await.number())
|
||||
@ -386,7 +390,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
match get_single_header(&client, tip).await {
|
||||
Ok(tip_header) => {
|
||||
info!(target: "reth::cli", ?tip, "Successfully fetched tip");
|
||||
return tip_header
|
||||
return tip_header;
|
||||
}
|
||||
Err(error) => {
|
||||
fetch_failures += 1;
|
||||
@ -470,6 +474,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
pruning: self.pruning,
|
||||
engine: self.engine,
|
||||
ingest_dir: self.ingest_dir,
|
||||
local_ingest_dir: self.local_ingest_dir,
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -498,6 +503,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
|
||||
datadir: self.datadir.clone(),
|
||||
engine: self.engine.clone(),
|
||||
ingest_dir: self.ingest_dir.clone(),
|
||||
local_ingest_dir: self.local_ingest_dir.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -240,7 +240,7 @@ where
|
||||
if attributes.custom == 0 {
|
||||
return Err(EngineObjectValidationError::invalid_params(
|
||||
CustomError::CustomFieldIsNotZero,
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@ -451,7 +451,5 @@ async fn main() -> eyre::Result<()> {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("Node started");
|
||||
|
||||
handle.node_exit_future.await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user