jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add --local.fallback-threshold, --s3.polling-interval

Browse Source
This commit is contained in:
sprites0
2025-08-25 22:25:12 -04:00
parent 0f9c2c5897
commit 52909eea3f
8 changed files with 121 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/pseudo_peer/sources/cached.rs
View File

@ -41,4 +41,8 @@ impl BlockSource for CachedBlockSource {
fn recommended_chunk_size(&self) -> u64 {
self.block_source.recommended_chunk_size()
}
fn polling_interval(&self) -> std::time::Duration {
self.block_source.polling_interval()
}
}

38
src/pseudo_peer/sources/hl_node/mod.rs
View File

@ -14,35 +14,39 @@ use self::{
use super::{BlockSource, BlockSourceBoxed};
use crate::node::types::BlockAndReceipts;
use futures::future::BoxFuture;
use serde::{Deserialize, Serialize};
use std::{
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use time::{Duration, OffsetDateTime};
use time::OffsetDateTime;
use tokio::sync::Mutex;
use tracing::{info, warn};
const TAIL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(25);
const HOURLY_SUBDIR: &str = "hourly";
const CACHE_SIZE: u32 = 8000; // 3660 blocks per hour
const MAX_ALLOWED_THRESHOLD_BEFORE_FALLBACK: Duration = Duration::milliseconds(5000);
const ONE_HOUR: Duration = Duration::from_secs(60 * 60);
const TAIL_INTERVAL: Duration = Duration::from_millis(25);
#[derive(Serialize, Deserialize, Debug, Clone)]
pub(crate) struct LocalBlockAndReceipts(String, BlockAndReceipts);
#[derive(Debug, Clone)]
pub struct HlNodeBlockSourceArgs {
pub root: PathBuf,
pub fallback_threshold: Duration,
}
/// Block source that monitors the local ingest directory for the HL node.
#[derive(Debug, Clone)]
pub struct HlNodeBlockSource {
pub fallback: BlockSourceBoxed,
pub local_ingest_dir: PathBuf,
pub local_blocks_cache: Arc<Mutex<LocalBlocksCache>>,
pub last_local_fetch: Arc<Mutex<Option<(u64, OffsetDateTime)>>>,
pub args: HlNodeBlockSourceArgs,
}
impl BlockSource for HlNodeBlockSource {
fn collect_block(&self, height: u64) -> BoxFuture<'static, eyre::Result<BlockAndReceipts>> {
let fallback = self.fallback.clone();
let args = self.args.clone();
let local_blocks_cache = self.local_blocks_cache.clone();
let last_local_fetch = self.last_local_fetch.clone();
Box::pin(async move {
@ -55,7 +59,7 @@ impl BlockSource for HlNodeBlockSource {
if let Some((last_height, last_poll_time)) = *last_local_fetch.lock().await {
let more_recent = last_height < height;
let too_soon = now - last_poll_time < MAX_ALLOWED_THRESHOLD_BEFORE_FALLBACK;
let too_soon = now - last_poll_time < args.fallback_threshold;
if more_recent && too_soon {
return Err(eyre::eyre!(
"Not found locally; limiting polling rate before fallback so that hl-node has chance to catch up"
@ -71,12 +75,12 @@ impl BlockSource for HlNodeBlockSource {
fn find_latest_block_number(&self) -> BoxFuture<'static, Option<u64>> {
let fallback = self.fallback.clone();
let local_ingest_dir = self.local_ingest_dir.clone();
let args = self.args.clone();
Box::pin(async move {
let Some(dir) = FileOperations::find_latest_hourly_file(&local_ingest_dir) else {
let Some(dir) = FileOperations::find_latest_hourly_file(&args.root) else {
warn!(
"No EVM blocks from hl-node found at {:?}; fallback to s3/ingest-dir",
local_ingest_dir
args.root
);
return fallback.find_latest_block_number().await;
};
@ -160,7 +164,7 @@ impl HlNodeBlockSource {
}
async fn start_local_ingest_loop(&self, current_head: u64) {
let root = self.local_ingest_dir.to_owned();
let root = self.args.root.to_owned();
let cache = self.local_blocks_cache.clone();
tokio::spawn(async move {
let mut next_height = current_head;
@ -185,8 +189,8 @@ impl HlNodeBlockSource {
cache.lock().await.load_scan_result(scan_result);
}
let now = OffsetDateTime::now_utc();
if dt + Duration::HOUR < now {
dt += Duration::HOUR;
if dt + ONE_HOUR < now {
dt += ONE_HOUR;
(hour, day_str, last_line) = (dt.hour(), TimeUtils::date_from_datetime(dt), 0);
info!(
"Moving to new file: {:?}",
@ -201,7 +205,7 @@ impl HlNodeBlockSource {
pub(crate) async fn run(&self, next_block_number: u64) -> eyre::Result<()> {
let _ = Self::try_backfill_local_blocks(
&self.local_ingest_dir,
&self.args.root,
&self.local_blocks_cache,
next_block_number,
)
@ -212,12 +216,12 @@ impl HlNodeBlockSource {
pub async fn new(
fallback: BlockSourceBoxed,
local_ingest_dir: PathBuf,
args: HlNodeBlockSourceArgs,
next_block_number: u64,
) -> Self {
let block_source = Self {
fallback,
local_ingest_dir,
args,
local_blocks_cache: Arc::new(Mutex::new(LocalBlocksCache::new(CACHE_SIZE))),
last_local_fetch: Arc::new(Mutex::new(None)),
};

20
src/pseudo_peer/sources/hl_node/tests.rs
View File

@ -1,11 +1,13 @@
use super::*;
use crate::{
node::types::{reth_compat, ReadPrecompileCalls},
pseudo_peer::sources::LocalBlockSource,
pseudo_peer::sources::{hl_node::scan::LocalBlockAndReceipts, LocalBlockSource},
};
use alloy_consensus::{BlockBody, Header};
use alloy_primitives::{Address, Bloom, Bytes, B256, B64, U256};
use std::{io::Write, time::Duration as StdDuration};
use std::{io::Write, time::Duration};
const DEFAULT_FALLBACK_THRESHOLD_FOR_TEST: Duration = Duration::from_millis(5000);
#[test]
fn test_datetime_from_path() {
@ -111,7 +113,10 @@ async fn setup_block_source_hierarchy() -> eyre::Result<BlockSourceHierarchy> {
// Setup fallback block source
let block_source_fallback = HlNodeBlockSource::new(
BlockSourceBoxed::new(Box::new(LocalBlockSource::new("/nonexistent"))),
PathBuf::from("/nonexistent"),
HlNodeBlockSourceArgs {
root: { PathBuf::from("/nonexistent") },
fallback_threshold: DEFAULT_FALLBACK_THRESHOLD_FOR_TEST,
},
1000000,
)
.await;
@ -124,7 +129,10 @@ async fn setup_block_source_hierarchy() -> eyre::Result<BlockSourceHierarchy> {
let block_source = HlNodeBlockSource::new(
BlockSourceBoxed::new(Box::new(block_source_fallback.clone())),
temp_dir1.path().to_path_buf(),
HlNodeBlockSourceArgs {
root: temp_dir1.path().to_path_buf(),
fallback_threshold: DEFAULT_FALLBACK_THRESHOLD_FOR_TEST,
},
1000000,
)
.await;
@ -159,7 +167,7 @@ async fn test_update_last_fetch_no_fallback() -> eyre::Result<()> {
assert!(block.is_err());
writeln!(&mut file1, "{}", serde_json::to_string(&future_block_hl_node)?)?;
tokio::time::sleep(StdDuration::from_millis(100)).await;
tokio::time::sleep(Duration::from_millis(100)).await;
let block = block_source.collect_block(1000001).await.unwrap();
assert_eq!(block, future_block_hl_node.1);
@ -177,7 +185,7 @@ async fn test_update_last_fetch_fallback() -> eyre::Result<()> {
let block = block_source.collect_block(1000000).await.unwrap();
assert_eq!(block, current_block.1);
tokio::time::sleep(MAX_ALLOWED_THRESHOLD_BEFORE_FALLBACK.unsigned_abs()).await;
tokio::time::sleep(DEFAULT_FALLBACK_THRESHOLD_FOR_TEST).await;
writeln!(&mut file1, "{}", serde_json::to_string(&future_block_fallback)?)?;
let block = block_source.collect_block(1000001).await.unwrap();

27
src/pseudo_peer/sources/mod.rs
View File

@ -1,6 +1,7 @@
use crate::node::types::BlockAndReceipts;
use auto_impl::auto_impl;
use futures::future::BoxFuture;
use std::sync::Arc;
use std::{sync::Arc, time::Duration};
// Module declarations
mod cached;
@ -11,11 +12,14 @@ mod utils;
// Public exports
pub use cached::CachedBlockSource;
pub use hl_node::HlNodeBlockSource;
pub use hl_node::{HlNodeBlockSource, HlNodeBlockSourceArgs};
pub use local::LocalBlockSource;
pub use s3::S3BlockSource;
const DEFAULT_POLLING_INTERVAL: Duration = Duration::from_millis(25);
/// Trait for block sources that can retrieve blocks from various sources
#[auto_impl(&, &mut, Box, Arc)]
pub trait BlockSource: Send + Sync + std::fmt::Debug + Unpin + 'static {
/// Retrieves a block at the specified height
fn collect_block(&self, height: u64) -> BoxFuture<'static, eyre::Result<BlockAndReceipts>>;
@ -25,21 +29,12 @@ pub trait BlockSource: Send + Sync + std::fmt::Debug + Unpin + 'static {
/// Returns the recommended chunk size for batch operations
fn recommended_chunk_size(&self) -> u64;
/// Returns the polling interval
fn polling_interval(&self) -> Duration {
DEFAULT_POLLING_INTERVAL
}
}
/// Type alias for a boxed block source
pub type BlockSourceBoxed = Arc<Box<dyn BlockSource>>;
impl BlockSource for BlockSourceBoxed {
fn collect_block(&self, height: u64) -> BoxFuture<'static, eyre::Result<BlockAndReceipts>> {
self.as_ref().collect_block(height)
}
fn find_latest_block_number(&self) -> BoxFuture<'static, Option<u64>> {
self.as_ref().find_latest_block_number()
}
fn recommended_chunk_size(&self) -> u64 {
self.as_ref().recommended_chunk_size()
}
}

11
src/pseudo_peer/sources/s3.rs
View File

@ -2,7 +2,7 @@ use super::{utils, BlockSource};
use crate::node::types::BlockAndReceipts;
use aws_sdk_s3::types::RequestPayer;
use futures::{future::BoxFuture, FutureExt};
use std::sync::Arc;
use std::{sync::Arc, time::Duration};
use tracing::info;
/// Block source that reads blocks from S3 (--s3)
@ -10,11 +10,12 @@ use tracing::info;
pub struct S3BlockSource {
client: Arc<aws_sdk_s3::Client>,
bucket: String,
polling_interval: Duration,
}
impl S3BlockSource {
pub fn new(client: aws_sdk_s3::Client, bucket: String) -> Self {
Self { client: client.into(), bucket }
pub fn new(client: aws_sdk_s3::Client, bucket: String, polling_interval: Duration) -> Self {
Self { client: client.into(), bucket, polling_interval }
}
async fn pick_path_with_highest_number(
@ -87,4 +88,8 @@ impl BlockSource for S3BlockSource {
fn recommended_chunk_size(&self) -> u64 {
1000
}
fn polling_interval(&self) -> Duration {
self.polling_interval
}
}