jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: remove StaticFileProvider field from StaticFileProducer (#8583)

Browse Source
This commit is contained in:
joshieDo
2024-06-04 14:29:26 +02:00
committed by GitHub
parent 60f6657988
commit 6dd8400f12
12 changed files with 50 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bin/reth/src/commands/debug_cmd/execution.rs
View File

@ -31,7 +31,7 @@ use reth_primitives::{
};
use reth_provider::{
providers::StaticFileProvider, BlockExecutionWriter, HeaderSyncMode, ProviderFactory,
StageCheckpointReader, StaticFileProviderFactory,
StageCheckpointReader,
};
use reth_stages::{
sets::DefaultStages,
@ -223,11 +223,8 @@ impl Command {
)
.await?;
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Configure the pipeline
let fetch_client = network.fetch_client().await?;

8
bin/reth/src/commands/debug_cmd/replay_engine.rs
View File

@ -25,7 +25,7 @@ use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService};
use reth_primitives::{ChainSpec, PruneModes};
use reth_provider::{
providers::{BlockchainProvider, StaticFileProvider},
CanonStateSubscriptions, ProviderFactory, StaticFileProviderFactory,
CanonStateSubscriptions, ProviderFactory,
};
use reth_stages::Pipeline;
use reth_static_file::StaticFileProducer;
@ -181,11 +181,7 @@ impl Command {
network_client,
Pipeline::builder().build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
),
blockchain_db.clone(),
Box::new(ctx.task_executor.clone()),

7
bin/reth/src/commands/import.rs
View File

@ -30,7 +30,6 @@ use reth_primitives::{stage::StageId, ChainSpec, PruneModes, B256};
use reth_provider::{
providers::StaticFileProvider, BlockNumReader, ChainSpecProvider, HeaderProvider,
HeaderSyncMode, ProviderError, ProviderFactory, StageCheckpointReader,
StaticFileProviderFactory,
};
use reth_stages::{prelude::*, Pipeline, StageSet};
use reth_static_file::StaticFileProducer;
@ -146,11 +145,7 @@ impl ImportCommand {
provider_factory.clone(),
&consensus,
Arc::new(file_client),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
self.no_state,
)
.await?;

11
bin/reth/src/commands/import_op.rs
View File

@ -20,10 +20,7 @@ use reth_downloaders::file_client::{
use reth_node_core::args::DatadirArgs;
use reth_optimism_primitives::bedrock_import::is_dup_tx;
use reth_primitives::{stage::StageId, PruneModes};
use reth_provider::{
providers::StaticFileProvider, ProviderFactory, StageCheckpointReader,
StaticFileProviderFactory,
};
use reth_provider::{providers::StaticFileProvider, ProviderFactory, StageCheckpointReader};
use reth_static_file::StaticFileProducer;
use std::{path::PathBuf, sync::Arc};
use tracing::{debug, error, info};
@ -134,11 +131,7 @@ impl ImportOpCommand {
provider_factory.clone(),
&consensus,
Arc::new(file_client),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
true,
)
.await?;

6
bin/reth/src/commands/stage/unwind.rs
View File

@ -158,11 +158,7 @@ impl Command {
)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory, PruneModes::default()),
);
Ok(pipeline)
}

8
crates/consensus/beacon/src/engine/sync.rs
View File

@ -436,7 +436,6 @@ mod tests {
};
use reth_provider::{
test_utils::create_test_provider_factory_with_chain_spec, BundleStateWithReceipts,
StaticFileProviderFactory,
};
use reth_stages::{test_utils::TestStages, ExecOutput, StageError};
use reth_static_file::StaticFileProducer;
@ -499,11 +498,8 @@ mod tests {
let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec);
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
pipeline.build(provider_factory, static_file_producer)
}

9
crates/consensus/beacon/src/engine/test_utils.rs
View File

@ -24,7 +24,7 @@ use reth_payload_builder::test_utils::spawn_test_payload_service;
use reth_primitives::{BlockNumber, ChainSpec, FinishedExExHeight, PruneModes, B256};
use reth_provider::{
providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec,
BundleStateWithReceipts, HeaderSyncMode, StaticFileProviderFactory,
BundleStateWithReceipts, HeaderSyncMode,
};
use reth_prune::Pruner;
use reth_rpc_types::engine::{
@ -349,11 +349,8 @@ where
}
};
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Setup pipeline
let (tip_tx, tip_rx) = watch::channel(B256::default());

2
crates/node/builder/src/launch/common.rs
View File

@ -373,7 +373,6 @@ where
factory.clone(),
StaticFileProducer::new(
factory.clone(),
factory.static_file_provider(),
self.prune_modes().unwrap_or_default(),
),
);
@ -432,7 +431,6 @@ where
pub fn static_file_producer(&self) -> StaticFileProducer<DB> {
StaticFileProducer::new(
self.provider_factory().clone(),
self.static_file_provider(),
self.prune_modes().unwrap_or_default(),
)
}

36
crates/stages/api/src/pipeline/mod.rs
View File

@ -647,11 +647,7 @@ mod tests {
.with_max_block(10)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let events = pipeline.events();
@ -726,11 +722,7 @@ mod tests {
.with_max_block(10)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let events = pipeline.events();
@ -858,11 +850,7 @@ mod tests {
.with_max_block(10)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let events = pipeline.events();
@ -972,11 +960,7 @@ mod tests {
.with_max_block(10)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let events = pipeline.events();
@ -1083,11 +1067,7 @@ mod tests {
.with_max_block(10)
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let result = pipeline.run().await;
assert_matches!(result, Ok(()));
@ -1100,11 +1080,7 @@ mod tests {
)))
.build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
StaticFileProducer::new(provider_factory.clone(), PruneModes::default()),
);
let result = pipeline.run().await;
assert_matches!(

1
crates/stages/stages/src/lib.rs
View File

@ -48,7 +48,6 @@
//! # let executor_provider = EthExecutorProvider::mainnet();
//! # let static_file_producer = StaticFileProducer::new(
//! # provider_factory.clone(),
//! # provider_factory.static_file_provider(),
//! # PruneModes::default()
//! # );
//! // Create a pipeline that can fully sync

7
crates/stages/stages/src/sets.rs
View File

@ -23,11 +23,8 @@
//! # fn create(exec: impl BlockExecutorProvider) {
//!
//! let provider_factory = create_test_provider_factory();
//! let static_file_producer = StaticFileProducer::new(
//! provider_factory.clone(),
//! provider_factory.static_file_provider(),
//! PruneModes::default(),
//! );
//! let static_file_producer =
//! StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
//! // Build a pipeline with all offline stages.
//! let pipeline = Pipeline::builder()
//! .add_stages(OfflineStages::new(exec, StageConfig::default(), PruneModes::default()))

80
crates/static-file/src/static_file_producer.rs
View File

@ -5,10 +5,7 @@ use parking_lot::Mutex;
use rayon::prelude::*;
use reth_db::database::Database;
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber, PruneModes};
use reth_provider::{
providers::{StaticFileProvider, StaticFileWriter},
ProviderFactory,
};
use reth_provider::{providers::StaticFileWriter, ProviderFactory, StaticFileProviderFactory};
use reth_storage_errors::provider::ProviderResult;
use reth_tokio_util::{EventSender, EventStream};
use std::{
@ -31,16 +28,8 @@ pub struct StaticFileProducer<DB>(Arc<Mutex<StaticFileProducerInner<DB>>>);
impl<DB: Database> StaticFileProducer<DB> {
/// Creates a new [`StaticFileProducer`].
pub fn new(
provider_factory: ProviderFactory<DB>,
static_file_provider: StaticFileProvider,
prune_modes: PruneModes,
) -> Self {
Self(Arc::new(Mutex::new(StaticFileProducerInner::new(
provider_factory,
static_file_provider,
prune_modes,
))))
pub fn new(provider_factory: ProviderFactory<DB>, prune_modes: PruneModes) -> Self {
Self(Arc::new(Mutex::new(StaticFileProducerInner::new(provider_factory, prune_modes))))
}
}
@ -58,8 +47,6 @@ impl<DB> Deref for StaticFileProducer<DB> {
pub struct StaticFileProducerInner<DB> {
/// Provider factory
provider_factory: ProviderFactory<DB>,
/// Static File provider
static_file_provider: StaticFileProvider,
/// Pruning configuration for every part of the data that can be pruned. Set by user, and
/// needed in [`StaticFileProducerInner`] to prevent attempting to move prunable data to static
/// files. See [`StaticFileProducerInner::get_static_file_targets`].
@ -102,17 +89,8 @@ impl StaticFileTargets {
}
impl<DB: Database> StaticFileProducerInner<DB> {
fn new(
provider_factory: ProviderFactory<DB>,
static_file_provider: StaticFileProvider,
prune_modes: PruneModes,
) -> Self {
Self {
provider_factory,
static_file_provider,
prune_modes,
event_sender: Default::default(),
}
fn new(provider_factory: ProviderFactory<DB>, prune_modes: PruneModes) -> Self {
Self { provider_factory, prune_modes, event_sender: Default::default() }
}
/// Listen for events on the `static_file_producer`.
@ -123,8 +101,9 @@ impl<DB: Database> StaticFileProducerInner<DB> {
/// Run the `static_file_producer`.
///
/// For each [Some] target in [`StaticFileTargets`], initializes a corresponding [Segment] and
/// runs it with the provided block range using [`StaticFileProvider`] and a read-only
/// database transaction from [`ProviderFactory`]. All segments are run in parallel.
/// runs it with the provided block range using [`reth_provider::providers::StaticFileProvider`]
/// and a read-only database transaction from [`ProviderFactory`]. All segments are run in
/// parallel.
///
/// NOTE: it doesn't delete the data from database, and the actual deleting (aka pruning) logic
/// lives in the `prune` crate.
@ -135,7 +114,7 @@ impl<DB: Database> StaticFileProducerInner<DB> {
}
debug_assert!(targets.is_contiguous_to_highest_static_files(
self.static_file_provider.get_highest_static_files()
self.provider_factory.static_file_provider().get_highest_static_files()
));
self.event_sender.notify(StaticFileProducerEvent::Started { targets: targets.clone() });
@ -162,7 +141,7 @@ impl<DB: Database> StaticFileProducerInner<DB> {
// Create a new database transaction on every segment to prevent long-lived read-only
// transactions
let provider = self.provider_factory.provider()?.disable_long_read_transaction_safety();
segment.copy_to_static_files(provider, self.static_file_provider.clone(), block_range.clone())?;
segment.copy_to_static_files(provider, self.provider_factory.static_file_provider(), block_range.clone())?;
let elapsed = start.elapsed(); // TODO(alexey): track in metrics
debug!(target: "static_file", segment = %segment.segment(), ?block_range, ?elapsed, "Finished StaticFileProducer segment");
@ -170,9 +149,11 @@ impl<DB: Database> StaticFileProducerInner<DB> {
Ok(())
})?;
self.static_file_provider.commit()?;
self.provider_factory.static_file_provider().commit()?;
for (segment, block_range) in segments {
self.static_file_provider.update_index(segment.segment(), Some(*block_range.end()))?;
self.provider_factory
.static_file_provider()
.update_index(segment.segment(), Some(*block_range.end()))?;
}
let elapsed = start.elapsed(); // TODO(alexey): track in metrics
@ -186,12 +167,13 @@ impl<DB: Database> StaticFileProducerInner<DB> {
/// Returns a static file targets at the provided finalized block numbers per segment.
/// The target is determined by the check against highest `static_files` using
/// [`StaticFileProvider::get_highest_static_files`].
/// [`reth_provider::providers::StaticFileProvider::get_highest_static_files`].
pub fn get_static_file_targets(
&self,
finalized_block_numbers: HighestStaticFiles,
) -> ProviderResult<StaticFileTargets> {
let highest_static_files = self.static_file_provider.get_highest_static_files();
let highest_static_files =
self.provider_factory.static_file_provider().get_highest_static_files();
let targets = StaticFileTargets {
headers: finalized_block_numbers.headers.and_then(|finalized_block_number| {
@ -251,8 +233,7 @@ mod tests {
static_file::HighestStaticFiles, PruneModes, StaticFileSegment, B256, U256,
};
use reth_provider::{
providers::{StaticFileProvider, StaticFileWriter},
ProviderError, ProviderFactory, StaticFileProviderFactory,
providers::StaticFileWriter, ProviderError, ProviderFactory, StaticFileProviderFactory,
};
use reth_stages::test_utils::{StorageKind, TestStageDB};
use reth_testing_utils::{
@ -265,7 +246,7 @@ mod tests {
};
use tempfile::TempDir;
fn setup() -> (ProviderFactory<Arc<TempDatabase<DatabaseEnv>>>, StaticFileProvider, TempDir) {
fn setup() -> (ProviderFactory<Arc<TempDatabase<DatabaseEnv>>>, TempDir) {
let mut rng = generators::rng();
let db = TestStageDB::default();
@ -297,19 +278,15 @@ mod tests {
db.insert_receipts(receipts).expect("insert receipts");
let provider_factory = db.factory;
let static_file_provider = provider_factory.static_file_provider();
(provider_factory, static_file_provider, db.temp_static_files_dir)
(provider_factory, db.temp_static_files_dir)
}
#[test]
fn run() {
let (provider_factory, static_file_provider, _temp_static_files_dir) = setup();
let (provider_factory, _temp_static_files_dir) = setup();
let static_file_producer = StaticFileProducerInner::new(
provider_factory,
static_file_provider.clone(),
PruneModes::default(),
);
let static_file_producer =
StaticFileProducerInner::new(provider_factory.clone(), PruneModes::default());
let targets = static_file_producer
.get_static_file_targets(HighestStaticFiles {
@ -328,7 +305,7 @@ mod tests {
);
assert_matches!(static_file_producer.run(targets), Ok(_));
assert_eq!(
static_file_provider.get_highest_static_files(),
provider_factory.static_file_provider().get_highest_static_files(),
HighestStaticFiles { headers: Some(1), receipts: Some(1), transactions: Some(1) }
);
@ -349,7 +326,7 @@ mod tests {
);
assert_matches!(static_file_producer.run(targets), Ok(_));
assert_eq!(
static_file_provider.get_highest_static_files(),
provider_factory.static_file_provider().get_highest_static_files(),
HighestStaticFiles { headers: Some(3), receipts: Some(3), transactions: Some(3) }
);
@ -373,7 +350,7 @@ mod tests {
Err(ProviderError::BlockBodyIndicesNotFound(4))
);
assert_eq!(
static_file_provider.get_highest_static_files(),
provider_factory.static_file_provider().get_highest_static_files(),
HighestStaticFiles { headers: Some(3), receipts: Some(3), transactions: Some(3) }
);
}
@ -381,10 +358,9 @@ mod tests {
/// Tests that a cloneable [`StaticFileProducer`] type is not susceptible to any race condition.
#[test]
fn only_one() {
let (provider_factory, static_file_provider, _temp_static_files_dir) = setup();
let (provider_factory, _temp_static_files_dir) = setup();
let static_file_producer =
StaticFileProducer::new(provider_factory, static_file_provider, PruneModes::default());
let static_file_producer = StaticFileProducer::new(provider_factory, PruneModes::default());
let (tx, rx) = channel();