From 405b73045549611c3381020cef9302b4ee3f483c Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:40:07 -0400 Subject: [PATCH] chore: remove prune_modes from BlockWriter (#9231) --- .../src/commands/debug_cmd/build_block.rs | 8 +++- .../commands/debug_cmd/in_memory_merkle.rs | 1 - bin/reth/src/commands/debug_cmd/merkle.rs | 2 +- .../src/commands/debug_cmd/replay_engine.rs | 8 +++- crates/blockchain-tree/src/blockchain_tree.rs | 38 +++++++++--------- crates/cli/commands/src/common.rs | 7 ++-- crates/consensus/beacon/src/engine/mod.rs | 1 - .../consensus/beacon/src/engine/test_utils.rs | 3 +- crates/engine/tree/src/persistence.rs | 9 +---- crates/exex/exex/src/backfill.rs | 1 - crates/node/builder/src/launch/common.rs | 19 ++++----- crates/stages/stages/src/stages/execution.rs | 35 ++++++----------- .../stages/src/stages/hashing_account.rs | 2 +- crates/stages/stages/src/stages/headers.rs | 1 - crates/stages/stages/src/stages/mod.rs | 10 ++--- .../provider/src/providers/database/mod.rs | 39 ++++++++++++------- .../src/providers/database/provider.rs | 26 +++++++------ crates/storage/provider/src/traits/block.rs | 10 +---- testing/ef-tests/src/cases/blockchain_test.rs | 2 - 19 files changed, 105 insertions(+), 117 deletions(-) diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index 69e0dc1cf..afaf79964 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -29,6 +29,7 @@ use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, BlockWriter, ChainSpecProvider, ProviderFactory, StageCheckpointReader, StateProviderFactory, }; +use reth_prune::PruneModes; use reth_revm::{database::StateProviderDatabase, primitives::EnvKzgSettings}; use reth_rpc_types::engine::{BlobsBundleV1, PayloadAttributes}; use reth_stages::StageId; @@ -122,7 +123,11 @@ impl Command { // configure blockchain tree let tree_externals = TreeExternals::new(provider_factory.clone(), Arc::clone(&consensus), executor); - let tree = BlockchainTree::new(tree_externals, BlockchainTreeConfig::default(), None)?; + let tree = BlockchainTree::new( + tree_externals, + BlockchainTreeConfig::default(), + PruneModes::none(), + )?; let blockchain_tree = Arc::new(ShareableBlockchainTree::new(tree)); // fetch the best block from the database @@ -298,7 +303,6 @@ impl Command { execution_outcome, hashed_post_state, trie_updates, - None, )?; info!(target: "reth::cli", "Successfully appended built block"); } diff --git a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs index 352d7846e..bd97bfcf3 100644 --- a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs +++ b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs @@ -163,7 +163,6 @@ impl Command { .clone() .try_seal_with_senders() .map_err(|_| BlockValidationError::SenderRecoveryError)?, - None, )?; execution_outcome.write_to_storage(provider_rw.tx_ref(), None, OriginalValuesKnown::No)?; let storage_lists = provider_rw.changed_storages_with_range(block.number..=block.number)?; diff --git a/bin/reth/src/commands/debug_cmd/merkle.rs b/bin/reth/src/commands/debug_cmd/merkle.rs index e1d044e66..cfd787375 100644 --- a/bin/reth/src/commands/debug_cmd/merkle.rs +++ b/bin/reth/src/commands/debug_cmd/merkle.rs @@ -144,7 +144,7 @@ impl Command { .map_err(|block| eyre::eyre!("Error sealing block with senders: {block:?}"))?; trace!(target: "reth::cli", block_number, "Executing block"); - provider_rw.insert_block(sealed_block.clone(), None)?; + provider_rw.insert_block(sealed_block.clone())?; td += sealed_block.difficulty; let mut executor = executor_provider.batch_executor( diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index 429ef625f..02a7205a4 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -85,7 +85,11 @@ impl Command { // Configure blockchain tree let tree_externals = TreeExternals::new(provider_factory.clone(), Arc::clone(&consensus), executor); - let tree = BlockchainTree::new(tree_externals, BlockchainTreeConfig::default(), None)?; + let tree = BlockchainTree::new( + tree_externals, + BlockchainTreeConfig::default(), + PruneModes::none(), + )?; let blockchain_tree = Arc::new(ShareableBlockchainTree::new(tree)); // Set up the blockchain provider @@ -144,7 +148,7 @@ impl Command { network_client, Pipeline::builder().build( provider_factory.clone(), - StaticFileProducer::new(provider_factory.clone(), PruneModes::default()), + StaticFileProducer::new(provider_factory.clone(), PruneModes::none()), ), blockchain_db.clone(), Box::new(ctx.task_executor.clone()), diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 66ed04478..da0a147ac 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -65,8 +65,6 @@ pub struct BlockchainTree { externals: TreeExternals, /// Tree configuration config: BlockchainTreeConfig, - /// Prune modes. - prune_modes: Option, /// Broadcast channel for canon state changes notifications. canon_state_notification_sender: CanonStateNotificationSender, /// Metrics for sync stages. @@ -115,9 +113,9 @@ where /// storage space efficiently. It's important to validate this configuration to ensure it does /// not lead to unintended data loss. pub fn new( - externals: TreeExternals, + mut externals: TreeExternals, config: BlockchainTreeConfig, - prune_modes: Option, + prune_modes: PruneModes, ) -> ProviderResult { let max_reorg_depth = config.max_reorg_depth() as usize; // The size of the broadcast is twice the maximum reorg depth, because at maximum reorg @@ -125,6 +123,9 @@ where let (canon_state_notification_sender, _receiver) = tokio::sync::broadcast::channel(max_reorg_depth * 2); + // Set the prune modes argument, on the provider + externals.provider_factory = externals.provider_factory.with_prune_modes(prune_modes); + let last_canonical_hashes = externals.fetch_latest_canonical_hashes(config.num_of_canonical_hashes() as usize)?; @@ -138,7 +139,6 @@ where config.max_unconnected_blocks(), ), config, - prune_modes, canon_state_notification_sender, sync_metrics_tx: None, metrics: Default::default(), @@ -1258,7 +1258,6 @@ where state, hashed_state, trie_updates, - self.prune_modes.as_ref(), ) .map_err(|e| CanonicalError::CanonicalCommit(e.to_string()))?; @@ -1424,7 +1423,6 @@ mod tests { provider .insert_historical_block( genesis.try_seal_with_senders().expect("invalid tx signature in genesis"), - None, ) .unwrap(); @@ -1545,7 +1543,6 @@ mod tests { SealedBlock::new(chain_spec.sealed_genesis_header(), Default::default()) .try_seal_with_senders() .unwrap(), - None, ) .unwrap(); let account = Account { balance: initial_signer_balance, ..Default::default() }; @@ -1647,7 +1644,7 @@ mod tests { let mut tree = BlockchainTree::new( TreeExternals::new(provider_factory, consensus, executor_provider), BlockchainTreeConfig::default(), - None, + PruneModes::default(), ) .expect("failed to create tree"); @@ -1727,7 +1724,8 @@ mod tests { // make tree let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let mut tree = BlockchainTree::new(externals, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"); // genesis block 10 is already canonical tree.make_canonical(B256::ZERO).unwrap(); @@ -1803,7 +1801,8 @@ mod tests { // make tree let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let mut tree = BlockchainTree::new(externals, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"); // genesis block 10 is already canonical tree.make_canonical(B256::ZERO).unwrap(); @@ -1888,7 +1887,8 @@ mod tests { // make tree let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let mut tree = BlockchainTree::new(externals, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"); // genesis block 10 is already canonical tree.make_canonical(B256::ZERO).unwrap(); @@ -1986,7 +1986,8 @@ mod tests { // make tree let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let mut tree = BlockchainTree::new(externals, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"); let mut canon_notif = tree.subscribe_canon_state(); // genesis block 10 is already canonical @@ -2379,7 +2380,8 @@ mod tests { // make tree let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let mut tree = BlockchainTree::new(externals, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"); assert_eq!( tree.insert_block(block1.clone(), BlockValidationKind::Exhaustive).unwrap(), @@ -2399,8 +2401,8 @@ mod tests { tree.make_canonical(block2.hash()).unwrap(); // restart - let mut tree = - BlockchainTree::new(cloned_externals_1, config, None).expect("failed to create tree"); + let mut tree = BlockchainTree::new(cloned_externals_1, config, PruneModes::default()) + .expect("failed to create tree"); assert_eq!(tree.block_indices().last_finalized_block(), 0); let mut block1a = block1; @@ -2416,8 +2418,8 @@ mod tests { tree.finalize_block(block1a.number).unwrap(); // restart - let tree = - BlockchainTree::new(cloned_externals_2, config, None).expect("failed to create tree"); + let tree = BlockchainTree::new(cloned_externals_2, config, PruneModes::default()) + .expect("failed to create tree"); assert_eq!(tree.block_indices().last_finalized_block(), block1a.number); } diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 41758a9f0..b382f7312 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -109,7 +109,10 @@ impl EnvironmentArgs { static_file_provider: StaticFileProvider, ) -> eyre::Result>> { let has_receipt_pruning = config.prune.as_ref().map_or(false, |a| a.has_receipts_pruning()); - let factory = ProviderFactory::new(db, self.chain.clone(), static_file_provider); + let prune_modes = + config.prune.as_ref().map(|prune| prune.segments.clone()).unwrap_or_default(); + let factory = ProviderFactory::new(db, self.chain.clone(), static_file_provider) + .with_prune_modes(prune_modes.clone()); info!(target: "reth::cli", "Verifying storage consistency."); @@ -123,8 +126,6 @@ impl EnvironmentArgs { return Ok(factory) } - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); - // Highly unlikely to happen, and given its destructive nature, it's better to panic // instead. assert_ne!(unwind_target, PipelineTarget::Unwind(0), "A static file <> database inconsistency was found that would trigger an unwind to block 0"); diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index eba82c295..b0a0284b6 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2159,7 +2159,6 @@ mod tests { provider .insert_block( b.clone().try_seal_with_senders().expect("invalid tx signature in block"), - None, ) .map(drop) }) diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index f58063d3e..7b5ee65ee 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -393,7 +393,8 @@ where let externals = TreeExternals::new(provider_factory.clone(), consensus, executor_factory); let config = BlockchainTreeConfig::new(1, 2, 3, 2); let tree = Arc::new(ShareableBlockchainTree::new( - BlockchainTree::new(externals, config, None).expect("failed to create tree"), + BlockchainTree::new(externals, config, PruneModes::default()) + .expect("failed to create tree"), )); let latest = self.base_config.chain_spec.genesis_header().seal_slow(); let blockchain_provider = diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 989ed5f76..96268ae64 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -61,16 +61,9 @@ impl Persistence { // * indices (already done basically) // Insert the blocks for block in blocks { - // TODO: prune modes - a bit unsure that it should be at this level of abstraction and - // not another - // - // ie, an external consumer of providers (or the database task) really does not care - // about pruning, just the node. Maybe we are the biggest user, and use it enough that - // we need a helper, but I'd rather make the pruning behavior more explicit then - let prune_modes = None; let sealed_block = block.block().clone().try_with_senders_unchecked(block.senders().clone()).unwrap(); - provider_rw.insert_block(sealed_block, prune_modes)?; + provider_rw.insert_block(sealed_block)?; // Write state and changesets to the database. // Must be written after blocks because of the receipt lookup. diff --git a/crates/exex/exex/src/backfill.rs b/crates/exex/exex/src/backfill.rs index 212082081..ffd87e52d 100644 --- a/crates/exex/exex/src/backfill.rs +++ b/crates/exex/exex/src/backfill.rs @@ -323,7 +323,6 @@ mod tests { outcome_batch, Default::default(), Default::default(), - None, )?; provider_rw.commit()?; diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 83e48927d..804c194f4 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -320,9 +320,9 @@ impl LaunchContextWith> { self.toml_config().prune.clone().or_else(|| self.node_config().prune_config()) } - /// Returns the configured [`PruneModes`] - pub fn prune_modes(&self) -> Option { - self.prune_config().map(|config| config.segments) + /// Returns the configured [`PruneModes`], returning the default if no config was available. + pub fn prune_modes(&self) -> PruneModes { + self.prune_config().map(|config| config.segments).unwrap_or_default() } /// Returns an initialized [`PrunerBuilder`] based on the configured [`PruneConfig`] @@ -364,6 +364,7 @@ where self.chain_spec(), StaticFileProvider::read_write(self.data_dir().static_files())?, ) + .with_prune_modes(self.prune_modes()) .with_static_files_metrics(); let has_receipt_pruning = @@ -395,14 +396,11 @@ where NoopBodiesDownloader::default(), NoopBlockExecutorProvider::default(), self.toml_config().stages.clone(), - self.prune_modes().unwrap_or_default(), + self.prune_modes(), )) .build( factory.clone(), - StaticFileProducer::new( - factory.clone(), - self.prune_modes().unwrap_or_default(), - ), + StaticFileProducer::new(factory.clone(), self.prune_modes()), ); // Unwinds to block @@ -705,10 +703,7 @@ where /// Creates a new [`StaticFileProducer`] with the attached database. pub fn static_file_producer(&self) -> StaticFileProducer { - StaticFileProducer::new( - self.provider_factory().clone(), - self.prune_modes().unwrap_or_default(), - ) + StaticFileProducer::new(self.provider_factory().clone(), self.prune_modes()) } /// Returns the current head block. diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index 4294132c7..6ba6346e2 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -700,12 +700,9 @@ mod tests { .try_seal_with_senders() .map_err(|_| BlockValidationError::SenderRecoveryError) .unwrap(), - None, ) .unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -745,10 +742,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_seal_with_senders().unwrap(), None).unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -788,10 +783,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_seal_with_senders().unwrap(), None).unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -825,10 +818,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_seal_with_senders().unwrap(), None).unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -976,10 +967,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_seal_with_senders().unwrap(), None).unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -1095,10 +1084,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_seal_with_senders().unwrap(), None).unwrap(); - provider - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) - .unwrap(); + provider.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); + provider.insert_historical_block(block.clone().try_seal_with_senders().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index fe8bc8547..6f59be9f4 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -75,7 +75,7 @@ impl AccountHashingStage { let blocks = random_block_range(&mut rng, opts.blocks.clone(), B256::ZERO, opts.txs); for block in blocks { - provider.insert_historical_block(block.try_seal_with_senders().unwrap(), None).unwrap(); + provider.insert_historical_block(block.try_seal_with_senders().unwrap()).unwrap(); } provider .static_file_provider() diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 3d05ea52d..71c57cae4 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -631,7 +631,6 @@ mod tests { ExecutionOutcome::default(), HashedPostState::default(), TrieUpdates::default(), - None, ) .unwrap(); provider.commit().unwrap(); diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 9c96f4b30..4b65523ba 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -84,11 +84,9 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); + provider_rw.insert_historical_block(genesis.try_seal_with_senders().unwrap()).unwrap(); provider_rw - .insert_historical_block(genesis.try_seal_with_senders().unwrap(), None) - .unwrap(); - provider_rw - .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) + .insert_historical_block(block.clone().try_seal_with_senders().unwrap()) .unwrap(); // Fill with bogus blocks to respect PruneMode distance. @@ -97,9 +95,7 @@ mod tests { for block_number in 2..=tip { let nblock = random_block(&mut rng, block_number, Some(head), Some(0), Some(0)); head = nblock.hash(); - provider_rw - .insert_historical_block(nblock.try_seal_with_senders().unwrap(), None) - .unwrap(); + provider_rw.insert_historical_block(nblock.try_seal_with_senders().unwrap()).unwrap(); } provider_rw .static_file_provider() diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 9c65103a4..c8d97a6e9 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -18,7 +18,7 @@ use reth_primitives::{ TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256, }; -use reth_prune_types::{PruneCheckpoint, PruneSegment}; +use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_errors::provider::ProviderResult; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; @@ -46,6 +46,8 @@ pub struct ProviderFactory { chain_spec: Arc, /// Static File Provider static_file_provider: StaticFileProvider, + /// Optional pruning configuration + prune_modes: PruneModes, } impl ProviderFactory { @@ -55,7 +57,7 @@ impl ProviderFactory { chain_spec: Arc, static_file_provider: StaticFileProvider, ) -> Self { - Self { db: Arc::new(db), chain_spec, static_file_provider } + Self { db: Arc::new(db), chain_spec, static_file_provider, prune_modes: PruneModes::none() } } /// Enables metrics on the static file provider. @@ -64,6 +66,12 @@ impl ProviderFactory { self } + /// Sets the pruning configuration for an existing [`ProviderFactory`]. + pub fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { + self.prune_modes = prune_modes; + self + } + /// Returns reference to the underlying database. pub fn db_ref(&self) -> &DB { &self.db @@ -89,6 +97,7 @@ impl ProviderFactory { db: Arc::new(init_db(path, args).map_err(RethError::msg)?), chain_spec, static_file_provider, + prune_modes: PruneModes::none(), }) } } @@ -97,12 +106,16 @@ impl ProviderFactory { /// Returns a provider with a created `DbTx` inside, which allows fetching data from the /// database using different types of providers. Example: [`HeaderProvider`] /// [`BlockHashReader`]. This may fail if the inner read database transaction fails to open. + /// + /// This sets the [`PruneModes`] to [`None`], because they should only be relevant for writing + /// data. #[track_caller] pub fn provider(&self) -> ProviderResult> { Ok(DatabaseProvider::new( self.db.tx()?, self.chain_spec.clone(), self.static_file_provider.clone(), + self.prune_modes.clone(), )) } @@ -116,6 +129,7 @@ impl ProviderFactory { self.db.tx_mut()?, self.chain_spec.clone(), self.static_file_provider.clone(), + self.prune_modes.clone(), ))) } @@ -576,6 +590,7 @@ impl Clone for ProviderFactory { db: Arc::clone(&self.db), chain_spec: self.chain_spec.clone(), static_file_provider: self.static_file_provider.clone(), + prune_modes: self.prune_modes.clone(), } } } @@ -661,7 +676,7 @@ mod tests { { let provider = factory.provider_rw().unwrap(); assert_matches!( - provider.insert_block(block.clone().try_seal_with_senders().unwrap(), None), + provider.insert_block(block.clone().try_seal_with_senders().unwrap()), Ok(_) ); assert_matches!( @@ -672,16 +687,14 @@ mod tests { } { - let provider = factory.provider_rw().unwrap(); + let prune_modes = PruneModes { + sender_recovery: Some(PruneMode::Full), + transaction_lookup: Some(PruneMode::Full), + ..PruneModes::none() + }; + let provider = factory.with_prune_modes(prune_modes).provider_rw().unwrap(); assert_matches!( - provider.insert_block( - block.clone().try_seal_with_senders().unwrap(), - Some(&PruneModes { - sender_recovery: Some(PruneMode::Full), - transaction_lookup: Some(PruneMode::Full), - ..PruneModes::none() - }) - ), + provider.insert_block(block.clone().try_seal_with_senders().unwrap(),), Ok(_) ); assert_matches!(provider.transaction_sender(0), Ok(None)); @@ -701,7 +714,7 @@ mod tests { let provider = factory.provider_rw().unwrap(); assert_matches!( - provider.insert_block(block.clone().try_seal_with_senders().unwrap(), None), + provider.insert_block(block.clone().try_seal_with_senders().unwrap()), Ok(_) ); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 32b666c50..61c295b54 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -104,6 +104,8 @@ pub struct DatabaseProvider { chain_spec: Arc, /// Static File provider static_file_provider: StaticFileProvider, + /// Pruning configuration + prune_modes: PruneModes, } impl DatabaseProvider { @@ -119,8 +121,9 @@ impl DatabaseProvider { tx: TX, chain_spec: Arc, static_file_provider: StaticFileProvider, + prune_modes: PruneModes, ) -> Self { - Self { tx, chain_spec, static_file_provider } + Self { tx, chain_spec, static_file_provider, prune_modes } } } @@ -174,7 +177,6 @@ impl DatabaseProvider { pub fn insert_historical_block( &self, block: SealedBlockWithSenders, - prune_modes: Option<&PruneModes>, ) -> ProviderResult { let ttd = if block.number == 0 { block.difficulty @@ -198,7 +200,7 @@ impl DatabaseProvider { writer.append_header(block.header.as_ref().clone(), ttd, block.hash())?; - self.insert_block(block, prune_modes) + self.insert_block(block) } } @@ -256,8 +258,9 @@ impl DatabaseProvider { tx: TX, chain_spec: Arc, static_file_provider: StaticFileProvider, + prune_modes: PruneModes, ) -> Self { - Self { tx, chain_spec, static_file_provider } + Self { tx, chain_spec, static_file_provider, prune_modes } } /// Consume `DbTx` or `DbTxMut`. @@ -2621,7 +2624,6 @@ impl BlockWriter for DatabaseProvider { fn insert_block( &self, block: SealedBlockWithSenders, - prune_modes: Option<&PruneModes>, ) -> ProviderResult { let block_number = block.number; @@ -2678,8 +2680,10 @@ impl BlockWriter for DatabaseProvider { for (transaction, sender) in block.block.body.into_iter().zip(block.senders.iter()) { let hash = transaction.hash(); - if prune_modes - .and_then(|modes| modes.sender_recovery) + if self + .prune_modes + .sender_recovery + .as_ref() .filter(|prune_mode| prune_mode.is_full()) .is_none() { @@ -2703,8 +2707,9 @@ impl BlockWriter for DatabaseProvider { } transactions_elapsed += elapsed; - if prune_modes - .and_then(|modes| modes.transaction_lookup) + if self + .prune_modes + .transaction_lookup .filter(|prune_mode| prune_mode.is_full()) .is_none() { @@ -2765,7 +2770,6 @@ impl BlockWriter for DatabaseProvider { execution_outcome: ExecutionOutcome, hashed_state: HashedPostState, trie_updates: TrieUpdates, - prune_modes: Option<&PruneModes>, ) -> ProviderResult<()> { if blocks.is_empty() { debug!(target: "providers::db", "Attempted to append empty block range"); @@ -2781,7 +2785,7 @@ impl BlockWriter for DatabaseProvider { // Insert the blocks for block in blocks { - self.insert_block(block, prune_modes)?; + self.insert_block(block)?; durations_recorder.record_relative(metrics::Action::InsertBlock); } diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 7211cb691..3d0cf3c0c 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -1,7 +1,6 @@ use reth_db_api::models::StoredBlockBodyIndices; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{BlockNumber, SealedBlockWithSenders}; -use reth_prune_types::PruneModes; use reth_storage_api::BlockReader; use reth_storage_errors::provider::ProviderResult; use reth_trie::{updates::TrieUpdates, HashedPostState}; @@ -41,11 +40,8 @@ pub trait BlockWriter: Send + Sync { /// /// Return [StoredBlockBodyIndices] that contains indices of the first and last transactions and /// transition in the block. - fn insert_block( - &self, - block: SealedBlockWithSenders, - prune_modes: Option<&PruneModes>, - ) -> ProviderResult; + fn insert_block(&self, block: SealedBlockWithSenders) + -> ProviderResult; /// Appends a batch of sealed blocks to the blockchain, including sender information, and /// updates the post-state. @@ -57,7 +53,6 @@ pub trait BlockWriter: Send + Sync { /// /// - `blocks`: Vector of `SealedBlockWithSenders` instances to append. /// - `state`: Post-state information to update after appending. - /// - `prune_modes`: Optional pruning configuration. /// /// # Returns /// @@ -68,6 +63,5 @@ pub trait BlockWriter: Send + Sync { execution_outcome: ExecutionOutcome, hashed_state: HashedPostState, trie_updates: TrieUpdates, - prune_modes: Option<&PruneModes>, ) -> ProviderResult<()>; } diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index 87c3b8df1..7ecd7cb33 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -102,7 +102,6 @@ impl Case for BlockchainTestCase { ) .try_seal_with_senders() .unwrap(), - None, )?; case.pre.write_to_db(provider.tx_ref())?; @@ -121,7 +120,6 @@ impl Case for BlockchainTestCase { let decoded = SealedBlock::decode(&mut block.rlp.as_ref())?; provider.insert_historical_block( decoded.clone().try_seal_with_senders().unwrap(), - None, )?; Ok::, Error>(Some(decoded)) })?;