feat(pruner, primitives): move prune batch sizes to ChainSpec (#4318)

Co-authored-by: joshieDo <ranriver@protonmail.com>

bin/reth/src/node/mod.rs

@@ -51,7 +51,6 @@ use reth_provider::{
     providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions,
     HeaderProvider, ProviderFactory, StageCheckpointReader,
 };
-use reth_prune::BatchSizes;
 use reth_revm::Factory;
 use reth_revm_inspectors::stack::Hook;
 use reth_rpc_engine_api::EngineApi;
@@ -429,7 +428,7 @@ impl<Ext: RethCliExt> NodeCommand<Ext> {
                 self.chain.clone(),
                 prune_config.block_interval,
                 prune_config.parts,
-                BatchSizes::default(),
+                self.chain.prune_batch_sizes,
             )
         });
 

crates/config/src/config.rs

@@ -283,14 +283,14 @@ impl Default for IndexHistoryConfig {
 #[serde(default)]
 pub struct PruneConfig {
     /// Minimum pruning interval measured in blocks.
-    pub block_interval: u64,
+    pub block_interval: usize,
     /// Pruning configuration for every part of the data that can be pruned.
     pub parts: PruneModes,
 }
 
 impl Default for PruneConfig {
     fn default() -> Self {
-        Self { block_interval: 10, parts: PruneModes::default() }
+        Self { block_interval: 5, parts: PruneModes::default() }
     }
 }
 

crates/consensus/beacon/src/engine/test_utils.rs

@@ -20,12 +20,12 @@ use reth_interfaces::{
     test_utils::{NoopFullBlockClient, TestConsensus},
 };
 use reth_payload_builder::test_utils::spawn_test_payload_service;
-use reth_primitives::{BlockNumber, ChainSpec, PruneModes, H256, U256};
+use reth_primitives::{BlockNumber, ChainSpec, PruneBatchSizes, PruneModes, H256, U256};
 use reth_provider::{
     providers::BlockchainProvider, test_utils::TestExecutorFactory, BlockExecutor, ExecutorFactory,
     ProviderFactory, StateProvider,
 };
-use reth_prune::{BatchSizes, Pruner};
+use reth_prune::Pruner;
 use reth_revm::Factory;
 use reth_rpc_types::engine::{ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus};
 use reth_stages::{
@@ -470,7 +470,7 @@ where
             self.base_config.chain_spec.clone(),
             5,
             PruneModes::default(),
-            BatchSizes::default(),
+            PruneBatchSizes::default(),
         );
 
         let (mut engine, handle) = BeaconConsensusEngine::new(

crates/primitives/src/chain/spec.rs

@@ -7,7 +7,7 @@ use crate::{
     header::Head,
     proofs::genesis_state_root,
     Address, BlockNumber, Chain, ForkFilter, ForkHash, ForkId, Genesis, Hardfork, Header,
-    SealedHeader, H160, H256, U256,
+    PruneBatchSizes, SealedHeader, H160, H256, U256,
 };
 use hex_literal::hex;
 use once_cell::sync::Lazy;
@@ -63,7 +63,8 @@ pub static MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
             11052984,
             H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
         )),
-        ..Default::default()
+        base_fee_params: BaseFeeParams::ethereum(),
+        prune_batch_sizes: PruneBatchSizes::mainnet(),
     }
     .into()
 });
@@ -104,7 +105,8 @@ pub static GOERLI: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
             4367322,
             H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
         )),
-        ..Default::default()
+        base_fee_params: BaseFeeParams::ethereum(),
+        prune_batch_sizes: PruneBatchSizes::testnet(),
     }
     .into()
 });
@@ -149,7 +151,8 @@ pub static SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
             1273020,
             H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
         )),
-        ..Default::default()
+        base_fee_params: BaseFeeParams::ethereum(),
+        prune_batch_sizes: PruneBatchSizes::testnet(),
     }
     .into()
 });
@@ -203,7 +206,7 @@ pub struct BaseFeeParams {
 }
 
 impl BaseFeeParams {
-    /// Get the base fee parameters for ethereum mainnet
+    /// Get the base fee parameters for Ethereum mainnet
     pub const fn ethereum() -> BaseFeeParams {
         BaseFeeParams {
             max_change_denominator: EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR,
@@ -247,12 +250,18 @@ pub struct ChainSpec {
     /// The active hard forks and their activation conditions
     pub hardforks: BTreeMap<Hardfork, ForkCondition>,
 
-    /// The deposit contract deployed for PoS.
+    /// The deposit contract deployed for PoS
     #[serde(skip, default)]
     pub deposit_contract: Option<DepositContract>,
 
     /// The parameters that configure how a block's base fee is computed
     pub base_fee_params: BaseFeeParams,
+
+    /// The batch sizes for pruner, per block. In the actual pruner run it will be multiplied by
+    /// the amount of blocks between pruner runs to account for the difference in amount of new
+    /// data coming in.
+    #[serde(default)]
+    pub prune_batch_sizes: PruneBatchSizes,
 }
 
 impl Default for ChainSpec {
@@ -266,6 +275,7 @@ impl Default for ChainSpec {
             hardforks: Default::default(),
             deposit_contract: Default::default(),
             base_fee_params: BaseFeeParams::ethereum(),
+            prune_batch_sizes: Default::default(),
         }
     }
 }

crates/primitives/src/lib.rs

@@ -81,8 +81,8 @@ pub use net::{
 };
 pub use peer::{PeerId, WithPeerId};
 pub use prune::{
-    PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, ReceiptsLogPruneConfig,
-    MINIMUM_PRUNING_DISTANCE,
+    PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError,
+    ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE,
 };
 pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef};
 pub use revm_primitives::JumpMap;

crates/primitives/src/prune/batch_sizes.rs

@@ -0,0 +1,83 @@
+use paste::paste;
+use serde::{Deserialize, Serialize};
+
+/// Batch sizes for configuring the pruner.
+/// The batch size for each prune part should be both large enough to prune the data which was
+/// generated with each new block, and small enough to not generate an excessive load on the
+/// database due to deletion of too many rows at once.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PruneBatchSizes {
+    /// Maximum number of receipts to prune, per block.
+    receipts: usize,
+    /// Maximum number of transaction lookup entries to prune, per block.
+    transaction_lookup: usize,
+    /// Maximum number of transaction senders to prune, per block.
+    transaction_senders: usize,
+    /// Maximum number of account history entries to prune, per block.
+    /// Measured in the number of `AccountChangeSet` table rows.
+    account_history: usize,
+    /// Maximum number of storage history entries to prune, per block.
+    /// Measured in the number of `StorageChangeSet` table rows.
+    storage_history: usize,
+}
+
+macro_rules! impl_prune_batch_size_methods {
+    ($(($human_name:expr, $name:ident)),+) => {
+        paste! {
+            impl PruneBatchSizes {
+                $(
+                    #[doc = concat!("Maximum number of ", $human_name, " to prune, accounting for the block interval.")]
+                    pub fn $name(&self, block_interval: usize) -> usize {
+                        self.$name * block_interval
+                    }
+
+                    #[doc = concat!("Set the maximum number of ", $human_name, " to prune per block.")]
+                    pub fn [<with_ $name>](mut self, batch_size: usize) -> Self {
+                        self.$name = batch_size;
+                        self
+                    }
+                )+
+            }
+        }
+    };
+}
+
+impl_prune_batch_size_methods!(
+    ("receipts", receipts),
+    ("transaction lookup entries", transaction_lookup),
+    ("transaction senders", transaction_senders),
+    ("account history entries", account_history),
+    ("storage history entries", storage_history)
+);
+
+impl PruneBatchSizes {
+    /// Default prune batch sizes for Ethereum mainnet.
+    /// These settings are sufficient to prune more data than generated with each new block.
+    pub const fn mainnet() -> Self {
+        Self {
+            receipts: 250,
+            transaction_lookup: 250,
+            transaction_senders: 250,
+            account_history: 1000,
+            storage_history: 1000,
+        }
+    }
+
+    /// Default prune batch sizes for Ethereum testnets.
+    /// These settings are sufficient to prune more data than generated with each new block.
+    pub const fn testnet() -> Self {
+        Self {
+            receipts: 100,
+            transaction_lookup: 100,
+            transaction_senders: 100,
+            account_history: 500,
+            storage_history: 500,
+        }
+    }
+}
+
+impl Default for PruneBatchSizes {
+    fn default() -> Self {
+        Self::mainnet()
+    }
+}

crates/primitives/src/prune/mod.rs

@@ -1,9 +1,11 @@
+mod batch_sizes;
 mod checkpoint;
 mod mode;
 mod part;
 mod target;
 
 use crate::{Address, BlockNumber};
+pub use batch_sizes::PruneBatchSizes;
 pub use checkpoint::PruneCheckpoint;
 pub use mode::PruneMode;
 pub use part::{PrunePart, PrunePartError};

crates/prune/src/lib.rs

@@ -4,4 +4,4 @@ mod pruner;
 
 use crate::metrics::Metrics;
 pub use error::PrunerError;
-pub use pruner::{BatchSizes, Pruner, PrunerResult, PrunerWithResult};
+pub use pruner::{Pruner, PrunerResult, PrunerWithResult};

crates/prune/src/pruner.rs

@@ -12,8 +12,8 @@ use reth_db::{
     BlockNumberList,
 };
 use reth_primitives::{
-    BlockNumber, ChainSpec, PruneCheckpoint, PruneMode, PruneModes, PrunePart, TxNumber,
-    MINIMUM_PRUNING_DISTANCE,
+    BlockNumber, ChainSpec, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart,
+    TxNumber, MINIMUM_PRUNING_DISTANCE,
 };
 use reth_provider::{
     BlockReader, DatabaseProviderRW, ProviderFactory, PruneCheckpointReader, PruneCheckpointWriter,
@@ -31,46 +31,19 @@ pub type PrunerResult = Result<bool, PrunerError>;
 /// The pruner type itself with the result of [Pruner::run]
 pub type PrunerWithResult<DB> = (Pruner<DB>, PrunerResult);
 
-pub struct BatchSizes {
-    /// Maximum number of receipts to prune in one run.
-    receipts: usize,
-    /// Maximum number of transaction lookup entries to prune in one run.
-    transaction_lookup: usize,
-    /// Maximum number of transaction senders to prune in one run.
-    transaction_senders: usize,
-    /// Maximum number of account history entries to prune in one run.
-    /// Measured in the number of [tables::AccountChangeSet] rows.
-    account_history: usize,
-    /// Maximum number of storage history entries to prune in one run.
-    /// Measured in the number of [tables::StorageChangeSet] rows.
-    storage_history: usize,
-}
-
-impl Default for BatchSizes {
-    fn default() -> Self {
-        Self {
-            receipts: 1000,
-            transaction_lookup: 1000,
-            transaction_senders: 1000,
-            account_history: 1000,
-            storage_history: 1000,
-        }
-    }
-}
-
 /// Pruning routine. Main pruning logic happens in [Pruner::run].
 pub struct Pruner<DB> {
     metrics: Metrics,
     provider_factory: ProviderFactory<DB>,
     /// Minimum pruning interval measured in blocks. All prune parts are checked and, if needed,
     /// pruned, when the chain advances by the specified number of blocks.
-    min_block_interval: u64,
+    min_block_interval: usize,
     /// Last pruned block number. Used in conjunction with `min_block_interval` to determine
     /// when the pruning needs to be initiated.
     last_pruned_block_number: Option<BlockNumber>,
     modes: PruneModes,
-    /// Maximum entries to prune per one run, per prune part.
-    batch_sizes: BatchSizes,
+    /// Maximum entries to prune per block, per prune part.
+    batch_sizes: PruneBatchSizes,
 }
 
 impl<DB: Database> Pruner<DB> {
@@ -78,9 +51,9 @@ impl<DB: Database> Pruner<DB> {
     pub fn new(
         db: DB,
         chain_spec: Arc<ChainSpec>,
-        min_block_interval: u64,
+        min_block_interval: usize,
         modes: PruneModes,
-        batch_sizes: BatchSizes,
+        batch_sizes: PruneBatchSizes,
     ) -> Self {
         Self {
             metrics: Metrics::default(),
@@ -263,7 +236,8 @@ impl<DB: Database> Pruner<DB> {
             // Saturating subtraction is needed for the case when the chain was reverted, meaning
             // current block number might be less than the previously pruned block number. If
             // that's the case, no pruning is needed as outdated data is also reverted.
-            tip_block_number.saturating_sub(last_pruned_block_number) >= self.min_block_interval
+            tip_block_number.saturating_sub(last_pruned_block_number) >=
+                self.min_block_interval as u64
         }) {
             debug!(
                 target: "pruner",
@@ -372,7 +346,7 @@ impl<DB: Database> Pruner<DB> {
         let mut last_pruned_transaction = tx_range_end;
         let (deleted, done) = provider.prune_table_with_range::<tables::Receipts>(
             tx_range,
-            self.batch_sizes.receipts,
+            self.batch_sizes.receipts(self.min_block_interval),
             |_| false,
             |row| last_pruned_transaction = row.0,
         )?;
@@ -490,7 +464,7 @@ impl<DB: Database> Pruner<DB> {
             "Calculated block ranges and filtered addresses",
         );
 
-        let mut limit = self.batch_sizes.receipts;
+        let mut limit = self.batch_sizes.receipts(self.min_block_interval);
         let mut done = true;
         let mut last_pruned_transaction = None;
         for (start_block, end_block, num_addresses) in block_ranges {
@@ -603,7 +577,10 @@ impl<DB: Database> Pruner<DB> {
             }
         }
         .into_inner();
-        let tx_range = start..=(end.min(start + self.batch_sizes.transaction_lookup as u64 - 1));
+        let tx_range = start..=
+            (end.min(
+                start + self.batch_sizes.transaction_lookup(self.min_block_interval) as u64 - 1,
+            ));
         let tx_range_end = *tx_range.end();
 
         // Retrieve transactions in the range and calculate their hashes in parallel
@@ -624,7 +601,7 @@ impl<DB: Database> Pruner<DB> {
         let mut last_pruned_transaction = tx_range_end;
         let (deleted, done) = provider.prune_table_with_iterator::<tables::TxHashNumber>(
             hashes,
-            self.batch_sizes.transaction_lookup,
+            self.batch_sizes.transaction_lookup(self.min_block_interval),
             |row| last_pruned_transaction = row.1,
         )?;
         trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup");
@@ -673,7 +650,7 @@ impl<DB: Database> Pruner<DB> {
         let mut last_pruned_transaction = tx_range_end;
         let (deleted, done) = provider.prune_table_with_range::<tables::TxSenders>(
             tx_range,
-            self.batch_sizes.transaction_senders,
+            self.batch_sizes.transaction_senders(self.min_block_interval),
             |_| false,
             |row| last_pruned_transaction = row.0,
         )?;
@@ -722,7 +699,7 @@ impl<DB: Database> Pruner<DB> {
         let mut last_changeset_pruned_block = None;
         let (rows, done) = provider.prune_table_with_range::<tables::AccountChangeSet>(
             range,
-            self.batch_sizes.account_history,
+            self.batch_sizes.account_history(self.min_block_interval),
             |_| false,
             |row| last_changeset_pruned_block = Some(row.0),
         )?;
@@ -778,7 +755,7 @@ impl<DB: Database> Pruner<DB> {
         let mut last_changeset_pruned_block = None;
         let (rows, done) = provider.prune_table_with_range::<tables::StorageChangeSet>(
             BlockNumberAddress::range(range),
-            self.batch_sizes.storage_history,
+            self.batch_sizes.storage_history(self.min_block_interval),
             |_| false,
             |row| last_changeset_pruned_block = Some(row.0.block_number()),
         )?;
@@ -916,7 +893,7 @@ impl<DB: Database> Pruner<DB> {
 
 #[cfg(test)]
 mod tests {
-    use crate::{pruner::BatchSizes, Pruner};
+    use crate::Pruner;
     use assert_matches::assert_matches;
     use itertools::{
         FoldWhile::{Continue, Done},
@@ -934,8 +911,8 @@ mod tests {
         },
     };
     use reth_primitives::{
-        BlockNumber, PruneCheckpoint, PruneMode, PruneModes, PrunePart, ReceiptsLogPruneConfig,
-        TxNumber, H256, MAINNET,
+        BlockNumber, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart,
+        ReceiptsLogPruneConfig, TxNumber, H256, MAINNET,
     };
     use reth_provider::{PruneCheckpointReader, TransactionsProvider};
     use reth_stages::test_utils::TestTransaction;
@@ -945,14 +922,14 @@ mod tests {
     fn is_pruning_needed() {
         let db = create_test_rw_db();
         let pruner =
-            Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), BatchSizes::default());
+            Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), PruneBatchSizes::default());
 
         // No last pruned block number was set before
         let first_block_number = 1;
         assert!(pruner.is_pruning_needed(first_block_number));
 
         // Delta is not less than min block interval
-        let second_block_number = first_block_number + pruner.min_block_interval;
+        let second_block_number = first_block_number + pruner.min_block_interval as u64;
         assert!(pruner.is_pruning_needed(second_block_number));
 
         // Delta is less than min block interval
@@ -991,13 +968,10 @@ mod tests {
             let pruner = Pruner::new(
                 tx.inner_raw(),
                 MAINNET.clone(),
-                5,
+                1,
                 PruneModes { receipts: Some(prune_mode), ..Default::default() },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    receipts: 10,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_receipts(10),
             );
 
             let next_tx_number_to_prune = tx
@@ -1008,11 +982,12 @@ mod tests {
                 .map(|tx_number| tx_number + 1)
                 .unwrap_or_default();
 
-            let last_pruned_tx_number = blocks
-                .iter()
-                .map(|block| block.body.len())
-                .sum::<usize>()
-                .min(next_tx_number_to_prune as usize + pruner.batch_sizes.receipts - 1);
+            let last_pruned_tx_number =
+                blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
+                    next_tx_number_to_prune as usize +
+                        pruner.batch_sizes.receipts(pruner.min_block_interval) -
+                        1,
+                );
 
             let last_pruned_block_number = blocks
                 .iter()
@@ -1087,13 +1062,10 @@ mod tests {
             let pruner = Pruner::new(
                 tx.inner_raw(),
                 MAINNET.clone(),
-                5,
+                1,
                 PruneModes { transaction_lookup: Some(prune_mode), ..Default::default() },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    transaction_lookup: 10,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_transaction_lookup(10),
             );
 
             let next_tx_number_to_prune = tx
@@ -1106,7 +1078,9 @@ mod tests {
 
             let last_pruned_tx_number =
                 blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
-                    next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_lookup - 1,
+                    next_tx_number_to_prune as usize +
+                        pruner.batch_sizes.transaction_lookup(pruner.min_block_interval) -
+                        1,
                 );
 
             let last_pruned_block_number = blocks
@@ -1185,13 +1159,10 @@ mod tests {
             let pruner = Pruner::new(
                 tx.inner_raw(),
                 MAINNET.clone(),
-                5,
+                1,
                 PruneModes { sender_recovery: Some(prune_mode), ..Default::default() },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    transaction_senders: 10,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_transaction_senders(10),
             );
 
             let next_tx_number_to_prune = tx
@@ -1204,7 +1175,9 @@ mod tests {
 
             let last_pruned_tx_number =
                 blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
-                    next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_senders - 1,
+                    next_tx_number_to_prune as usize +
+                        pruner.batch_sizes.transaction_senders(pruner.min_block_interval) -
+                        1,
                 );
 
             let last_pruned_block_number = blocks
@@ -1292,13 +1265,10 @@ mod tests {
             let pruner = Pruner::new(
                 tx.inner_raw(),
                 MAINNET.clone(),
-                5,
+                1,
                 PruneModes { account_history: Some(prune_mode), ..Default::default() },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    account_history: 2000,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_account_history(2000),
             );
 
             let provider = tx.inner_rw();
@@ -1320,7 +1290,7 @@ mod tests {
                 .iter()
                 .enumerate()
                 .skip_while(|(i, (block_number, _))| {
-                    *i < pruner.batch_sizes.account_history * run &&
+                    *i < pruner.batch_sizes.account_history(pruner.min_block_interval) * run &&
                         *block_number <= to_block as usize
                 })
                 .next()
@@ -1422,13 +1392,10 @@ mod tests {
             let pruner = Pruner::new(
                 tx.inner_raw(),
                 MAINNET.clone(),
-                5,
+                1,
                 PruneModes { storage_history: Some(prune_mode), ..Default::default() },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    storage_history: 2000,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_storage_history(2000),
             );
 
             let provider = tx.inner_rw();
@@ -1452,7 +1419,7 @@ mod tests {
                 .iter()
                 .enumerate()
                 .skip_while(|(i, (block_number, _, _))| {
-                    *i < pruner.batch_sizes.storage_history * run &&
+                    *i < pruner.batch_sizes.storage_history(pruner.min_block_interval) * run &&
                         *block_number <= to_block as usize
                 })
                 .next()
@@ -1564,11 +1531,8 @@ mod tests {
                     receipts_log_filter: receipts_log_filter.clone(),
                     ..Default::default()
                 },
-                BatchSizes {
-                    // Less than total amount of blocks to prune to test the batching logic
-                    receipts: 10,
-                    ..Default::default()
-                },
+                // Less than total amount of blocks to prune to test the batching logic
+                PruneBatchSizes::default().with_storage_history(10),
             );
 
             let result = pruner.prune_receipts_by_logs(&provider, tip);