jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 19:09:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pruner, primitives): move prune batch sizes to ChainSpec (#4318)

Browse Source 
Co-authored-by: joshieDo <ranriver@protonmail.com>
This commit is contained in:
Alexey Shekhirin
2023-08-23 18:45:53 +01:00
committed by GitHub
parent 312cf724bc
commit 1eee5ee80a
9 changed files with 165 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/config/src/config.rs
View File

@ -283,14 +283,14 @@ impl Default for IndexHistoryConfig {
#[serde(default)]
pub struct PruneConfig {
/// Minimum pruning interval measured in blocks.
pub block_interval: u64,
pub block_interval: usize,
/// Pruning configuration for every part of the data that can be pruned.
pub parts: PruneModes,
}
impl Default for PruneConfig {
fn default() -> Self {
Self { block_interval: 10, parts: PruneModes::default() }
Self { block_interval: 5, parts: PruneModes::default() }
}
}

6
crates/consensus/beacon/src/engine/test_utils.rs
View File

@ -20,12 +20,12 @@ use reth_interfaces::{
test_utils::{NoopFullBlockClient, TestConsensus},
};
use reth_payload_builder::test_utils::spawn_test_payload_service;
use reth_primitives::{BlockNumber, ChainSpec, PruneModes, H256, U256};
use reth_primitives::{BlockNumber, ChainSpec, PruneBatchSizes, PruneModes, H256, U256};
use reth_provider::{
providers::BlockchainProvider, test_utils::TestExecutorFactory, BlockExecutor, ExecutorFactory,
ProviderFactory, StateProvider,
};
use reth_prune::{BatchSizes, Pruner};
use reth_prune::Pruner;
use reth_revm::Factory;
use reth_rpc_types::engine::{ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus};
use reth_stages::{
@ -470,7 +470,7 @@ where
self.base_config.chain_spec.clone(),
5,
PruneModes::default(),
BatchSizes::default(),
PruneBatchSizes::default(),
);
let (mut engine, handle) = BeaconConsensusEngine::new(

22
crates/primitives/src/chain/spec.rs
View File

@ -7,7 +7,7 @@ use crate::{
header::Head,
proofs::genesis_state_root,
Address, BlockNumber, Chain, ForkFilter, ForkHash, ForkId, Genesis, Hardfork, Header,
SealedHeader, H160, H256, U256,
PruneBatchSizes, SealedHeader, H160, H256, U256,
};
use hex_literal::hex;
use once_cell::sync::Lazy;
@ -63,7 +63,8 @@ pub static MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
11052984,
H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
)),
..Default::default()
base_fee_params: BaseFeeParams::ethereum(),
prune_batch_sizes: PruneBatchSizes::mainnet(),
}
.into()
});
@ -104,7 +105,8 @@ pub static GOERLI: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
4367322,
H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
)),
..Default::default()
base_fee_params: BaseFeeParams::ethereum(),
prune_batch_sizes: PruneBatchSizes::testnet(),
}
.into()
});
@ -149,7 +151,8 @@ pub static SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
1273020,
H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")),
)),
..Default::default()
base_fee_params: BaseFeeParams::ethereum(),
prune_batch_sizes: PruneBatchSizes::testnet(),
}
.into()
});
@ -203,7 +206,7 @@ pub struct BaseFeeParams {
}
impl BaseFeeParams {
/// Get the base fee parameters for ethereum mainnet
/// Get the base fee parameters for Ethereum mainnet
pub const fn ethereum() -> BaseFeeParams {
BaseFeeParams {
max_change_denominator: EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR,
@ -247,12 +250,18 @@ pub struct ChainSpec {
/// The active hard forks and their activation conditions
pub hardforks: BTreeMap<Hardfork, ForkCondition>,
/// The deposit contract deployed for PoS.
/// The deposit contract deployed for PoS
#[serde(skip, default)]
pub deposit_contract: Option<DepositContract>,
/// The parameters that configure how a block's base fee is computed
pub base_fee_params: BaseFeeParams,
/// The batch sizes for pruner, per block. In the actual pruner run it will be multiplied by
/// the amount of blocks between pruner runs to account for the difference in amount of new
/// data coming in.
#[serde(default)]
pub prune_batch_sizes: PruneBatchSizes,
}
impl Default for ChainSpec {
@ -266,6 +275,7 @@ impl Default for ChainSpec {
hardforks: Default::default(),
deposit_contract: Default::default(),
base_fee_params: BaseFeeParams::ethereum(),
prune_batch_sizes: Default::default(),
}
}
}

4
crates/primitives/src/lib.rs
View File

@ -81,8 +81,8 @@ pub use net::{
};
pub use peer::{PeerId, WithPeerId};
pub use prune::{
PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, ReceiptsLogPruneConfig,
MINIMUM_PRUNING_DISTANCE,
PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError,
ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE,
};
pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef};
pub use revm_primitives::JumpMap;

83
crates/primitives/src/prune/batch_sizes.rs Normal file
View File

@ -0,0 +1,83 @@
use paste::paste;
use serde::{Deserialize, Serialize};
/// Batch sizes for configuring the pruner.
/// The batch size for each prune part should be both large enough to prune the data which was
/// generated with each new block, and small enough to not generate an excessive load on the
/// database due to deletion of too many rows at once.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct PruneBatchSizes {
/// Maximum number of receipts to prune, per block.
receipts: usize,
/// Maximum number of transaction lookup entries to prune, per block.
transaction_lookup: usize,
/// Maximum number of transaction senders to prune, per block.
transaction_senders: usize,
/// Maximum number of account history entries to prune, per block.
/// Measured in the number of `AccountChangeSet` table rows.
account_history: usize,
/// Maximum number of storage history entries to prune, per block.
/// Measured in the number of `StorageChangeSet` table rows.
storage_history: usize,
}
macro_rules! impl_prune_batch_size_methods {
($(($human_name:expr, $name:ident)),+) => {
paste! {
impl PruneBatchSizes {
$(
#[doc = concat!("Maximum number of ", $human_name, " to prune, accounting for the block interval.")]
pub fn $name(&self, block_interval: usize) -> usize {
self.$name * block_interval
}
#[doc = concat!("Set the maximum number of ", $human_name, " to prune per block.")]
pub fn [<with_ $name>](mut self, batch_size: usize) -> Self {
self.$name = batch_size;
self
}
)+
}
}
};
}
impl_prune_batch_size_methods!(
("receipts", receipts),
("transaction lookup entries", transaction_lookup),
("transaction senders", transaction_senders),
("account history entries", account_history),
("storage history entries", storage_history)
);
impl PruneBatchSizes {
/// Default prune batch sizes for Ethereum mainnet.
/// These settings are sufficient to prune more data than generated with each new block.
pub const fn mainnet() -> Self {
Self {
receipts: 250,
transaction_lookup: 250,
transaction_senders: 250,
account_history: 1000,
storage_history: 1000,
}
}
/// Default prune batch sizes for Ethereum testnets.
/// These settings are sufficient to prune more data than generated with each new block.
pub const fn testnet() -> Self {
Self {
receipts: 100,
transaction_lookup: 100,
transaction_senders: 100,
account_history: 500,
storage_history: 500,
}
}
}
impl Default for PruneBatchSizes {
fn default() -> Self {
Self::mainnet()
}
}

2
crates/primitives/src/prune/mod.rs
View File

@ -1,9 +1,11 @@
mod batch_sizes;
mod checkpoint;
mod mode;
mod part;
mod target;
use crate::{Address, BlockNumber};
pub use batch_sizes::PruneBatchSizes;
pub use checkpoint::PruneCheckpoint;
pub use mode::PruneMode;
pub use part::{PrunePart, PrunePartError};

2
crates/prune/src/lib.rs
View File

@ -4,4 +4,4 @@ mod pruner;
use crate::metrics::Metrics;
pub use error::PrunerError;
pub use pruner::{BatchSizes, Pruner, PrunerResult, PrunerWithResult};
pub use pruner::{Pruner, PrunerResult, PrunerWithResult};

146
crates/prune/src/pruner.rs
View File

@ -12,8 +12,8 @@ use reth_db::{
BlockNumberList,
};
use reth_primitives::{
BlockNumber, ChainSpec, PruneCheckpoint, PruneMode, PruneModes, PrunePart, TxNumber,
MINIMUM_PRUNING_DISTANCE,
BlockNumber, ChainSpec, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart,
TxNumber, MINIMUM_PRUNING_DISTANCE,
};
use reth_provider::{
BlockReader, DatabaseProviderRW, ProviderFactory, PruneCheckpointReader, PruneCheckpointWriter,
@ -31,46 +31,19 @@ pub type PrunerResult = Result<bool, PrunerError>;
/// The pruner type itself with the result of [Pruner::run]
pub type PrunerWithResult<DB> = (Pruner<DB>, PrunerResult);
pub struct BatchSizes {
/// Maximum number of receipts to prune in one run.
receipts: usize,
/// Maximum number of transaction lookup entries to prune in one run.
transaction_lookup: usize,
/// Maximum number of transaction senders to prune in one run.
transaction_senders: usize,
/// Maximum number of account history entries to prune in one run.
/// Measured in the number of [tables::AccountChangeSet] rows.
account_history: usize,
/// Maximum number of storage history entries to prune in one run.
/// Measured in the number of [tables::StorageChangeSet] rows.
storage_history: usize,
}
impl Default for BatchSizes {
fn default() -> Self {
Self {
receipts: 1000,
transaction_lookup: 1000,
transaction_senders: 1000,
account_history: 1000,
storage_history: 1000,
}
}
}
/// Pruning routine. Main pruning logic happens in [Pruner::run].
pub struct Pruner<DB> {
metrics: Metrics,
provider_factory: ProviderFactory<DB>,
/// Minimum pruning interval measured in blocks. All prune parts are checked and, if needed,
/// pruned, when the chain advances by the specified number of blocks.
min_block_interval: u64,
min_block_interval: usize,
/// Last pruned block number. Used in conjunction with `min_block_interval` to determine
/// when the pruning needs to be initiated.
last_pruned_block_number: Option<BlockNumber>,
modes: PruneModes,
/// Maximum entries to prune per one run, per prune part.
batch_sizes: BatchSizes,
/// Maximum entries to prune per block, per prune part.
batch_sizes: PruneBatchSizes,
}
impl<DB: Database> Pruner<DB> {
@ -78,9 +51,9 @@ impl<DB: Database> Pruner<DB> {
pub fn new(
db: DB,
chain_spec: Arc<ChainSpec>,
min_block_interval: u64,
min_block_interval: usize,
modes: PruneModes,
batch_sizes: BatchSizes,
batch_sizes: PruneBatchSizes,
) -> Self {
Self {
metrics: Metrics::default(),
@ -263,7 +236,8 @@ impl<DB: Database> Pruner<DB> {
// Saturating subtraction is needed for the case when the chain was reverted, meaning
// current block number might be less than the previously pruned block number. If
// that's the case, no pruning is needed as outdated data is also reverted.
tip_block_number.saturating_sub(last_pruned_block_number) >= self.min_block_interval
tip_block_number.saturating_sub(last_pruned_block_number) >=
self.min_block_interval as u64
}) {
debug!(
target: "pruner",
@ -372,7 +346,7 @@ impl<DB: Database> Pruner<DB> {
let mut last_pruned_transaction = tx_range_end;
let (deleted, done) = provider.prune_table_with_range::<tables::Receipts>(
tx_range,
self.batch_sizes.receipts,
self.batch_sizes.receipts(self.min_block_interval),
|_| false,
|row| last_pruned_transaction = row.0,
)?;
@ -490,7 +464,7 @@ impl<DB: Database> Pruner<DB> {
"Calculated block ranges and filtered addresses",
);
let mut limit = self.batch_sizes.receipts;
let mut limit = self.batch_sizes.receipts(self.min_block_interval);
let mut done = true;
let mut last_pruned_transaction = None;
for (start_block, end_block, num_addresses) in block_ranges {
@ -603,7 +577,10 @@ impl<DB: Database> Pruner<DB> {
}
}
.into_inner();
let tx_range = start..=(end.min(start + self.batch_sizes.transaction_lookup as u64 - 1));
let tx_range = start..=
(end.min(
start + self.batch_sizes.transaction_lookup(self.min_block_interval) as u64 - 1,
));
let tx_range_end = *tx_range.end();
// Retrieve transactions in the range and calculate their hashes in parallel
@ -624,7 +601,7 @@ impl<DB: Database> Pruner<DB> {
let mut last_pruned_transaction = tx_range_end;
let (deleted, done) = provider.prune_table_with_iterator::<tables::TxHashNumber>(
hashes,
self.batch_sizes.transaction_lookup,
self.batch_sizes.transaction_lookup(self.min_block_interval),
|row| last_pruned_transaction = row.1,
)?;
trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup");
@ -673,7 +650,7 @@ impl<DB: Database> Pruner<DB> {
let mut last_pruned_transaction = tx_range_end;
let (deleted, done) = provider.prune_table_with_range::<tables::TxSenders>(
tx_range,
self.batch_sizes.transaction_senders,
self.batch_sizes.transaction_senders(self.min_block_interval),
|_| false,
|row| last_pruned_transaction = row.0,
)?;
@ -722,7 +699,7 @@ impl<DB: Database> Pruner<DB> {
let mut last_changeset_pruned_block = None;
let (rows, done) = provider.prune_table_with_range::<tables::AccountChangeSet>(
range,
self.batch_sizes.account_history,
self.batch_sizes.account_history(self.min_block_interval),
|_| false,
|row| last_changeset_pruned_block = Some(row.0),
)?;
@ -778,7 +755,7 @@ impl<DB: Database> Pruner<DB> {
let mut last_changeset_pruned_block = None;
let (rows, done) = provider.prune_table_with_range::<tables::StorageChangeSet>(
BlockNumberAddress::range(range),
self.batch_sizes.storage_history,
self.batch_sizes.storage_history(self.min_block_interval),
|_| false,
|row| last_changeset_pruned_block = Some(row.0.block_number()),
)?;
@ -916,7 +893,7 @@ impl<DB: Database> Pruner<DB> {
#[cfg(test)]
mod tests {
use crate::{pruner::BatchSizes, Pruner};
use crate::Pruner;
use assert_matches::assert_matches;
use itertools::{
FoldWhile::{Continue, Done},
@ -934,8 +911,8 @@ mod tests {
},
};
use reth_primitives::{
BlockNumber, PruneCheckpoint, PruneMode, PruneModes, PrunePart, ReceiptsLogPruneConfig,
TxNumber, H256, MAINNET,
BlockNumber, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart,
ReceiptsLogPruneConfig, TxNumber, H256, MAINNET,
};
use reth_provider::{PruneCheckpointReader, TransactionsProvider};
use reth_stages::test_utils::TestTransaction;
@ -945,14 +922,14 @@ mod tests {
fn is_pruning_needed() {
let db = create_test_rw_db();
let pruner =
Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), BatchSizes::default());
Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), PruneBatchSizes::default());
// No last pruned block number was set before
let first_block_number = 1;
assert!(pruner.is_pruning_needed(first_block_number));
// Delta is not less than min block interval
let second_block_number = first_block_number + pruner.min_block_interval;
let second_block_number = first_block_number + pruner.min_block_interval as u64;
assert!(pruner.is_pruning_needed(second_block_number));
// Delta is less than min block interval
@ -991,13 +968,10 @@ mod tests {
let pruner = Pruner::new(
tx.inner_raw(),
MAINNET.clone(),
5,
1,
PruneModes { receipts: Some(prune_mode), ..Default::default() },
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
receipts: 10,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_receipts(10),
);
let next_tx_number_to_prune = tx
@ -1008,11 +982,12 @@ mod tests {
.map(|tx_number| tx_number + 1)
.unwrap_or_default();
let last_pruned_tx_number = blocks
.iter()
.map(|block| block.body.len())
.sum::<usize>()
.min(next_tx_number_to_prune as usize + pruner.batch_sizes.receipts - 1);
let last_pruned_tx_number =
blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
next_tx_number_to_prune as usize +
pruner.batch_sizes.receipts(pruner.min_block_interval) -
1,
);
let last_pruned_block_number = blocks
.iter()
@ -1087,13 +1062,10 @@ mod tests {
let pruner = Pruner::new(
tx.inner_raw(),
MAINNET.clone(),
5,
1,
PruneModes { transaction_lookup: Some(prune_mode), ..Default::default() },
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
transaction_lookup: 10,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_transaction_lookup(10),
);
let next_tx_number_to_prune = tx
@ -1106,7 +1078,9 @@ mod tests {
let last_pruned_tx_number =
blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_lookup - 1,
next_tx_number_to_prune as usize +
pruner.batch_sizes.transaction_lookup(pruner.min_block_interval) -
1,
);
let last_pruned_block_number = blocks
@ -1185,13 +1159,10 @@ mod tests {
let pruner = Pruner::new(
tx.inner_raw(),
MAINNET.clone(),
5,
1,
PruneModes { sender_recovery: Some(prune_mode), ..Default::default() },
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
transaction_senders: 10,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_transaction_senders(10),
);
let next_tx_number_to_prune = tx
@ -1204,7 +1175,9 @@ mod tests {
let last_pruned_tx_number =
blocks.iter().map(|block| block.body.len()).sum::<usize>().min(
next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_senders - 1,
next_tx_number_to_prune as usize +
pruner.batch_sizes.transaction_senders(pruner.min_block_interval) -
1,
);
let last_pruned_block_number = blocks
@ -1292,13 +1265,10 @@ mod tests {
let pruner = Pruner::new(
tx.inner_raw(),
MAINNET.clone(),
5,
1,
PruneModes { account_history: Some(prune_mode), ..Default::default() },
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
account_history: 2000,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_account_history(2000),
);
let provider = tx.inner_rw();
@ -1320,7 +1290,7 @@ mod tests {
.iter()
.enumerate()
.skip_while(|(i, (block_number, _))| {
*i < pruner.batch_sizes.account_history * run &&
*i < pruner.batch_sizes.account_history(pruner.min_block_interval) * run &&
*block_number <= to_block as usize
})
.next()
@ -1422,13 +1392,10 @@ mod tests {
let pruner = Pruner::new(
tx.inner_raw(),
MAINNET.clone(),
5,
1,
PruneModes { storage_history: Some(prune_mode), ..Default::default() },
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
storage_history: 2000,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_storage_history(2000),
);
let provider = tx.inner_rw();
@ -1452,7 +1419,7 @@ mod tests {
.iter()
.enumerate()
.skip_while(|(i, (block_number, _, _))| {
*i < pruner.batch_sizes.storage_history * run &&
*i < pruner.batch_sizes.storage_history(pruner.min_block_interval) * run &&
*block_number <= to_block as usize
})
.next()
@ -1564,11 +1531,8 @@ mod tests {
receipts_log_filter: receipts_log_filter.clone(),
..Default::default()
},
BatchSizes {
// Less than total amount of blocks to prune to test the batching logic
receipts: 10,
..Default::default()
},
// Less than total amount of blocks to prune to test the batching logic
PruneBatchSizes::default().with_storage_history(10),
);
let result = pruner.prune_receipts_by_logs(&provider, tip);