jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(pruner): remove deletion limit per block, make it per run (#9446)

Browse Source
This commit is contained in:
Alexey Shekhirin
2024-07-11 21:16:30 +01:00
committed by GitHub
parent 95bde32cdf
commit 805673c656
11 changed files with 35 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
crates/chainspec/src/spec.rs
View File

@ -56,7 +56,7 @@ pub static MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"),
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 3500,
prune_delete_limit: 20000,
}
.into()
});
@ -78,7 +78,7 @@ pub static SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"),
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
}
.into()
});
@ -98,7 +98,7 @@ pub static HOLESKY: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"),
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
}
.into()
});
@ -143,7 +143,7 @@ pub static OP_MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
]
.into(),
),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
..Default::default()
}
.into()
@ -168,7 +168,7 @@ pub static OP_SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
]
.into(),
),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
..Default::default()
}
.into()
@ -193,7 +193,7 @@ pub static BASE_SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
]
.into(),
),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
..Default::default()
}
.into()
@ -218,7 +218,7 @@ pub static BASE_MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
]
.into(),
),
prune_delete_limit: 1700,
prune_delete_limit: 10000,
..Default::default()
}
.into()
@ -300,9 +300,7 @@ pub struct ChainSpec {
/// The parameters that configure how a block's base fee is computed
pub base_fee_params: BaseFeeParamsKind,
/// The delete limit for pruner, per block. In the actual pruner run it will be multiplied by
/// the amount of blocks between pruner runs to account for the difference in amount of new
/// data coming in.
/// The delete limit for pruner, per run.
pub prune_delete_limit: usize,
}

2
crates/cli/commands/src/prune.rs
View File

@ -30,7 +30,7 @@ impl PruneCommand {
info!(target: "reth::cli", ?prune_tip, ?prune_config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
let mut pruner = PrunerBuilder::new(prune_config)
.delete_limit_per_block(usize::MAX)
.delete_limit(usize::MAX)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;

10
crates/consensus/beacon/src/engine/test_utils.rs
View File

@ -391,10 +391,13 @@ where
// Setup blockchain tree
let externals = TreeExternals::new(provider_factory.clone(), consensus, executor_factory);
let config = BlockchainTreeConfig::new(1, 2, 3, 2);
let tree = Arc::new(ShareableBlockchainTree::new(
BlockchainTree::new(externals, config, PruneModes::default())
.expect("failed to create tree"),
BlockchainTree::new(
externals,
BlockchainTreeConfig::new(1, 2, 3, 2),
PruneModes::default(),
)
.expect("failed to create tree"),
));
let latest = self.base_config.chain_spec.genesis_header().seal_slow();
let blockchain_provider =
@ -405,7 +408,6 @@ where
vec![],
5,
self.base_config.chain_spec.prune_delete_limit,
config.max_reorg_depth() as usize,
None,
watch::channel(FinishedExExHeight::NoExExs).1,
);

1
crates/engine/tree/src/persistence.rs
View File

@ -314,7 +314,6 @@ mod tests {
vec![],
5,
0,
5,
None,
finished_exex_height_rx,
);

2
crates/node/builder/src/launch/common.rs
View File

@ -329,7 +329,7 @@ impl<R> LaunchContextWith<Attached<WithConfigs, R>> {
/// Returns an initialized [`PrunerBuilder`] based on the configured [`PruneConfig`]
pub fn pruner_builder(&self) -> PrunerBuilder {
PrunerBuilder::new(self.prune_config().unwrap_or_default())
.delete_limit_per_block(self.chain_spec().prune_delete_limit)
.delete_limit(self.chain_spec().prune_delete_limit)
.timeout(PrunerBuilder::DEFAULT_TIMEOUT)
}

5
crates/node/builder/src/launch/mod.rs
View File

@ -231,14 +231,11 @@ where
let initial_target = ctx.node_config().debug.tip;
let mut pruner_builder = ctx
.pruner_builder()
.prune_max_blocks_per_run(ctx.tree_config().max_reorg_depth() as usize);
let mut pruner_builder = ctx.pruner_builder();
if let Some(exex_manager_handle) = &exex_manager_handle {
pruner_builder =
pruner_builder.finished_exex_height(exex_manager_handle.finished_height());
}
let pruner = pruner_builder.build_with_provider_factory(ctx.provider_factory().clone());
let pruner_events = pruner.events();

29
crates/prune/prune/src/builder.rs
View File

@ -15,12 +15,8 @@ pub struct PrunerBuilder {
block_interval: usize,
/// Pruning configuration for every part of the data that can be pruned.
segments: PruneModes,
/// The maximum number of blocks that can be pruned per run.
prune_max_blocks_per_run: usize,
/// The delete limit for pruner, per block. In the actual pruner run it will be multiplied by
/// the amount of blocks between pruner runs to account for the difference in amount of new
/// data coming in.
delete_limit_per_block: usize,
/// The delete limit for pruner, per run.
delete_limit: usize,
/// Time a pruner job can run before timing out.
timeout: Option<Duration>,
/// The finished height of all `ExEx`'s.
@ -50,15 +46,9 @@ impl PrunerBuilder {
self
}
/// Sets the number of blocks that can be re-orged.
pub const fn prune_max_blocks_per_run(mut self, prune_max_blocks_per_run: usize) -> Self {
self.prune_max_blocks_per_run = prune_max_blocks_per_run;
self
}
/// Sets the delete limit for pruner, per block.
pub const fn delete_limit_per_block(mut self, delete_limit_per_block: usize) -> Self {
self.delete_limit_per_block = delete_limit_per_block;
/// Sets the delete limit for pruner, per run.
pub const fn delete_limit(mut self, prune_delete_limit: usize) -> Self {
self.delete_limit = prune_delete_limit;
self
}
@ -91,8 +81,7 @@ impl PrunerBuilder {
provider_factory,
segments.into_vec(),
self.block_interval,
self.delete_limit_per_block,
self.prune_max_blocks_per_run,
self.delete_limit,
self.timeout,
self.finished_exex_height,
)
@ -105,8 +94,7 @@ impl PrunerBuilder {
Pruner::<_, ()>::new(
segments.into_vec(),
self.block_interval,
self.delete_limit_per_block,
self.prune_max_blocks_per_run,
self.delete_limit,
self.timeout,
self.finished_exex_height,
)
@ -118,8 +106,7 @@ impl Default for PrunerBuilder {
Self {
block_interval: 5,
segments: PruneModes::none(),
prune_max_blocks_per_run: 64,
delete_limit_per_block: MAINNET.prune_delete_limit,
delete_limit: MAINNET.prune_delete_limit,
timeout: None,
finished_exex_height: watch::channel(FinishedExExHeight::NoExExs).1,
}

40
crates/prune/prune/src/pruner.rs
View File

@ -37,11 +37,8 @@ pub struct Pruner<DB, PF> {
/// number is updated with the tip block number the pruner was called with. It's used in
/// conjunction with `min_block_interval` to determine when the pruning needs to be initiated.
previous_tip_block_number: Option<BlockNumber>,
/// Maximum total entries to prune (delete from database) per block.
delete_limit_per_block: usize,
/// Maximum number of blocks to be pruned per run, as an additional restriction to
/// `previous_tip_block_number`.
prune_max_blocks_per_run: usize,
/// Maximum total entries to prune (delete from database) per run.
delete_limit: usize,
/// Maximum time for a one pruner run.
timeout: Option<Duration>,
/// The finished height of all `ExEx`'s.
@ -56,8 +53,7 @@ impl<DB> Pruner<DB, ()> {
pub fn new(
segments: Vec<Box<dyn Segment<DB>>>,
min_block_interval: usize,
delete_limit_per_block: usize,
prune_max_blocks_per_run: usize,
delete_limit: usize,
timeout: Option<Duration>,
finished_exex_height: watch::Receiver<FinishedExExHeight>,
) -> Self {
@ -66,8 +62,7 @@ impl<DB> Pruner<DB, ()> {
segments,
min_block_interval,
previous_tip_block_number: None,
delete_limit_per_block,
prune_max_blocks_per_run,
delete_limit,
timeout,
finished_exex_height,
metrics: Metrics::default(),
@ -82,8 +77,7 @@ impl<DB: Database> Pruner<DB, ProviderFactory<DB>> {
provider_factory: ProviderFactory<DB>,
segments: Vec<Box<dyn Segment<DB>>>,
min_block_interval: usize,
delete_limit_per_block: usize,
prune_max_blocks_per_run: usize,
delete_limit: usize,
timeout: Option<Duration>,
finished_exex_height: watch::Receiver<FinishedExExHeight>,
) -> Self {
@ -92,8 +86,7 @@ impl<DB: Database> Pruner<DB, ProviderFactory<DB>> {
segments,
min_block_interval,
previous_tip_block_number: None,
delete_limit_per_block,
prune_max_blocks_per_run,
delete_limit,
timeout,
finished_exex_height,
metrics: Metrics::default(),
@ -130,25 +123,7 @@ impl<DB: Database, S> Pruner<DB, S> {
debug!(target: "pruner", %tip_block_number, "Pruner started");
let start = Instant::now();
// Multiply `self.delete_limit` (number of rows to delete per block) by number of blocks
// since last pruner run. `self.previous_tip_block_number` is close to
// `tip_block_number`, usually within `self.block_interval` blocks, so
// `delete_limit` will not be too high. If it's too high, we additionally limit it by
// `self.prune_max_blocks_per_run`.
//
// Also see docs for `self.previous_tip_block_number`.
let blocks_since_last_run = self
.previous_tip_block_number
.map_or(1, |previous_tip_block_number| {
// Saturating subtraction is needed for the case when the chain was reverted,
// meaning current block number might be less than the previous tip
// block number.
tip_block_number.saturating_sub(previous_tip_block_number) as usize
})
.min(self.prune_max_blocks_per_run);
let mut limiter = PruneLimiter::default()
.set_deleted_entries_limit(self.delete_limit_per_block * blocks_since_last_run);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(self.delete_limit);
if let Some(timeout) = self.timeout {
limiter = limiter.set_time_limit(timeout);
};
@ -411,7 +386,6 @@ mod tests {
vec![],
5,
0,
5,
None,
finished_exex_height_rx,
);

2
crates/prune/types/src/limiter.rs
View File

@ -7,7 +7,7 @@ use std::{
/// or the time it can run.
#[derive(Debug, Clone, Default)]
pub struct PruneLimiter {
/// Maximum entries (rows in the database) to delete from the database per block.
/// Maximum entries (rows in the database) to delete from the database per run.
deleted_entries_limit: Option<PruneDeletedEntriesLimit>,
/// Maximum duration of one prune run.
time_limit: Option<PruneTimeLimit>,

2
crates/stages/api/src/pipeline/mod.rs
View File

@ -256,7 +256,7 @@ where
// Run the pruner so we don't potentially end up with higher height in the database vs
// static files during a pipeline unwind
let mut pruner = PrunerBuilder::new(Default::default())
.delete_limit_per_block(usize::MAX)
.delete_limit(usize::MAX)
.build_with_provider_factory(self.provider_factory.clone());
pruner.run(prune_tip)?;

3
crates/stages/stages/src/stages/prune.rs
View File

@ -42,8 +42,7 @@ impl<DB: Database> Stage<DB> for PruneStage {
) -> Result<ExecOutput, StageError> {
let mut pruner = PrunerBuilder::default()
.segments(self.prune_modes.clone())
.prune_max_blocks_per_run(1)
.delete_limit_per_block(self.commit_threshold)
.delete_limit(self.commit_threshold)
.build();
let result = pruner.run(provider, input.target())?;