mirror of
https://github.com/hl-archive-node/nanoreth.git
synced 2025-12-06 19:09:54 +00:00
refactor(pruner): dedicated static file segments (#9262)
This commit is contained in:
@ -3,7 +3,7 @@ use reth_chainspec::MAINNET;
|
||||
use reth_config::PruneConfig;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_provider::ProviderFactory;
|
||||
use reth_provider::{providers::StaticFileProvider, ProviderFactory, StaticFileProviderFactory};
|
||||
use reth_prune_types::PruneModes;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::watch;
|
||||
@ -75,7 +75,10 @@ impl PrunerBuilder {
|
||||
self,
|
||||
provider_factory: ProviderFactory<DB>,
|
||||
) -> Pruner<DB, ProviderFactory<DB>> {
|
||||
let segments = SegmentSet::<DB>::from_prune_modes(self.segments);
|
||||
let segments = SegmentSet::<DB>::from_components(
|
||||
provider_factory.static_file_provider(),
|
||||
self.segments,
|
||||
);
|
||||
|
||||
Pruner::<_, ProviderFactory<DB>>::new(
|
||||
provider_factory,
|
||||
@ -87,9 +90,9 @@ impl PrunerBuilder {
|
||||
)
|
||||
}
|
||||
|
||||
/// Builds a [Pruner] from the current configuration.
|
||||
pub fn build<DB: Database>(self) -> Pruner<DB, ()> {
|
||||
let segments = SegmentSet::<DB>::from_prune_modes(self.segments);
|
||||
/// Builds a [Pruner] from the current configuration with the given static file provider.
|
||||
pub fn build<DB: Database>(self, static_file_provider: StaticFileProvider) -> Pruner<DB, ()> {
|
||||
let segments = SegmentSet::<DB>::from_components(static_file_provider, self.segments);
|
||||
|
||||
Pruner::<_, ()>::new(
|
||||
segments.into_vec(),
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
//! Support for pruning.
|
||||
|
||||
use crate::{
|
||||
segments,
|
||||
segments::{PruneInput, Segment},
|
||||
Metrics, PrunerError, PrunerEvent,
|
||||
};
|
||||
@ -9,8 +8,7 @@ use alloy_primitives::BlockNumber;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_provider::{DatabaseProviderRW, ProviderFactory, PruneCheckpointReader};
|
||||
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PrunePurpose, PruneSegment};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_prune_types::{PruneLimiter, PruneProgress, PruneSegment};
|
||||
use reth_tokio_util::{EventSender, EventStream};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::watch;
|
||||
@ -168,31 +166,27 @@ impl<DB: Database, S> Pruner<DB, S> {
|
||||
tip_block_number: BlockNumber,
|
||||
limiter: &mut PruneLimiter,
|
||||
) -> Result<(PrunerStats, usize, PruneProgress), PrunerError> {
|
||||
let static_file_segments = self.static_file_segments(provider);
|
||||
let segments = static_file_segments
|
||||
.iter()
|
||||
.map(|segment| (segment, PrunePurpose::StaticFile))
|
||||
.chain(self.segments.iter().map(|segment| (segment, PrunePurpose::User)));
|
||||
|
||||
let mut stats = PrunerStats::new();
|
||||
let mut pruned = 0;
|
||||
let mut progress = PruneProgress::Finished;
|
||||
|
||||
for (segment, purpose) in segments {
|
||||
for segment in &self.segments {
|
||||
if limiter.is_limit_reached() {
|
||||
break
|
||||
}
|
||||
|
||||
if let Some((to_block, prune_mode)) = segment
|
||||
.mode()
|
||||
.map(|mode| mode.prune_target_block(tip_block_number, segment.segment(), purpose))
|
||||
.map(|mode| {
|
||||
mode.prune_target_block(tip_block_number, segment.segment(), segment.purpose())
|
||||
})
|
||||
.transpose()?
|
||||
.flatten()
|
||||
{
|
||||
debug!(
|
||||
target: "pruner",
|
||||
segment = ?segment.segment(),
|
||||
?purpose,
|
||||
purpose = ?segment.purpose(),
|
||||
%to_block,
|
||||
?prune_mode,
|
||||
"Segment pruning started"
|
||||
@ -226,7 +220,7 @@ impl<DB: Database, S> Pruner<DB, S> {
|
||||
debug!(
|
||||
target: "pruner",
|
||||
segment = ?segment.segment(),
|
||||
?purpose,
|
||||
purpose = ?segment.purpose(),
|
||||
%to_block,
|
||||
?prune_mode,
|
||||
%output.pruned,
|
||||
@ -239,43 +233,13 @@ impl<DB: Database, S> Pruner<DB, S> {
|
||||
stats.push((segment.segment(), output.pruned, output.progress));
|
||||
}
|
||||
} else {
|
||||
debug!(target: "pruner", segment = ?segment.segment(), ?purpose, "Nothing to prune for the segment");
|
||||
debug!(target: "pruner", segment = ?segment.segment(), purpose = ?segment.purpose(), "Nothing to prune for the segment");
|
||||
}
|
||||
}
|
||||
|
||||
Ok((stats, pruned, progress))
|
||||
}
|
||||
|
||||
/// Returns pre-configured segments that needs to be pruned according to the highest
|
||||
/// `static_files` for [`PruneSegment::Transactions`], [`PruneSegment::Headers`] and
|
||||
/// [`PruneSegment::Receipts`].
|
||||
fn static_file_segments(&self, provider: &DatabaseProviderRW<DB>) -> Vec<Box<dyn Segment<DB>>> {
|
||||
let mut segments = Vec::<Box<dyn Segment<DB>>>::new();
|
||||
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
|
||||
if let Some(to_block) =
|
||||
static_file_provider.get_highest_static_file_block(StaticFileSegment::Transactions)
|
||||
{
|
||||
segments
|
||||
.push(Box::new(segments::Transactions::new(PruneMode::before_inclusive(to_block))))
|
||||
}
|
||||
|
||||
if let Some(to_block) =
|
||||
static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers)
|
||||
{
|
||||
segments.push(Box::new(segments::Headers::new(PruneMode::before_inclusive(to_block))))
|
||||
}
|
||||
|
||||
if let Some(to_block) =
|
||||
static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts)
|
||||
{
|
||||
segments.push(Box::new(segments::Receipts::new(PruneMode::before_inclusive(to_block))))
|
||||
}
|
||||
|
||||
segments
|
||||
}
|
||||
|
||||
/// Returns `true` if the pruning is needed at the provided tip block number.
|
||||
/// This determined by the check against minimum pruning interval and last pruned block number.
|
||||
pub fn is_pruning_needed(&self, tip_block_number: BlockNumber) -> bool {
|
||||
|
||||
@ -1,34 +1,29 @@
|
||||
mod account_history;
|
||||
mod headers;
|
||||
pub(super) mod history;
|
||||
mod receipts;
|
||||
mod receipts_by_logs;
|
||||
mod sender_recovery;
|
||||
mod set;
|
||||
mod storage_history;
|
||||
mod transaction_lookup;
|
||||
mod transactions;
|
||||
mod static_file;
|
||||
mod user;
|
||||
|
||||
use crate::PrunerError;
|
||||
pub use account_history::AccountHistory;
|
||||
use alloy_primitives::{BlockNumber, TxNumber};
|
||||
pub use headers::Headers;
|
||||
pub use receipts::Receipts;
|
||||
pub use receipts_by_logs::ReceiptsByLogs;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{
|
||||
errors::provider::ProviderResult, BlockReader, DatabaseProviderRW, PruneCheckpointWriter,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
|
||||
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PrunePurpose,
|
||||
PruneSegment,
|
||||
};
|
||||
pub use sender_recovery::SenderRecovery;
|
||||
pub use set::SegmentSet;
|
||||
pub use static_file::{
|
||||
Headers as StaticFileHeaders, Receipts as StaticFileReceipts,
|
||||
Transactions as StaticFileTransactions,
|
||||
};
|
||||
use std::{fmt::Debug, ops::RangeInclusive};
|
||||
pub use storage_history::StorageHistory;
|
||||
use tracing::error;
|
||||
pub use transaction_lookup::TransactionLookup;
|
||||
pub use transactions::Transactions;
|
||||
pub use user::{
|
||||
AccountHistory, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery, StorageHistory,
|
||||
TransactionLookup,
|
||||
};
|
||||
|
||||
/// A segment represents a pruning of some portion of the data.
|
||||
///
|
||||
@ -41,9 +36,12 @@ pub trait Segment<DB: Database>: Debug + Send + Sync {
|
||||
/// Segment of data that's pruned.
|
||||
fn segment(&self) -> PruneSegment;
|
||||
|
||||
/// Prune mode with which the segment was initialized
|
||||
/// Prune mode with which the segment was initialized.
|
||||
fn mode(&self) -> Option<PruneMode>;
|
||||
|
||||
/// Purpose of the segment.
|
||||
fn purpose(&self) -> PrunePurpose;
|
||||
|
||||
/// Prune data for [`Self::segment`] using the provided input.
|
||||
fn prune(
|
||||
&self,
|
||||
|
||||
@ -1,5 +1,12 @@
|
||||
//! Common receipts pruning logic shared between user and static file pruning segments.
|
||||
//!
|
||||
//! - [`crate::segments::user::Receipts`] is responsible for pruning receipts according to the
|
||||
//! user-configured settings (for example, on a full node or with a custom prune config)
|
||||
//! - [`crate::segments::static_file::Receipts`] is responsible for pruning receipts on an archive
|
||||
//! node after static file producer has finished
|
||||
|
||||
use crate::{
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db::tables;
|
||||
@ -8,35 +15,13 @@ use reth_provider::{
|
||||
errors::provider::ProviderResult, DatabaseProviderRW, PruneCheckpointWriter,
|
||||
TransactionsProvider,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment};
|
||||
use tracing::{instrument, trace};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneProgress, PruneSegment};
|
||||
use tracing::trace;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Receipts {
|
||||
mode: PruneMode,
|
||||
}
|
||||
|
||||
impl Receipts {
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
}
|
||||
}
|
||||
|
||||
impl<DB: Database> Segment<DB> for Receipts {
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::Receipts
|
||||
}
|
||||
|
||||
fn mode(&self) -> Option<PruneMode> {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
pub(crate) fn prune<DB: Database>(
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
input: PruneInput,
|
||||
) -> Result<PruneOutput, PrunerError> {
|
||||
) -> Result<PruneOutput, PrunerError> {
|
||||
let tx_range = match input.get_next_tx_num_range(provider)? {
|
||||
Some(range) => range,
|
||||
None => {
|
||||
@ -74,13 +59,12 @@ impl<DB: Database> Segment<DB> for Receipts {
|
||||
tx_number: Some(last_pruned_transaction),
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn save_checkpoint(
|
||||
&self,
|
||||
pub(crate) fn save_checkpoint<DB: Database>(
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
checkpoint: PruneCheckpoint,
|
||||
) -> ProviderResult<()> {
|
||||
) -> ProviderResult<()> {
|
||||
provider.save_prune_checkpoint(PruneSegment::Receipts, checkpoint)?;
|
||||
|
||||
// `PruneSegment::Receipts` overrides `PruneSegment::ContractLogs`, so we can preemptively
|
||||
@ -88,12 +72,11 @@ impl<DB: Database> Segment<DB> for Receipts {
|
||||
provider.save_prune_checkpoint(PruneSegment::ContractLogs, checkpoint)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{PruneInput, PruneOutput, Receipts, Segment};
|
||||
use crate::segments::{PruneInput, PruneOutput};
|
||||
use alloy_primitives::{BlockNumber, TxNumber, B256};
|
||||
use assert_matches::assert_matches;
|
||||
use itertools::{
|
||||
@ -140,7 +123,6 @@ mod tests {
|
||||
|
||||
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let segment = Receipts::new(prune_mode);
|
||||
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
|
||||
let input = PruneInput {
|
||||
previous_checkpoint: db
|
||||
@ -175,7 +157,7 @@ mod tests {
|
||||
.sub(1);
|
||||
|
||||
let provider = db.factory.provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
let result = super::prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
assert_matches!(
|
||||
@ -184,8 +166,7 @@ mod tests {
|
||||
if (progress, pruned) == expected_result
|
||||
);
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
super::save_checkpoint(
|
||||
&provider,
|
||||
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
use crate::segments::{
|
||||
AccountHistory, Receipts, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory,
|
||||
TransactionLookup,
|
||||
AccountHistory, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory, TransactionLookup,
|
||||
UserReceipts,
|
||||
};
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::providers::StaticFileProvider;
|
||||
use reth_prune_types::PruneModes;
|
||||
|
||||
use super::{StaticFileHeaders, StaticFileReceipts, StaticFileTransactions};
|
||||
|
||||
/// Collection of [Segment]. Thread-safe, allocated on the heap.
|
||||
#[derive(Debug)]
|
||||
pub struct SegmentSet<DB: Database> {
|
||||
@ -36,8 +39,12 @@ impl<DB: Database> SegmentSet<DB> {
|
||||
self.inner
|
||||
}
|
||||
|
||||
/// Creates a [`SegmentSet`] from an existing [`PruneModes`].
|
||||
pub fn from_prune_modes(prune_modes: PruneModes) -> Self {
|
||||
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
|
||||
/// [`PruneModes`].
|
||||
pub fn from_components(
|
||||
static_file_provider: StaticFileProvider,
|
||||
prune_modes: PruneModes,
|
||||
) -> Self {
|
||||
let PruneModes {
|
||||
sender_recovery,
|
||||
transaction_lookup,
|
||||
@ -48,12 +55,18 @@ impl<DB: Database> SegmentSet<DB> {
|
||||
} = prune_modes;
|
||||
|
||||
Self::default()
|
||||
// Static file headers
|
||||
.segment(StaticFileHeaders::new(static_file_provider.clone()))
|
||||
// Static file transactions
|
||||
.segment(StaticFileTransactions::new(static_file_provider.clone()))
|
||||
// Static file receipts
|
||||
.segment(StaticFileReceipts::new(static_file_provider))
|
||||
// Account history
|
||||
.segment_opt(account_history.map(AccountHistory::new))
|
||||
// Storage history
|
||||
.segment_opt(storage_history.map(StorageHistory::new))
|
||||
// Receipts
|
||||
.segment_opt(receipts.map(Receipts::new))
|
||||
// User receipts
|
||||
.segment_opt(receipts.map(UserReceipts::new))
|
||||
// Receipts by logs
|
||||
.segment_opt(
|
||||
(!receipts_log_filter.is_empty())
|
||||
|
||||
@ -4,30 +4,30 @@ use crate::{
|
||||
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use alloy_primitives::BlockNumber;
|
||||
use itertools::Itertools;
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{
|
||||
use reth_db::{
|
||||
cursor::{DbCursorRO, RangeWalker},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
|
||||
use alloy_primitives::BlockNumber;
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PruneSegment};
|
||||
use tracing::{instrument, trace};
|
||||
use reth_provider::{providers::StaticFileProvider, DatabaseProviderRW};
|
||||
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PrunePurpose, PruneSegment};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use tracing::trace;
|
||||
|
||||
/// Number of header tables to prune in one step
|
||||
const HEADER_TABLES_TO_PRUNE: usize = 3;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Headers {
|
||||
mode: PruneMode,
|
||||
static_file_provider: StaticFileProvider,
|
||||
}
|
||||
|
||||
impl Headers {
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
|
||||
Self { static_file_provider }
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,10 +37,15 @@ impl<DB: Database> Segment<DB> for Headers {
|
||||
}
|
||||
|
||||
fn mode(&self) -> Option<PruneMode> {
|
||||
Some(self.mode)
|
||||
self.static_file_provider
|
||||
.get_highest_static_file_block(StaticFileSegment::Headers)
|
||||
.map(PruneMode::before_inclusive)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::StaticFile
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
@ -98,7 +103,6 @@ impl<DB: Database> Segment<DB> for Headers {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type Walker<'a, DB, T> = RangeWalker<'a, T, <<DB as Database>::TXMut as DbTxMut>::CursorMut<T>>;
|
||||
|
||||
#[allow(missing_debug_implementations)]
|
||||
@ -188,11 +192,15 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{
|
||||
static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneOutput,
|
||||
PruneOutputCheckpoint, Segment,
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256, U256};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::tables;
|
||||
use reth_db_api::transaction::DbTx;
|
||||
use reth_provider::PruneCheckpointReader;
|
||||
use reth_provider::{PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
|
||||
};
|
||||
@ -200,11 +208,6 @@ mod tests {
|
||||
use reth_testing_utils::{generators, generators::random_header_range};
|
||||
use tracing::trace;
|
||||
|
||||
use crate::segments::{
|
||||
headers::HEADER_TABLES_TO_PRUNE, Headers, PruneInput, PruneOutput, PruneOutputCheckpoint,
|
||||
Segment,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn prune() {
|
||||
reth_tracing::init_test_tracing();
|
||||
@ -224,8 +227,8 @@ mod tests {
|
||||
assert_eq!(db.table::<tables::HeaderTerminalDifficulties>().unwrap().len(), headers.len());
|
||||
|
||||
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
|
||||
let segment = super::Headers::new(db.factory.static_file_provider());
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let segment = Headers::new(prune_mode);
|
||||
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
|
||||
let input = PruneInput {
|
||||
previous_checkpoint: db
|
||||
@ -263,9 +266,9 @@ mod tests {
|
||||
PruneOutput {progress, pruned, checkpoint: Some(_)}
|
||||
if (progress, pruned) == expected_result
|
||||
);
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider,
|
||||
provider
|
||||
.save_prune_checkpoint(
|
||||
PruneSegment::Headers,
|
||||
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
@ -310,7 +313,6 @@ mod tests {
|
||||
fn prune_cannot_be_done() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
let segment = Headers::new(PruneMode::Full);
|
||||
let limiter = PruneLimiter::default().set_deleted_entries_limit(0);
|
||||
|
||||
let input = PruneInput {
|
||||
@ -321,6 +323,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let provider = db.factory.provider_rw().unwrap();
|
||||
let segment = super::Headers::new(db.factory.static_file_provider());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
7
crates/prune/prune/src/segments/static_file/mod.rs
Normal file
7
crates/prune/prune/src/segments/static_file/mod.rs
Normal file
@ -0,0 +1,7 @@
|
||||
mod headers;
|
||||
mod receipts;
|
||||
mod transactions;
|
||||
|
||||
pub use headers::Headers;
|
||||
pub use receipts::Receipts;
|
||||
pub use transactions::Transactions;
|
||||
53
crates/prune/prune/src/segments/static_file/receipts.rs
Normal file
53
crates/prune/prune/src/segments/static_file/receipts.rs
Normal file
@ -0,0 +1,53 @@
|
||||
use crate::{
|
||||
segments::{PruneInput, PruneOutput, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{
|
||||
errors::provider::ProviderResult, providers::StaticFileProvider, DatabaseProviderRW,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Receipts {
|
||||
static_file_provider: StaticFileProvider,
|
||||
}
|
||||
|
||||
impl Receipts {
|
||||
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
|
||||
Self { static_file_provider }
|
||||
}
|
||||
}
|
||||
|
||||
impl<DB: Database> Segment<DB> for Receipts {
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::Receipts
|
||||
}
|
||||
|
||||
fn mode(&self) -> Option<PruneMode> {
|
||||
self.static_file_provider
|
||||
.get_highest_static_file_block(StaticFileSegment::Receipts)
|
||||
.map(PruneMode::before_inclusive)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::StaticFile
|
||||
}
|
||||
|
||||
fn prune(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
input: PruneInput,
|
||||
) -> Result<PruneOutput, PrunerError> {
|
||||
crate::segments::receipts::prune(provider, input)
|
||||
}
|
||||
|
||||
fn save_checkpoint(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
checkpoint: PruneCheckpoint,
|
||||
) -> ProviderResult<()> {
|
||||
crate::segments::receipts::save_checkpoint(provider, checkpoint)
|
||||
}
|
||||
}
|
||||
@ -4,18 +4,19 @@ use crate::{
|
||||
};
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
|
||||
use tracing::{instrument, trace};
|
||||
use reth_provider::{providers::StaticFileProvider, DatabaseProviderRW, TransactionsProvider};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use tracing::trace;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Transactions {
|
||||
mode: PruneMode,
|
||||
static_file_provider: StaticFileProvider,
|
||||
}
|
||||
|
||||
impl Transactions {
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
|
||||
Self { static_file_provider }
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,10 +26,15 @@ impl<DB: Database> Segment<DB> for Transactions {
|
||||
}
|
||||
|
||||
fn mode(&self) -> Option<PruneMode> {
|
||||
Some(self.mode)
|
||||
self.static_file_provider
|
||||
.get_highest_static_file_block(StaticFileSegment::Transactions)
|
||||
.map(PruneMode::before_inclusive)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::StaticFile
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
@ -75,7 +81,7 @@ impl<DB: Database> Segment<DB> for Transactions {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{PruneInput, PruneOutput, Segment, Transactions};
|
||||
use crate::segments::{PruneInput, PruneOutput, Segment};
|
||||
use alloy_primitives::{BlockNumber, TxNumber, B256};
|
||||
use assert_matches::assert_matches;
|
||||
use itertools::{
|
||||
@ -83,7 +89,7 @@ mod tests {
|
||||
Itertools,
|
||||
};
|
||||
use reth_db::tables;
|
||||
use reth_provider::PruneCheckpointReader;
|
||||
use reth_provider::{PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
|
||||
};
|
||||
@ -104,8 +110,8 @@ mod tests {
|
||||
assert_eq!(db.table::<tables::Transactions>().unwrap().len(), transactions.len());
|
||||
|
||||
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
|
||||
let segment = super::Transactions::new(db.factory.static_file_provider());
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let segment = Transactions::new(prune_mode);
|
||||
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
|
||||
let input = PruneInput {
|
||||
previous_checkpoint: db
|
||||
@ -138,9 +144,9 @@ mod tests {
|
||||
if (progress, pruned) == expected_result
|
||||
);
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider,
|
||||
provider
|
||||
.save_prune_checkpoint(
|
||||
PruneSegment::Transactions,
|
||||
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
@ -1,6 +1,7 @@
|
||||
use crate::{
|
||||
segments::{
|
||||
history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint, Segment,
|
||||
user::history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint,
|
||||
Segment,
|
||||
},
|
||||
PrunerError,
|
||||
};
|
||||
@ -8,7 +9,9 @@ use itertools::Itertools;
|
||||
use reth_db::tables;
|
||||
use reth_db_api::{database::Database, models::ShardedKey};
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use reth_prune_types::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_prune_types::{
|
||||
PruneInterruptReason, PruneMode, PruneProgress, PrunePurpose, PruneSegment,
|
||||
};
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@ -38,6 +41,10 @@ impl<DB: Database> Segment<DB> for AccountHistory {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
@ -124,8 +131,8 @@ impl<DB: Database> Segment<DB> for AccountHistory {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{
|
||||
account_history::ACCOUNT_HISTORY_TABLES_TO_PRUNE, AccountHistory, PruneInput, PruneOutput,
|
||||
Segment,
|
||||
user::account_history::ACCOUNT_HISTORY_TABLES_TO_PRUNE, AccountHistory, PruneInput,
|
||||
PruneOutput, Segment,
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use assert_matches::assert_matches;
|
||||
14
crates/prune/prune/src/segments/user/mod.rs
Normal file
14
crates/prune/prune/src/segments/user/mod.rs
Normal file
@ -0,0 +1,14 @@
|
||||
mod account_history;
|
||||
mod history;
|
||||
mod receipts;
|
||||
mod receipts_by_logs;
|
||||
mod sender_recovery;
|
||||
mod storage_history;
|
||||
mod transaction_lookup;
|
||||
|
||||
pub use account_history::AccountHistory;
|
||||
pub use receipts::Receipts;
|
||||
pub use receipts_by_logs::ReceiptsByLogs;
|
||||
pub use sender_recovery::SenderRecovery;
|
||||
pub use storage_history::StorageHistory;
|
||||
pub use transaction_lookup::TransactionLookup;
|
||||
50
crates/prune/prune/src/segments/user/receipts.rs
Normal file
50
crates/prune/prune/src/segments/user/receipts.rs
Normal file
@ -0,0 +1,50 @@
|
||||
use crate::{
|
||||
segments::{PruneInput, PruneOutput, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{errors::provider::ProviderResult, DatabaseProviderRW};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use tracing::instrument;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Receipts {
|
||||
mode: PruneMode,
|
||||
}
|
||||
|
||||
impl Receipts {
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
}
|
||||
}
|
||||
|
||||
impl<DB: Database> Segment<DB> for Receipts {
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::Receipts
|
||||
}
|
||||
|
||||
fn mode(&self) -> Option<PruneMode> {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
input: PruneInput,
|
||||
) -> Result<PruneOutput, PrunerError> {
|
||||
crate::segments::receipts::prune(provider, input)
|
||||
}
|
||||
|
||||
fn save_checkpoint(
|
||||
&self,
|
||||
provider: &DatabaseProviderRW<DB>,
|
||||
checkpoint: PruneCheckpoint,
|
||||
) -> ProviderResult<()> {
|
||||
crate::segments::receipts::save_checkpoint(provider, checkpoint)
|
||||
}
|
||||
}
|
||||
@ -31,6 +31,10 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
|
||||
None
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
@ -215,7 +219,7 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{receipts_by_logs::ReceiptsByLogs, PruneInput, Segment};
|
||||
use crate::segments::{PruneInput, ReceiptsByLogs, Segment};
|
||||
use alloy_primitives::B256;
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db::tables;
|
||||
@ -5,7 +5,7 @@ use crate::{
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
#[derive(Debug)]
|
||||
@ -28,6 +28,10 @@ impl<DB: Database> Segment<DB> for SenderRecovery {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
@ -1,6 +1,7 @@
|
||||
use crate::{
|
||||
segments::{
|
||||
history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint, Segment,
|
||||
user::history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint,
|
||||
Segment,
|
||||
},
|
||||
PrunerError,
|
||||
};
|
||||
@ -11,7 +12,9 @@ use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress},
|
||||
};
|
||||
use reth_provider::DatabaseProviderRW;
|
||||
use reth_prune_types::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_prune_types::{
|
||||
PruneInterruptReason, PruneMode, PruneProgress, PrunePurpose, PruneSegment,
|
||||
};
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@ -41,6 +44,10 @@ impl<DB: Database> Segment<DB> for StorageHistory {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
@ -132,7 +139,7 @@ impl<DB: Database> Segment<DB> for StorageHistory {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{
|
||||
storage_history::STORAGE_HISTORY_TABLES_TO_PRUNE, PruneInput, PruneOutput, Segment,
|
||||
user::storage_history::STORAGE_HISTORY_TABLES_TO_PRUNE, PruneInput, PruneOutput, Segment,
|
||||
StorageHistory,
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
@ -6,7 +6,7 @@ use rayon::prelude::*;
|
||||
use reth_db::tables;
|
||||
use reth_db_api::database::Database;
|
||||
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
#[derive(Debug)]
|
||||
@ -29,6 +29,10 @@ impl<DB: Database> Segment<DB> for TransactionLookup {
|
||||
Some(self.mode)
|
||||
}
|
||||
|
||||
fn purpose(&self) -> PrunePurpose {
|
||||
PrunePurpose::User
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
|
||||
fn prune(
|
||||
&self,
|
||||
@ -43,7 +43,7 @@ impl<DB: Database> Stage<DB> for PruneStage {
|
||||
let mut pruner = PrunerBuilder::default()
|
||||
.segments(self.prune_modes.clone())
|
||||
.delete_limit(self.commit_threshold)
|
||||
.build();
|
||||
.build(provider.static_file_provider().clone());
|
||||
|
||||
let result = pruner.run(provider, input.target())?;
|
||||
if result.is_finished() {
|
||||
|
||||
Reference in New Issue
Block a user