jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(pruner): dedicated static file segments (#9262)

Browse Source
This commit is contained in:
Alexey Shekhirin
2024-07-12 13:02:40 +01:00
committed by GitHub
parent d943e78294
commit 61c85bb3e5
18 changed files with 326 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
crates/prune/prune/src/builder.rs
View File

@ -3,7 +3,7 @@ use reth_chainspec::MAINNET;
use reth_config::PruneConfig;
use reth_db_api::database::Database;
use reth_exex_types::FinishedExExHeight;
use reth_provider::ProviderFactory;
use reth_provider::{providers::StaticFileProvider, ProviderFactory, StaticFileProviderFactory};
use reth_prune_types::PruneModes;
use std::time::Duration;
use tokio::sync::watch;
@ -75,7 +75,10 @@ impl PrunerBuilder {
self,
provider_factory: ProviderFactory<DB>,
) -> Pruner<DB, ProviderFactory<DB>> {
let segments = SegmentSet::<DB>::from_prune_modes(self.segments);
let segments = SegmentSet::<DB>::from_components(
provider_factory.static_file_provider(),
self.segments,
);
Pruner::<_, ProviderFactory<DB>>::new(
provider_factory,
@ -87,9 +90,9 @@ impl PrunerBuilder {
)
}
/// Builds a [Pruner] from the current configuration.
pub fn build<DB: Database>(self) -> Pruner<DB, ()> {
let segments = SegmentSet::<DB>::from_prune_modes(self.segments);
/// Builds a [Pruner] from the current configuration with the given static file provider.
pub fn build<DB: Database>(self, static_file_provider: StaticFileProvider) -> Pruner<DB, ()> {
let segments = SegmentSet::<DB>::from_components(static_file_provider, self.segments);
Pruner::<_, ()>::new(
segments.into_vec(),

52
crates/prune/prune/src/pruner.rs
View File

@ -1,7 +1,6 @@
//! Support for pruning.
use crate::{
segments,
segments::{PruneInput, Segment},
Metrics, PrunerError, PrunerEvent,
};
@ -9,8 +8,7 @@ use alloy_primitives::BlockNumber;
use reth_db_api::database::Database;
use reth_exex_types::FinishedExExHeight;
use reth_provider::{DatabaseProviderRW, ProviderFactory, PruneCheckpointReader};
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PrunePurpose, PruneSegment};
use reth_static_file_types::StaticFileSegment;
use reth_prune_types::{PruneLimiter, PruneProgress, PruneSegment};
use reth_tokio_util::{EventSender, EventStream};
use std::time::{Duration, Instant};
use tokio::sync::watch;
@ -168,31 +166,27 @@ impl<DB: Database, S> Pruner<DB, S> {
tip_block_number: BlockNumber,
limiter: &mut PruneLimiter,
) -> Result<(PrunerStats, usize, PruneProgress), PrunerError> {
let static_file_segments = self.static_file_segments(provider);
let segments = static_file_segments
.iter()
.map(|segment| (segment, PrunePurpose::StaticFile))
.chain(self.segments.iter().map(|segment| (segment, PrunePurpose::User)));
let mut stats = PrunerStats::new();
let mut pruned = 0;
let mut progress = PruneProgress::Finished;
for (segment, purpose) in segments {
for segment in &self.segments {
if limiter.is_limit_reached() {
break
}
if let Some((to_block, prune_mode)) = segment
.mode()
.map(|mode| mode.prune_target_block(tip_block_number, segment.segment(), purpose))
.map(|mode| {
mode.prune_target_block(tip_block_number, segment.segment(), segment.purpose())
})
.transpose()?
.flatten()
{
debug!(
target: "pruner",
segment = ?segment.segment(),
?purpose,
purpose = ?segment.purpose(),
%to_block,
?prune_mode,
"Segment pruning started"
@ -226,7 +220,7 @@ impl<DB: Database, S> Pruner<DB, S> {
debug!(
target: "pruner",
segment = ?segment.segment(),
?purpose,
purpose = ?segment.purpose(),
%to_block,
?prune_mode,
%output.pruned,
@ -239,43 +233,13 @@ impl<DB: Database, S> Pruner<DB, S> {
stats.push((segment.segment(), output.pruned, output.progress));
}
} else {
debug!(target: "pruner", segment = ?segment.segment(), ?purpose, "Nothing to prune for the segment");
debug!(target: "pruner", segment = ?segment.segment(), purpose = ?segment.purpose(), "Nothing to prune for the segment");
}
}
Ok((stats, pruned, progress))
}
/// Returns pre-configured segments that needs to be pruned according to the highest
/// `static_files` for [`PruneSegment::Transactions`], [`PruneSegment::Headers`] and
/// [`PruneSegment::Receipts`].
fn static_file_segments(&self, provider: &DatabaseProviderRW<DB>) -> Vec<Box<dyn Segment<DB>>> {
let mut segments = Vec::<Box<dyn Segment<DB>>>::new();
let static_file_provider = provider.static_file_provider();
if let Some(to_block) =
static_file_provider.get_highest_static_file_block(StaticFileSegment::Transactions)
{
segments
.push(Box::new(segments::Transactions::new(PruneMode::before_inclusive(to_block))))
}
if let Some(to_block) =
static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers)
{
segments.push(Box::new(segments::Headers::new(PruneMode::before_inclusive(to_block))))
}
if let Some(to_block) =
static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts)
{
segments.push(Box::new(segments::Receipts::new(PruneMode::before_inclusive(to_block))))
}
segments
}
/// Returns `true` if the pruning is needed at the provided tip block number.
/// This determined by the check against minimum pruning interval and last pruned block number.
pub fn is_pruning_needed(&self, tip_block_number: BlockNumber) -> bool {

34
crates/prune/prune/src/segments/mod.rs
View File

@ -1,34 +1,29 @@
mod account_history;
mod headers;
pub(super) mod history;
mod receipts;
mod receipts_by_logs;
mod sender_recovery;
mod set;
mod storage_history;
mod transaction_lookup;
mod transactions;
mod static_file;
mod user;
use crate::PrunerError;
pub use account_history::AccountHistory;
use alloy_primitives::{BlockNumber, TxNumber};
pub use headers::Headers;
pub use receipts::Receipts;
pub use receipts_by_logs::ReceiptsByLogs;
use reth_db_api::database::Database;
use reth_provider::{
errors::provider::ProviderResult, BlockReader, DatabaseProviderRW, PruneCheckpointWriter,
};
use reth_prune_types::{
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PrunePurpose,
PruneSegment,
};
pub use sender_recovery::SenderRecovery;
pub use set::SegmentSet;
pub use static_file::{
Headers as StaticFileHeaders, Receipts as StaticFileReceipts,
Transactions as StaticFileTransactions,
};
use std::{fmt::Debug, ops::RangeInclusive};
pub use storage_history::StorageHistory;
use tracing::error;
pub use transaction_lookup::TransactionLookup;
pub use transactions::Transactions;
pub use user::{
AccountHistory, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery, StorageHistory,
TransactionLookup,
};
/// A segment represents a pruning of some portion of the data.
///
@ -41,9 +36,12 @@ pub trait Segment<DB: Database>: Debug + Send + Sync {
/// Segment of data that's pruned.
fn segment(&self) -> PruneSegment;
/// Prune mode with which the segment was initialized
/// Prune mode with which the segment was initialized.
fn mode(&self) -> Option<PruneMode>;
/// Purpose of the segment.
fn purpose(&self) -> PrunePurpose;
/// Prune data for [`Self::segment`] using the provided input.
fn prune(
&self,

153
crates/prune/prune/src/segments/receipts.rs
View File

@ -1,5 +1,12 @@
//! Common receipts pruning logic shared between user and static file pruning segments.
//!
//! - [`crate::segments::user::Receipts`] is responsible for pruning receipts according to the
//! user-configured settings (for example, on a full node or with a custom prune config)
//! - [`crate::segments::static_file::Receipts`] is responsible for pruning receipts on an archive
//! node after static file producer has finished
use crate::{
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint},
PrunerError,
};
use reth_db::tables;
@ -8,92 +15,68 @@ use reth_provider::{
errors::provider::ProviderResult, DatabaseProviderRW, PruneCheckpointWriter,
TransactionsProvider,
};
use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment};
use tracing::{instrument, trace};
use reth_prune_types::{PruneCheckpoint, PruneProgress, PruneSegment};
use tracing::trace;
#[derive(Debug)]
pub struct Receipts {
mode: PruneMode,
pub(crate) fn prune<DB: Database>(
provider: &DatabaseProviderRW<DB>,
input: PruneInput,
) -> Result<PruneOutput, PrunerError> {
let tx_range = match input.get_next_tx_num_range(provider)? {
Some(range) => range,
None => {
trace!(target: "pruner", "No receipts to prune");
return Ok(PruneOutput::done())
}
};
let tx_range_end = *tx_range.end();
let mut limiter = input.limiter;
let mut last_pruned_transaction = tx_range_end;
let (pruned, done) = provider.prune_table_with_range::<tables::Receipts>(
tx_range,
&mut limiter,
|_| false,
|row| last_pruned_transaction = row.0,
)?;
trace!(target: "pruner", %pruned, %done, "Pruned receipts");
let last_pruned_block = provider
.transaction_block(last_pruned_transaction)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
// If there's more receipts to prune, set the checkpoint block number to previous,
// so we could finish pruning its receipts on the next run.
.checked_sub(if done { 0 } else { 1 });
let progress = PruneProgress::new(done, &limiter);
Ok(PruneOutput {
progress,
pruned,
checkpoint: Some(PruneOutputCheckpoint {
block_number: last_pruned_block,
tx_number: Some(last_pruned_transaction),
}),
})
}
impl Receipts {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
}
}
pub(crate) fn save_checkpoint<DB: Database>(
provider: &DatabaseProviderRW<DB>,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
provider.save_prune_checkpoint(PruneSegment::Receipts, checkpoint)?;
impl<DB: Database> Segment<DB> for Receipts {
fn segment(&self) -> PruneSegment {
PruneSegment::Receipts
}
// `PruneSegment::Receipts` overrides `PruneSegment::ContractLogs`, so we can preemptively
// limit their pruning start point.
provider.save_prune_checkpoint(PruneSegment::ContractLogs, checkpoint)?;
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
provider: &DatabaseProviderRW<DB>,
input: PruneInput,
) -> Result<PruneOutput, PrunerError> {
let tx_range = match input.get_next_tx_num_range(provider)? {
Some(range) => range,
None => {
trace!(target: "pruner", "No receipts to prune");
return Ok(PruneOutput::done())
}
};
let tx_range_end = *tx_range.end();
let mut limiter = input.limiter;
let mut last_pruned_transaction = tx_range_end;
let (pruned, done) = provider.prune_table_with_range::<tables::Receipts>(
tx_range,
&mut limiter,
|_| false,
|row| last_pruned_transaction = row.0,
)?;
trace!(target: "pruner", %pruned, %done, "Pruned receipts");
let last_pruned_block = provider
.transaction_block(last_pruned_transaction)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
// If there's more receipts to prune, set the checkpoint block number to previous,
// so we could finish pruning its receipts on the next run.
.checked_sub(if done { 0 } else { 1 });
let progress = PruneProgress::new(done, &limiter);
Ok(PruneOutput {
progress,
pruned,
checkpoint: Some(PruneOutputCheckpoint {
block_number: last_pruned_block,
tx_number: Some(last_pruned_transaction),
}),
})
}
fn save_checkpoint(
&self,
provider: &DatabaseProviderRW<DB>,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
provider.save_prune_checkpoint(PruneSegment::Receipts, checkpoint)?;
// `PruneSegment::Receipts` overrides `PruneSegment::ContractLogs`, so we can preemptively
// limit their pruning start point.
provider.save_prune_checkpoint(PruneSegment::ContractLogs, checkpoint)?;
Ok(())
}
Ok(())
}
#[cfg(test)]
mod tests {
use crate::segments::{PruneInput, PruneOutput, Receipts, Segment};
use crate::segments::{PruneInput, PruneOutput};
use alloy_primitives::{BlockNumber, TxNumber, B256};
use assert_matches::assert_matches;
use itertools::{
@ -140,7 +123,6 @@ mod tests {
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
let prune_mode = PruneMode::Before(to_block);
let segment = Receipts::new(prune_mode);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
let input = PruneInput {
previous_checkpoint: db
@ -175,7 +157,7 @@ mod tests {
.sub(1);
let provider = db.factory.provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
let result = super::prune(&provider, input).unwrap();
limiter.increment_deleted_entries_count_by(result.pruned);
assert_matches!(
@ -184,12 +166,11 @@ mod tests {
if (progress, pruned) == expected_result
);
segment
.save_checkpoint(
&provider,
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
super::save_checkpoint(
&provider,
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
provider.commit().expect("commit");
let last_pruned_block_number = blocks

25
crates/prune/prune/src/segments/set.rs
View File

@ -1,10 +1,13 @@
use crate::segments::{
AccountHistory, Receipts, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory,
TransactionLookup,
AccountHistory, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory, TransactionLookup,
UserReceipts,
};
use reth_db_api::database::Database;
use reth_provider::providers::StaticFileProvider;
use reth_prune_types::PruneModes;
use super::{StaticFileHeaders, StaticFileReceipts, StaticFileTransactions};
/// Collection of [Segment]. Thread-safe, allocated on the heap.
#[derive(Debug)]
pub struct SegmentSet<DB: Database> {
@ -36,8 +39,12 @@ impl<DB: Database> SegmentSet<DB> {
self.inner
}
/// Creates a [`SegmentSet`] from an existing [`PruneModes`].
pub fn from_prune_modes(prune_modes: PruneModes) -> Self {
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
/// [`PruneModes`].
pub fn from_components(
static_file_provider: StaticFileProvider,
prune_modes: PruneModes,
) -> Self {
let PruneModes {
sender_recovery,
transaction_lookup,
@ -48,12 +55,18 @@ impl<DB: Database> SegmentSet<DB> {
} = prune_modes;
Self::default()
// Static file headers
.segment(StaticFileHeaders::new(static_file_provider.clone()))
// Static file transactions
.segment(StaticFileTransactions::new(static_file_provider.clone()))
// Static file receipts
.segment(StaticFileReceipts::new(static_file_provider))
// Account history
.segment_opt(account_history.map(AccountHistory::new))
// Storage history
.segment_opt(storage_history.map(StorageHistory::new))
// Receipts
.segment_opt(receipts.map(Receipts::new))
// User receipts
.segment_opt(receipts.map(UserReceipts::new))
// Receipts by logs
.segment_opt(
(!receipts_log_filter.is_empty())

51
crates/prune/prune/src/segments/headers.rs → crates/prune/prune/src/segments/static_file/headers.rs
View File

@ -4,30 +4,30 @@ use crate::{
segments::{PruneInput, PruneOutput, PruneOutputCheckpoint, Segment},
PrunerError,
};
use alloy_primitives::BlockNumber;
use itertools::Itertools;
use reth_db::tables;
use reth_db_api::{
use reth_db::{
cursor::{DbCursorRO, RangeWalker},
database::Database,
tables,
transaction::DbTxMut,
};
use alloy_primitives::BlockNumber;
use reth_provider::DatabaseProviderRW;
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PruneSegment};
use tracing::{instrument, trace};
use reth_provider::{providers::StaticFileProvider, DatabaseProviderRW};
use reth_prune_types::{PruneLimiter, PruneMode, PruneProgress, PrunePurpose, PruneSegment};
use reth_static_file_types::StaticFileSegment;
use tracing::trace;
/// Number of header tables to prune in one step
const HEADER_TABLES_TO_PRUNE: usize = 3;
#[derive(Debug)]
pub struct Headers {
mode: PruneMode,
static_file_provider: StaticFileProvider,
}
impl Headers {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
Self { static_file_provider }
}
}
@ -37,10 +37,15 @@ impl<DB: Database> Segment<DB> for Headers {
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
self.static_file_provider
.get_highest_static_file_block(StaticFileSegment::Headers)
.map(PruneMode::before_inclusive)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::StaticFile
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
provider: &DatabaseProviderRW<DB>,
@ -98,7 +103,6 @@ impl<DB: Database> Segment<DB> for Headers {
})
}
}
type Walker<'a, DB, T> = RangeWalker<'a, T, <<DB as Database>::TXMut as DbTxMut>::CursorMut<T>>;
#[allow(missing_debug_implementations)]
@ -188,11 +192,15 @@ where
#[cfg(test)]
mod tests {
use crate::segments::{
static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneOutput,
PruneOutputCheckpoint, Segment,
};
use alloy_primitives::{BlockNumber, B256, U256};
use assert_matches::assert_matches;
use reth_db::tables;
use reth_db_api::transaction::DbTx;
use reth_provider::PruneCheckpointReader;
use reth_provider::{PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory};
use reth_prune_types::{
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
};
@ -200,11 +208,6 @@ mod tests {
use reth_testing_utils::{generators, generators::random_header_range};
use tracing::trace;
use crate::segments::{
headers::HEADER_TABLES_TO_PRUNE, Headers, PruneInput, PruneOutput, PruneOutputCheckpoint,
Segment,
};
#[test]
fn prune() {
reth_tracing::init_test_tracing();
@ -224,8 +227,8 @@ mod tests {
assert_eq!(db.table::<tables::HeaderTerminalDifficulties>().unwrap().len(), headers.len());
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
let segment = super::Headers::new(db.factory.static_file_provider());
let prune_mode = PruneMode::Before(to_block);
let segment = Headers::new(prune_mode);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
let input = PruneInput {
previous_checkpoint: db
@ -263,9 +266,9 @@ mod tests {
PruneOutput {progress, pruned, checkpoint: Some(_)}
if (progress, pruned) == expected_result
);
segment
.save_checkpoint(
&provider,
provider
.save_prune_checkpoint(
PruneSegment::Headers,
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
@ -310,7 +313,6 @@ mod tests {
fn prune_cannot_be_done() {
let db = TestStageDB::default();
let segment = Headers::new(PruneMode::Full);
let limiter = PruneLimiter::default().set_deleted_entries_limit(0);
let input = PruneInput {
@ -321,6 +323,7 @@ mod tests {
};
let provider = db.factory.provider_rw().unwrap();
let segment = super::Headers::new(db.factory.static_file_provider());
let result = segment.prune(&provider, input).unwrap();
assert_eq!(
result,

7
crates/prune/prune/src/segments/static_file/mod.rs Normal file
View File

@ -0,0 +1,7 @@
mod headers;
mod receipts;
mod transactions;
pub use headers::Headers;
pub use receipts::Receipts;
pub use transactions::Transactions;

53
crates/prune/prune/src/segments/static_file/receipts.rs Normal file
View File

@ -0,0 +1,53 @@
use crate::{
segments::{PruneInput, PruneOutput, Segment},
PrunerError,
};
use reth_db_api::database::Database;
use reth_provider::{
errors::provider::ProviderResult, providers::StaticFileProvider, DatabaseProviderRW,
};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use reth_static_file_types::StaticFileSegment;
#[derive(Debug)]
pub struct Receipts {
static_file_provider: StaticFileProvider,
}
impl Receipts {
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
Self { static_file_provider }
}
}
impl<DB: Database> Segment<DB> for Receipts {
fn segment(&self) -> PruneSegment {
PruneSegment::Receipts
}
fn mode(&self) -> Option<PruneMode> {
self.static_file_provider
.get_highest_static_file_block(StaticFileSegment::Receipts)
.map(PruneMode::before_inclusive)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::StaticFile
}
fn prune(
&self,
provider: &DatabaseProviderRW<DB>,
input: PruneInput,
) -> Result<PruneOutput, PrunerError> {
crate::segments::receipts::prune(provider, input)
}
fn save_checkpoint(
&self,
provider: &DatabaseProviderRW<DB>,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
crate::segments::receipts::save_checkpoint(provider, checkpoint)
}
}

34
crates/prune/prune/src/segments/transactions.rs → crates/prune/prune/src/segments/static_file/transactions.rs
View File

@ -4,18 +4,19 @@ use crate::{
};
use reth_db::tables;
use reth_db_api::database::Database;
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
use tracing::{instrument, trace};
use reth_provider::{providers::StaticFileProvider, DatabaseProviderRW, TransactionsProvider};
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
use reth_static_file_types::StaticFileSegment;
use tracing::trace;
#[derive(Debug)]
pub struct Transactions {
mode: PruneMode,
static_file_provider: StaticFileProvider,
}
impl Transactions {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
pub const fn new(static_file_provider: StaticFileProvider) -> Self {
Self { static_file_provider }
}
}
@ -25,10 +26,15 @@ impl<DB: Database> Segment<DB> for Transactions {
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
self.static_file_provider
.get_highest_static_file_block(StaticFileSegment::Transactions)
.map(PruneMode::before_inclusive)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::StaticFile
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
provider: &DatabaseProviderRW<DB>,
@ -75,7 +81,7 @@ impl<DB: Database> Segment<DB> for Transactions {
#[cfg(test)]
mod tests {
use crate::segments::{PruneInput, PruneOutput, Segment, Transactions};
use crate::segments::{PruneInput, PruneOutput, Segment};
use alloy_primitives::{BlockNumber, TxNumber, B256};
use assert_matches::assert_matches;
use itertools::{
@ -83,7 +89,7 @@ mod tests {
Itertools,
};
use reth_db::tables;
use reth_provider::PruneCheckpointReader;
use reth_provider::{PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory};
use reth_prune_types::{
PruneCheckpoint, PruneInterruptReason, PruneLimiter, PruneMode, PruneProgress, PruneSegment,
};
@ -104,8 +110,8 @@ mod tests {
assert_eq!(db.table::<tables::Transactions>().unwrap().len(), transactions.len());
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
let segment = super::Transactions::new(db.factory.static_file_provider());
let prune_mode = PruneMode::Before(to_block);
let segment = Transactions::new(prune_mode);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
let input = PruneInput {
previous_checkpoint: db
@ -138,9 +144,9 @@ mod tests {
if (progress, pruned) == expected_result
);
segment
.save_checkpoint(
&provider,
provider
.save_prune_checkpoint(
PruneSegment::Transactions,
result.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();

15
crates/prune/prune/src/segments/account_history.rs → crates/prune/prune/src/segments/user/account_history.rs
View File

@ -1,6 +1,7 @@
use crate::{
segments::{
history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint, Segment,
user::history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint,
Segment,
},
PrunerError,
};
@ -8,7 +9,9 @@ use itertools::Itertools;
use reth_db::tables;
use reth_db_api::{database::Database, models::ShardedKey};
use reth_provider::DatabaseProviderRW;
use reth_prune_types::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
use reth_prune_types::{
PruneInterruptReason, PruneMode, PruneProgress, PrunePurpose, PruneSegment,
};
use rustc_hash::FxHashMap;
use tracing::{instrument, trace};
@ -38,6 +41,10 @@ impl<DB: Database> Segment<DB> for AccountHistory {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
@ -124,8 +131,8 @@ impl<DB: Database> Segment<DB> for AccountHistory {
#[cfg(test)]
mod tests {
use crate::segments::{
account_history::ACCOUNT_HISTORY_TABLES_TO_PRUNE, AccountHistory, PruneInput, PruneOutput,
Segment,
user::account_history::ACCOUNT_HISTORY_TABLES_TO_PRUNE, AccountHistory, PruneInput,
PruneOutput, Segment,
};
use alloy_primitives::{BlockNumber, B256};
use assert_matches::assert_matches;

0
crates/prune/prune/src/segments/history.rs → crates/prune/prune/src/segments/user/history.rs
View File

14
crates/prune/prune/src/segments/user/mod.rs Normal file
View File

@ -0,0 +1,14 @@
mod account_history;
mod history;
mod receipts;
mod receipts_by_logs;
mod sender_recovery;
mod storage_history;
mod transaction_lookup;
pub use account_history::AccountHistory;
pub use receipts::Receipts;
pub use receipts_by_logs::ReceiptsByLogs;
pub use sender_recovery::SenderRecovery;
pub use storage_history::StorageHistory;
pub use transaction_lookup::TransactionLookup;

50
crates/prune/prune/src/segments/user/receipts.rs Normal file
View File

@ -0,0 +1,50 @@
use crate::{
segments::{PruneInput, PruneOutput, Segment},
PrunerError,
};
use reth_db_api::database::Database;
use reth_provider::{errors::provider::ProviderResult, DatabaseProviderRW};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use tracing::instrument;
#[derive(Debug)]
pub struct Receipts {
mode: PruneMode,
}
impl Receipts {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
}
}
impl<DB: Database> Segment<DB> for Receipts {
fn segment(&self) -> PruneSegment {
PruneSegment::Receipts
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
provider: &DatabaseProviderRW<DB>,
input: PruneInput,
) -> Result<PruneOutput, PrunerError> {
crate::segments::receipts::prune(provider, input)
}
fn save_checkpoint(
&self,
provider: &DatabaseProviderRW<DB>,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
crate::segments::receipts::save_checkpoint(provider, checkpoint)
}
}

6
crates/prune/prune/src/segments/receipts_by_logs.rs → crates/prune/prune/src/segments/user/receipts_by_logs.rs
View File

@ -31,6 +31,10 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
None
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
@ -215,7 +219,7 @@ impl<DB: Database> Segment<DB> for ReceiptsByLogs {
#[cfg(test)]
mod tests {
use crate::segments::{receipts_by_logs::ReceiptsByLogs, PruneInput, Segment};
use crate::segments::{PruneInput, ReceiptsByLogs, Segment};
use alloy_primitives::B256;
use assert_matches::assert_matches;
use reth_db::tables;

6
crates/prune/prune/src/segments/sender_recovery.rs → crates/prune/prune/src/segments/user/sender_recovery.rs
View File

@ -5,7 +5,7 @@ use crate::{
use reth_db::tables;
use reth_db_api::database::Database;
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
use tracing::{instrument, trace};
#[derive(Debug)]
@ -28,6 +28,10 @@ impl<DB: Database> Segment<DB> for SenderRecovery {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,

13
crates/prune/prune/src/segments/storage_history.rs → crates/prune/prune/src/segments/user/storage_history.rs
View File

@ -1,6 +1,7 @@
use crate::{
segments::{
history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint, Segment,
user::history::prune_history_indices, PruneInput, PruneOutput, PruneOutputCheckpoint,
Segment,
},
PrunerError,
};
@ -11,7 +12,9 @@ use reth_db_api::{
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress},
};
use reth_provider::DatabaseProviderRW;
use reth_prune_types::{PruneInterruptReason, PruneMode, PruneProgress, PruneSegment};
use reth_prune_types::{
PruneInterruptReason, PruneMode, PruneProgress, PrunePurpose, PruneSegment,
};
use rustc_hash::FxHashMap;
use tracing::{instrument, trace};
@ -41,6 +44,10 @@ impl<DB: Database> Segment<DB> for StorageHistory {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,
@ -132,7 +139,7 @@ impl<DB: Database> Segment<DB> for StorageHistory {
#[cfg(test)]
mod tests {
use crate::segments::{
storage_history::STORAGE_HISTORY_TABLES_TO_PRUNE, PruneInput, PruneOutput, Segment,
user::storage_history::STORAGE_HISTORY_TABLES_TO_PRUNE, PruneInput, PruneOutput, Segment,
StorageHistory,
};
use alloy_primitives::{BlockNumber, B256};

6
crates/prune/prune/src/segments/transaction_lookup.rs → crates/prune/prune/src/segments/user/transaction_lookup.rs
View File

@ -6,7 +6,7 @@ use rayon::prelude::*;
use reth_db::tables;
use reth_db_api::database::Database;
use reth_provider::{DatabaseProviderRW, TransactionsProvider};
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
use reth_prune_types::{PruneMode, PruneProgress, PrunePurpose, PruneSegment};
use tracing::{instrument, trace};
#[derive(Debug)]
@ -29,6 +29,10 @@ impl<DB: Database> Segment<DB> for TransactionLookup {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(
&self,