feat(stages, storage): pruning configuration (#3341)

Co-authored-by: Roman Krasiuk <rokrassyuk@gmail.com>

Cargo.lock

@@ -513,7 +513,7 @@ dependencies = [
  "bitflags 2.3.2",
  "boa_interner",
  "boa_macros",
- "indexmap",
+ "indexmap 1.9.3",
  "num-bigint",
  "rustc-hash",
 ]
@@ -535,7 +535,7 @@ dependencies = [
  "dashmap",
  "fast-float",
  "icu_normalizer",
- "indexmap",
+ "indexmap 1.9.3",
  "itertools",
  "num-bigint",
  "num-integer",
@@ -587,7 +587,7 @@ dependencies = [
  "boa_gc",
  "boa_macros",
  "hashbrown 0.14.0",
- "indexmap",
+ "indexmap 1.9.3",
  "once_cell",
  "phf",
  "rustc-hash",
@@ -860,7 +860,7 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
 dependencies = [
  "bitflags 1.3.2",
  "clap_lex 0.2.4",
- "indexmap",
+ "indexmap 1.9.3",
  "textwrap",
 ]
 
@@ -1874,6 +1874,12 @@ dependencies = [
  "syn 2.0.18",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -2013,7 +2019,7 @@ dependencies = [
  "serde",
  "serde_json",
  "syn 2.0.18",
- "toml 0.7.3",
+ "toml 0.7.5",
  "walkdir",
 ]
 
@@ -2549,7 +2555,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 1.9.3",
  "slab",
  "tokio",
  "tokio-util",
@@ -3074,6 +3080,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
 [[package]]
 name = "inferno"
 version = "0.11.15"
@@ -3081,7 +3097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2fb7c1b80a1dfa604bb4a649a5c5aeef3d913f7c520cb42b40e534e8a61bcdfc"
 dependencies = [
  "ahash 0.8.3",
- "indexmap",
+ "indexmap 1.9.3",
  "is-terminal",
  "itoa",
  "log",
@@ -3651,7 +3667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70"
 dependencies = [
  "hyper",
- "indexmap",
+ "indexmap 1.9.3",
  "ipnet",
  "metrics",
  "metrics-util",
@@ -3699,7 +3715,7 @@ dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
  "hashbrown 0.12.3",
- "indexmap",
+ "indexmap 1.9.3",
  "metrics",
  "num_cpus",
  "ordered-float",
@@ -4972,7 +4988,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
- "toml 0.7.3",
+ "toml 0.7.5",
  "tracing",
  "tui",
  "vergen",
@@ -5081,6 +5097,8 @@ dependencies = [
  "reth-downloaders",
  "reth-net-nat",
  "reth-network",
+ "reth-primitives",
+ "reth-stages",
  "secp256k1",
  "serde",
  "serde_json",
@@ -5330,7 +5348,7 @@ dependencies = [
  "byteorder",
  "criterion",
  "derive_more",
- "indexmap",
+ "indexmap 1.9.3",
  "libc",
  "lifetimed-bytes",
  "parking_lot 0.12.1",
@@ -5485,6 +5503,7 @@ name = "reth-primitives"
 version = "0.1.0-alpha.1"
 dependencies = [
  "arbitrary",
+ "assert_matches",
  "bytes",
  "crc",
  "criterion",
@@ -5519,6 +5538,7 @@ dependencies = [
  "tiny-keccak",
  "tokio",
  "tokio-stream",
+ "toml 0.7.5",
  "tracing",
  "triehash",
  "url",
@@ -5814,6 +5834,8 @@ dependencies = [
  "reth-revm",
  "reth-rlp",
  "reth-trie",
+ "serde",
+ "serde_json",
  "thiserror",
  "tokio",
  "tokio-stream",
@@ -6389,9 +6411,9 @@ dependencies = [
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.1"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4"
+checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
 dependencies = [
  "serde",
 ]
@@ -6417,7 +6439,7 @@ dependencies = [
  "base64 0.13.1",
  "chrono",
  "hex",
- "indexmap",
+ "indexmap 1.9.3",
  "serde",
  "serde_json",
  "serde_with_macros",
@@ -7190,9 +7212,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.3"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21"
+checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -7202,20 +7224,20 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.1"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"
+checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.19.8"
+version = "0.19.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13"
+checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7"
 dependencies = [
- "indexmap",
+ "indexmap 2.0.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -7244,7 +7266,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "hdrhistogram",
- "indexmap",
+ "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
  "rand 0.8.5",
@@ -8069,9 +8091,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
 [[package]]
 name = "winnow"
-version = "0.4.1"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28"
+checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448"
 dependencies = [
  "memchr",
 ]

crates/config/Cargo.toml

@@ -10,17 +10,19 @@ repository.workspace = true
 [dependencies]
 # reth
 reth-network = { path = "../net/network" }
-reth-net-nat = { path = "../../crates/net/nat" }
+reth-net-nat = { path = "../net/nat" }
-reth-discv4 = { path = "../../crates/net/discv4" }
+reth-discv4 = { path = "../net/discv4" }
-reth-downloaders = { path = "../../crates/net/downloaders" }
+reth-downloaders = { path = "../net/downloaders" }
+reth-stages = { path = "../../crates/stages" }
+reth-primitives = { path = "../primitives" }
 
 # io
 serde = { workspace = true }
 serde_json = { workspace = true }
 
-#crypto 
+# crypto
 secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] }
 
 confy = "0.5"
 
-tempfile = "3.4"
+tempfile = "3.4"

crates/config/src/config.rs

@@ -5,6 +5,7 @@ use reth_downloaders::{
     headers::reverse_headers::ReverseHeadersDownloaderBuilder,
 };
 use reth_network::{NetworkConfigBuilder, PeersConfig, SessionsConfig};
+use reth_primitives::PruneMode;
 use secp256k1::SecretKey;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
@@ -16,6 +17,9 @@ pub struct Config {
     /// Configuration for each stage in the pipeline.
     // TODO(onbjerg): Can we make this easier to maintain when we add/remove stages?
     pub stages: StageConfig,
+    /// Configuration for pruning.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub prune: Option<PruneConfig>,
     /// Configuration for the discovery service.
     pub peers: PeersConfig,
     /// Configuration for peer sessions.
@@ -276,6 +280,43 @@ impl Default for IndexHistoryConfig {
     }
 }
 
+/// Pruning configuration.
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
+#[serde(default)]
+pub struct PruneConfig {
+    /// Minimum pruning interval measured in blocks.
+    pub block_interval: u64,
+    /// Pruning configuration for every part of the data that can be pruned.
+    pub parts: PruneParts,
+}
+
+impl Default for PruneConfig {
+    fn default() -> Self {
+        Self { block_interval: 10, parts: PruneParts::default() }
+    }
+}
+
+/// Pruning configuration for every part of the data that can be pruned.
+#[derive(Debug, Clone, Default, Copy, Deserialize, PartialEq, Serialize)]
+#[serde(default)]
+pub struct PruneParts {
+    /// Sender Recovery pruning configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sender_recovery: Option<PruneMode>,
+    /// Transaction Lookup pruning configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub transaction_lookup: Option<PruneMode>,
+    /// Receipts pruning configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub receipts: Option<PruneMode>,
+    /// Account History pruning configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub account_history: Option<PruneMode>,
+    /// Storage History pruning configuration.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub storage_history: Option<PruneMode>,
+}
+
 #[cfg(test)]
 mod tests {
     use super::Config;

crates/primitives/Cargo.toml

@@ -13,7 +13,6 @@ description = "Commonly used types in reth."
 reth-rlp = { workspace = true, features = ["std", "derive", "ethereum-types"] }
 reth-rlp-derive = { path = "../rlp/rlp-derive" }
 reth-codecs = { path = "../storage/codecs" }
-
 revm-primitives = { workspace = true, features = ["serde"] }
 
 # ethereum
@@ -79,6 +78,8 @@ revm-primitives = { workspace = true, features = ["arbitrary"] }
 arbitrary = { version = "1.1.7", features = ["derive"] }
 proptest = { version = "1.0" }
 proptest-derive = "0.3"
+assert_matches = "1.5.0"
+toml = "0.7.4"
 
 # necessary so we don't hit a "undeclared 'std'": 
 # https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 

crates/primitives/src/lib.rs

@@ -39,6 +39,7 @@ pub mod listener;
 mod log;
 mod net;
 mod peer;
+mod prune;
 mod receipt;
 pub mod stage;
 mod storage;
@@ -76,6 +77,7 @@ pub use net::{
     SEPOLIA_BOOTNODES,
 };
 pub use peer::{PeerId, WithPeerId};
+pub use prune::{PruneCheckpoint, PruneMode};
 pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef};
 pub use revm_primitives::JumpMap;
 pub use serde_helper::JsonU256;

crates/primitives/src/prune/checkpoint.rs

@@ -0,0 +1,13 @@
+use crate::{prune::PruneMode, BlockNumber};
+use reth_codecs::{main_codec, Compact};
+
+/// Saves the pruning progress of a stage.
+#[main_codec]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[cfg_attr(test, derive(Default))]
+pub struct PruneCheckpoint {
+    /// Highest pruned block number.
+    block_number: BlockNumber,
+    /// Prune mode.
+    prune_mode: PruneMode,
+}

crates/primitives/src/prune/mod.rs

@@ -0,0 +1,5 @@
+mod checkpoint;
+mod mode;
+
+pub use checkpoint::PruneCheckpoint;
+pub use mode::PruneMode;

crates/primitives/src/prune/mode.rs

@@ -0,0 +1,56 @@
+use crate::BlockNumber;
+use reth_codecs::{main_codec, Compact};
+
+/// Prune mode.
+#[main_codec]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[serde(rename_all = "lowercase")]
+pub enum PruneMode {
+    /// Prune all blocks.
+    Full,
+    /// Prune blocks before the `head-N` block number. In other words, keep last N blocks.
+    Distance(u64),
+    /// Prune blocks before the specified block number. The specified block number is not pruned.
+    Before(BlockNumber),
+}
+
+#[cfg(test)]
+impl Default for PruneMode {
+    fn default() -> Self {
+        Self::Distance(0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::prune::PruneMode;
+    use assert_matches::assert_matches;
+    use serde::Deserialize;
+
+    #[test]
+    fn prune_mode_deserialize() {
+        #[derive(Debug, Deserialize)]
+        struct Config {
+            a: Option<PruneMode>,
+            b: Option<PruneMode>,
+            c: Option<PruneMode>,
+            d: Option<PruneMode>,
+        }
+
+        let toml_str = r#"
+        a = "full"
+        b = { distance = 10 }
+        c = { before = 20 }
+    "#;
+
+        assert_matches!(
+            toml::from_str(toml_str),
+            Ok(Config {
+                a: Some(PruneMode::Full),
+                b: Some(PruneMode::Distance(10)),
+                c: Some(PruneMode::Before(20)),
+                d: None
+            })
+        );
+    }
+}

crates/stages/Cargo.toml

@@ -34,6 +34,9 @@ pin-project = { workspace = true }
 # observability
 tracing = { workspace = true }
 
+# io
+serde = { workspace = true }
+
 # misc
 thiserror = { workspace = true }
 aquamarine = "0.3.0"
@@ -63,6 +66,9 @@ paste = "1.0"
 pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] }
 criterion = { version = "0.4.0", features = ["async_futures"] }
 
+# io
+serde_json = { workspace = true }
+
 [features]
 test-utils = []
 

crates/stages/src/stage.rs

@@ -3,7 +3,7 @@ use async_trait::async_trait;
 use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx};
 use reth_primitives::{
     stage::{StageCheckpoint, StageId},
-    BlockNumber, TxNumber,
+    BlockNumber, PruneMode, TxNumber,
 };
 use reth_provider::{BlockReader, DatabaseProviderRW, ProviderError};
 use std::{
@@ -194,3 +194,31 @@ pub trait Stage<DB: Database>: Send + Sync {
         input: UnwindInput,
     ) -> Result<UnwindOutput, StageError>;
 }
+
+/// Prune target.
+#[derive(Debug, Clone, Copy)]
+pub enum PruneTarget {
+    /// Prune all blocks, i.e. not save any data.
+    All,
+    /// Prune blocks up to the specified block number, inclusive.
+    Block(BlockNumber),
+}
+
+impl PruneTarget {
+    /// Returns new target to prune towards, according to stage prune mode [PruneMode]
+    /// and current head [BlockNumber].
+    pub fn new(prune_mode: PruneMode, head: BlockNumber) -> Self {
+        match prune_mode {
+            PruneMode::Full => PruneTarget::All,
+            PruneMode::Distance(distance) => {
+                Self::Block(head.saturating_sub(distance).saturating_sub(1))
+            }
+            PruneMode::Before(before_block) => Self::Block(before_block.saturating_sub(1)),
+        }
+    }
+
+    /// Returns true if the target is [PruneTarget::All], i.e. prune all blocks.
+    pub fn is_all(&self) -> bool {
+        matches!(self, Self::All)
+    }
+}

crates/storage/db/src/tables/codecs/compact.rs

@@ -48,7 +48,8 @@ impl_compression_for_compact!(
     AccountBeforeTx,
     TransactionSignedNoHash,
     CompactU256,
-    StageCheckpoint
+    StageCheckpoint,
+    PruneCheckpoint
 );
 
 macro_rules! impl_compression_fixed_compact {

crates/storage/db/src/tables/models/mod.rs

@@ -73,6 +73,7 @@ impl Decode for Address {
         Ok(Address::from_slice(value.as_ref()))
     }
 }
+
 impl Encode for H256 {
     type Encoded = [u8; 32];
     fn encode(self) -> Self::Encoded {