diff --git a/Cargo.lock b/Cargo.lock index 06036c4a1..ce0c97370 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,7 +513,7 @@ dependencies = [ "bitflags 2.3.2", "boa_interner", "boa_macros", - "indexmap", + "indexmap 1.9.3", "num-bigint", "rustc-hash", ] @@ -535,7 +535,7 @@ dependencies = [ "dashmap", "fast-float", "icu_normalizer", - "indexmap", + "indexmap 1.9.3", "itertools", "num-bigint", "num-integer", @@ -587,7 +587,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.0", - "indexmap", + "indexmap 1.9.3", "once_cell", "phf", "rustc-hash", @@ -860,7 +860,7 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", - "indexmap", + "indexmap 1.9.3", "textwrap", ] @@ -1874,6 +1874,12 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + [[package]] name = "errno" version = "0.2.8" @@ -2013,7 +2019,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.18", - "toml 0.7.3", + "toml 0.7.5", "walkdir", ] @@ -2549,7 +2555,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -3074,6 +3080,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inferno" version = "0.11.15" @@ -3081,7 +3097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fb7c1b80a1dfa604bb4a649a5c5aeef3d913f7c520cb42b40e534e8a61bcdfc" dependencies = [ "ahash 0.8.3", - "indexmap", + "indexmap 1.9.3", "is-terminal", "itoa", "log", @@ -3651,7 +3667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" dependencies = [ "hyper", - "indexmap", + "indexmap 1.9.3", "ipnet", "metrics", "metrics-util", @@ -3699,7 +3715,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.12.3", - "indexmap", + "indexmap 1.9.3", "metrics", "num_cpus", "ordered-float", @@ -4972,7 +4988,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "toml 0.7.3", + "toml 0.7.5", "tracing", "tui", "vergen", @@ -5081,6 +5097,8 @@ dependencies = [ "reth-downloaders", "reth-net-nat", "reth-network", + "reth-primitives", + "reth-stages", "secp256k1", "serde", "serde_json", @@ -5330,7 +5348,7 @@ dependencies = [ "byteorder", "criterion", "derive_more", - "indexmap", + "indexmap 1.9.3", "libc", "lifetimed-bytes", "parking_lot 0.12.1", @@ -5485,6 +5503,7 @@ name = "reth-primitives" version = "0.1.0-alpha.1" dependencies = [ "arbitrary", + "assert_matches", "bytes", "crc", "criterion", @@ -5519,6 +5538,7 @@ dependencies = [ "tiny-keccak", "tokio", "tokio-stream", + "toml 0.7.5", "tracing", "triehash", "url", @@ -5814,6 +5834,8 @@ dependencies = [ "reth-revm", "reth-rlp", "reth-trie", + "serde", + "serde_json", "thiserror", "tokio", "tokio-stream", @@ -6389,9 +6411,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -6417,7 +6439,7 @@ dependencies = [ "base64 0.13.1", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "serde_with_macros", @@ -7190,9 +7212,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" dependencies = [ "serde", "serde_spanned", @@ -7202,20 +7224,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -7244,7 +7266,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand 0.8.5", @@ -8069,9 +8091,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" dependencies = [ "memchr", ] diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index ac8084ae1..c278440cf 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -10,17 +10,19 @@ repository.workspace = true [dependencies] # reth reth-network = { path = "../net/network" } -reth-net-nat = { path = "../../crates/net/nat" } -reth-discv4 = { path = "../../crates/net/discv4" } -reth-downloaders = { path = "../../crates/net/downloaders" } +reth-net-nat = { path = "../net/nat" } +reth-discv4 = { path = "../net/discv4" } +reth-downloaders = { path = "../net/downloaders" } +reth-stages = { path = "../../crates/stages" } +reth-primitives = { path = "../primitives" } # io serde = { workspace = true } serde_json = { workspace = true } -#crypto +# crypto secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] } confy = "0.5" -tempfile = "3.4" +tempfile = "3.4" \ No newline at end of file diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 95d41e022..4593dedd7 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -5,6 +5,7 @@ use reth_downloaders::{ headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; use reth_network::{NetworkConfigBuilder, PeersConfig, SessionsConfig}; +use reth_primitives::PruneMode; use secp256k1::SecretKey; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -16,6 +17,9 @@ pub struct Config { /// Configuration for each stage in the pipeline. // TODO(onbjerg): Can we make this easier to maintain when we add/remove stages? pub stages: StageConfig, + /// Configuration for pruning. + #[serde(skip_serializing_if = "Option::is_none")] + pub prune: Option, /// Configuration for the discovery service. pub peers: PeersConfig, /// Configuration for peer sessions. @@ -276,6 +280,43 @@ impl Default for IndexHistoryConfig { } } +/// Pruning configuration. +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub struct PruneConfig { + /// Minimum pruning interval measured in blocks. + pub block_interval: u64, + /// Pruning configuration for every part of the data that can be pruned. + pub parts: PruneParts, +} + +impl Default for PruneConfig { + fn default() -> Self { + Self { block_interval: 10, parts: PruneParts::default() } + } +} + +/// Pruning configuration for every part of the data that can be pruned. +#[derive(Debug, Clone, Default, Copy, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub struct PruneParts { + /// Sender Recovery pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub sender_recovery: Option, + /// Transaction Lookup pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub transaction_lookup: Option, + /// Receipts pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub receipts: Option, + /// Account History pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub account_history: Option, + /// Storage History pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub storage_history: Option, +} + #[cfg(test)] mod tests { use super::Config; diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index a34e21094..12822404d 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -13,7 +13,6 @@ description = "Commonly used types in reth." reth-rlp = { workspace = true, features = ["std", "derive", "ethereum-types"] } reth-rlp-derive = { path = "../rlp/rlp-derive" } reth-codecs = { path = "../storage/codecs" } - revm-primitives = { workspace = true, features = ["serde"] } # ethereum @@ -79,6 +78,8 @@ revm-primitives = { workspace = true, features = ["arbitrary"] } arbitrary = { version = "1.1.7", features = ["derive"] } proptest = { version = "1.0" } proptest-derive = "0.3" +assert_matches = "1.5.0" +toml = "0.7.4" # necessary so we don't hit a "undeclared 'std'": # https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 73845c2fd..ada92bd09 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -39,6 +39,7 @@ pub mod listener; mod log; mod net; mod peer; +mod prune; mod receipt; pub mod stage; mod storage; @@ -76,6 +77,7 @@ pub use net::{ SEPOLIA_BOOTNODES, }; pub use peer::{PeerId, WithPeerId}; +pub use prune::{PruneCheckpoint, PruneMode}; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; diff --git a/crates/primitives/src/prune/checkpoint.rs b/crates/primitives/src/prune/checkpoint.rs new file mode 100644 index 000000000..a0c445fdf --- /dev/null +++ b/crates/primitives/src/prune/checkpoint.rs @@ -0,0 +1,13 @@ +use crate::{prune::PruneMode, BlockNumber}; +use reth_codecs::{main_codec, Compact}; + +/// Saves the pruning progress of a stage. +#[main_codec] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[cfg_attr(test, derive(Default))] +pub struct PruneCheckpoint { + /// Highest pruned block number. + block_number: BlockNumber, + /// Prune mode. + prune_mode: PruneMode, +} diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs new file mode 100644 index 000000000..2814f8bc4 --- /dev/null +++ b/crates/primitives/src/prune/mod.rs @@ -0,0 +1,5 @@ +mod checkpoint; +mod mode; + +pub use checkpoint::PruneCheckpoint; +pub use mode::PruneMode; diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs new file mode 100644 index 000000000..47e6778e8 --- /dev/null +++ b/crates/primitives/src/prune/mode.rs @@ -0,0 +1,56 @@ +use crate::BlockNumber; +use reth_codecs::{main_codec, Compact}; + +/// Prune mode. +#[main_codec] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[serde(rename_all = "lowercase")] +pub enum PruneMode { + /// Prune all blocks. + Full, + /// Prune blocks before the `head-N` block number. In other words, keep last N blocks. + Distance(u64), + /// Prune blocks before the specified block number. The specified block number is not pruned. + Before(BlockNumber), +} + +#[cfg(test)] +impl Default for PruneMode { + fn default() -> Self { + Self::Distance(0) + } +} + +#[cfg(test)] +mod tests { + use crate::prune::PruneMode; + use assert_matches::assert_matches; + use serde::Deserialize; + + #[test] + fn prune_mode_deserialize() { + #[derive(Debug, Deserialize)] + struct Config { + a: Option, + b: Option, + c: Option, + d: Option, + } + + let toml_str = r#" + a = "full" + b = { distance = 10 } + c = { before = 20 } + "#; + + assert_matches!( + toml::from_str(toml_str), + Ok(Config { + a: Some(PruneMode::Full), + b: Some(PruneMode::Distance(10)), + c: Some(PruneMode::Before(20)), + d: None + }) + ); + } +} diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index d60bc3dc2..bfe2b6288 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -34,6 +34,9 @@ pin-project = { workspace = true } # observability tracing = { workspace = true } +# io +serde = { workspace = true } + # misc thiserror = { workspace = true } aquamarine = "0.3.0" @@ -63,6 +66,9 @@ paste = "1.0" pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } criterion = { version = "0.4.0", features = ["async_futures"] } +# io +serde_json = { workspace = true } + [features] test-utils = [] diff --git a/crates/stages/src/stage.rs b/crates/stages/src/stage.rs index 7580c6bab..71c5f1ad8 100644 --- a/crates/stages/src/stage.rs +++ b/crates/stages/src/stage.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx}; use reth_primitives::{ stage::{StageCheckpoint, StageId}, - BlockNumber, TxNumber, + BlockNumber, PruneMode, TxNumber, }; use reth_provider::{BlockReader, DatabaseProviderRW, ProviderError}; use std::{ @@ -194,3 +194,31 @@ pub trait Stage: Send + Sync { input: UnwindInput, ) -> Result; } + +/// Prune target. +#[derive(Debug, Clone, Copy)] +pub enum PruneTarget { + /// Prune all blocks, i.e. not save any data. + All, + /// Prune blocks up to the specified block number, inclusive. + Block(BlockNumber), +} + +impl PruneTarget { + /// Returns new target to prune towards, according to stage prune mode [PruneMode] + /// and current head [BlockNumber]. + pub fn new(prune_mode: PruneMode, head: BlockNumber) -> Self { + match prune_mode { + PruneMode::Full => PruneTarget::All, + PruneMode::Distance(distance) => { + Self::Block(head.saturating_sub(distance).saturating_sub(1)) + } + PruneMode::Before(before_block) => Self::Block(before_block.saturating_sub(1)), + } + } + + /// Returns true if the target is [PruneTarget::All], i.e. prune all blocks. + pub fn is_all(&self) -> bool { + matches!(self, Self::All) + } +} diff --git a/crates/storage/db/src/tables/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs index 1f12b7ddc..882fe6b31 100644 --- a/crates/storage/db/src/tables/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -48,7 +48,8 @@ impl_compression_for_compact!( AccountBeforeTx, TransactionSignedNoHash, CompactU256, - StageCheckpoint + StageCheckpoint, + PruneCheckpoint ); macro_rules! impl_compression_fixed_compact { diff --git a/crates/storage/db/src/tables/models/mod.rs b/crates/storage/db/src/tables/models/mod.rs index ddd2c6b1c..da746efda 100644 --- a/crates/storage/db/src/tables/models/mod.rs +++ b/crates/storage/db/src/tables/models/mod.rs @@ -73,6 +73,7 @@ impl Decode for Address { Ok(Address::from_slice(value.as_ref())) } } + impl Encode for H256 { type Encoded = [u8; 32]; fn encode(self) -> Self::Encoded {