From e66e3e3556956864ea5a8d1da41a1e45c8f11e32 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 1 Sep 2023 13:51:34 +0100 Subject: [PATCH] feat(storage, tree): respect `Transaction Lookup` pruning in the blockchain tree (#4410) --- Cargo.lock | 1 + Cargo.toml | 1 + bin/reth/src/debug_cmd/in_memory_merkle.rs | 2 +- bin/reth/src/debug_cmd/merkle.rs | 2 +- bin/reth/src/node/mod.rs | 7 ++-- crates/blockchain-tree/Cargo.toml | 2 +- crates/blockchain-tree/src/blockchain_tree.rs | 15 +++++-- crates/consensus/beacon/Cargo.toml | 2 +- crates/consensus/beacon/src/engine/mod.rs | 6 ++- .../consensus/beacon/src/engine/test_utils.rs | 2 +- crates/consensus/common/Cargo.toml | 2 +- crates/net/downloaders/Cargo.toml | 2 +- crates/primitives/Cargo.toml | 2 +- crates/primitives/src/prune/mode.rs | 5 +++ crates/prune/Cargo.toml | 2 +- crates/rpc/rpc-engine-api/Cargo.toml | 2 +- crates/rpc/rpc/Cargo.toml | 2 +- crates/stages/Cargo.toml | 2 +- crates/stages/src/stages/execution.rs | 24 +++++------ crates/stages/src/stages/hashing_account.rs | 2 +- crates/stages/src/stages/mod.rs | 6 +-- crates/storage/db/Cargo.toml | 2 +- crates/storage/provider/Cargo.toml | 1 + .../provider/src/providers/database/mod.rs | 40 ++++++++++++++++++- .../src/providers/database/provider.rs | 21 +++++++--- crates/storage/provider/src/traits/block.rs | 6 ++- crates/transaction-pool/Cargo.toml | 2 +- examples/rpc-db.rs | 1 + testing/ef-tests/src/cases/blockchain_test.rs | 3 +- 29 files changed, 118 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f43893b5..5c8386b30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5817,6 +5817,7 @@ dependencies = [ name = "reth-provider" version = "0.1.0-alpha.8" dependencies = [ + "assert_matches", "auto_impl", "derive_more", "itertools 0.11.0", diff --git a/Cargo.toml b/Cargo.toml index e80531040..529715acc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,3 +156,4 @@ c-kzg = { git = "https://github.com/ethereum/c-kzg-4844" } ### misc-testing proptest = "1.0" arbitrary = "1.1" +assert_matches = "1.5.0" diff --git a/bin/reth/src/debug_cmd/in_memory_merkle.rs b/bin/reth/src/debug_cmd/in_memory_merkle.rs index 701b2196f..837c31c75 100644 --- a/bin/reth/src/debug_cmd/in_memory_merkle.rs +++ b/bin/reth/src/debug_cmd/in_memory_merkle.rs @@ -193,7 +193,7 @@ impl Command { let provider_rw = factory.provider_rw()?; // Insert block, state and hashes - provider_rw.insert_block(block.clone(), None)?; + provider_rw.insert_block(block.clone(), None, None)?; block_state.write_to_db(provider_rw.tx_ref(), block.number)?; let storage_lists = provider_rw.changed_storages_with_range(block.number..=block.number)?; let storages = provider_rw.plainstate_storages(storage_lists)?; diff --git a/bin/reth/src/debug_cmd/merkle.rs b/bin/reth/src/debug_cmd/merkle.rs index 47e37c369..add25f6cd 100644 --- a/bin/reth/src/debug_cmd/merkle.rs +++ b/bin/reth/src/debug_cmd/merkle.rs @@ -185,7 +185,7 @@ impl Command { continue } }; - provider_rw.insert_block(sealed_block.block, Some(sealed_block.senders))?; + provider_rw.insert_block(sealed_block.block, Some(sealed_block.senders), None)?; } // Check if any of hashing or merkle stages aren't on the same block number as diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index f06baaaf2..58d7bccb8 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -267,6 +267,9 @@ impl NodeCommand { let metrics_listener = MetricsListener::new(metrics_rx); ctx.task_executor.spawn_critical("metrics listener task", metrics_listener); + let prune_config = + self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); + // configure blockchain tree let tree_externals = TreeExternals::new( db.clone(), @@ -284,6 +287,7 @@ impl NodeCommand { tree_externals, canon_state_notification_sender.clone(), tree_config, + prune_config.clone().map(|config| config.parts), )? .with_sync_metrics_tx(metrics_tx.clone()), ); @@ -365,9 +369,6 @@ impl NodeCommand { None }; - let prune_config = - self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); - // Configure the pipeline let (mut pipeline, client) = if self.dev.dev { info!(target: "reth::cli", "Starting Reth in dev mode"); diff --git a/crates/blockchain-tree/Cargo.toml b/crates/blockchain-tree/Cargo.toml index 70e3e1803..81a8b81d2 100644 --- a/crates/blockchain-tree/Cargo.toml +++ b/crates/blockchain-tree/Cargo.toml @@ -40,7 +40,7 @@ reth-interfaces = { workspace = true, features = ["test-utils"] } reth-primitives = { workspace = true , features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } parking_lot.workspace = true -assert_matches = "1.5" +assert_matches.workspace = true tokio = { workspace = true, features = ["macros", "sync"] } [features] diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 26508b4f8..ac17cd2f4 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -16,7 +16,7 @@ use reth_interfaces::{ Error, }; use reth_primitives::{ - BlockHash, BlockNumHash, BlockNumber, ForkBlock, Hardfork, Receipt, SealedBlock, + BlockHash, BlockNumHash, BlockNumber, ForkBlock, Hardfork, PruneModes, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, U256, }; use reth_provider::{ @@ -93,6 +93,7 @@ pub struct BlockchainTree { metrics: TreeMetrics, /// Metrics for sync stages. sync_metrics_tx: Option, + prune_modes: Option, } /// A container that wraps chains and block indices to allow searching for block hashes across all @@ -110,6 +111,7 @@ impl BlockchainTree externals: TreeExternals, canon_state_notification_sender: CanonStateNotificationSender, config: BlockchainTreeConfig, + prune_modes: Option, ) -> Result { let max_reorg_depth = config.max_reorg_depth(); @@ -145,6 +147,7 @@ impl BlockchainTree canon_state_notification_sender, metrics: Default::default(), sync_metrics_tx: None, + prune_modes, }) } @@ -1048,7 +1051,11 @@ impl BlockchainTree let (blocks, state) = chain.into_inner(); provider - .append_blocks_with_post_state(blocks.into_blocks().collect(), state) + .append_blocks_with_post_state( + blocks.into_blocks().collect(), + state, + self.prune_modes.as_ref(), + ) .map_err(|e| BlockExecutionError::CanonicalCommit { inner: e.to_string() })?; provider.commit()?; @@ -1173,7 +1180,7 @@ mod tests { let factory = ProviderFactory::new(&db, MAINNET.clone()); let provider = factory.provider_rw().unwrap(); - provider.insert_block(genesis, None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); // insert first 10 blocks for i in 0..10 { @@ -1279,7 +1286,7 @@ mod tests { let config = BlockchainTreeConfig::new(1, 2, 3, 2); let (sender, mut canon_notif) = tokio::sync::broadcast::channel(10); let mut tree = - BlockchainTree::new(externals, sender, config).expect("failed to create tree"); + BlockchainTree::new(externals, sender, config, None).expect("failed to create tree"); // genesis block 10 is already canonical assert!(tree.make_canonical(&H256::zero()).is_ok()); diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index 09e9ecc5f..f44de6c55 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -46,4 +46,4 @@ reth-tracing = { path = "../../tracing" } reth-revm = { path = "../../revm" } reth-downloaders = { path = "../../net/downloaders" } -assert_matches = "1.5" +assert_matches.workspace = true diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 30ca834d0..118eb82f3 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1880,7 +1880,9 @@ mod tests { BeaconForkChoiceUpdateError, }; use assert_matches::assert_matches; - use reth_primitives::{stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, H256, MAINNET}; + use reth_primitives::{ + stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, PruneModes, H256, MAINNET, + }; use reth_provider::{BlockWriter, ProviderFactory}; use reth_rpc_types::engine::{ ExecutionPayloadV1, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, @@ -2056,7 +2058,7 @@ mod tests { let factory = ProviderFactory::new(db, chain); let provider = factory.provider_rw().unwrap(); blocks - .try_for_each(|b| provider.insert_block(b.clone(), None).map(|_| ())) + .try_for_each(|b| provider.insert_block(b.clone(), None, None).map(|_| ())) .expect("failed to insert"); provider.commit().unwrap(); } diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 0fdda91da..dc963c3bd 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -463,7 +463,7 @@ where let config = BlockchainTreeConfig::new(1, 2, 3, 2); let (canon_state_notification_sender, _) = tokio::sync::broadcast::channel(3); let tree = ShareableBlockchainTree::new( - BlockchainTree::new(externals, canon_state_notification_sender, config) + BlockchainTree::new(externals, canon_state_notification_sender, config, None) .expect("failed to create tree"), ); let shareable_db = ProviderFactory::new(db.clone(), self.base_config.chain_spec.clone()); diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index 8611aa231..480b3df36 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -16,5 +16,5 @@ reth-provider.workspace = true [dev-dependencies] reth-interfaces = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } -assert_matches = "1.5.0" +assert_matches.workspace = true mockall = "0.11.3" diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index ccc803c8d..3c33ab23d 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -42,7 +42,7 @@ reth-db = { path = "../../storage/db", features = ["test-utils"] } reth-interfaces = { workspace = true, features = ["test-utils"] } reth-tracing = { path = "../../tracing" } -assert_matches = "1.5.0" +assert_matches.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } reth-rlp.workspace = true itertools.workspace = true diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index da697b8b2..d450c419d 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -85,7 +85,7 @@ revm-primitives = { workspace = true, features = ["arbitrary"] } arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true proptest-derive = "0.3" -assert_matches = "1.5.0" +assert_matches.workspace = true toml = "0.7.4" # necessary so we don't hit a "undeclared 'std'": diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs index 3e39f876d..a8bd046a7 100644 --- a/crates/primitives/src/prune/mode.rs +++ b/crates/primitives/src/prune/mode.rs @@ -49,6 +49,11 @@ impl PruneMode { PruneMode::Before(n) => *n > block, } } + + /// Returns true if the prune mode is [`PruneMode::Full`]. + pub fn is_full(&self) -> bool { + matches!(self, Self::Full) + } } #[cfg(test)] diff --git a/crates/prune/Cargo.toml b/crates/prune/Cargo.toml index be5346d02..4582d27f8 100644 --- a/crates/prune/Cargo.toml +++ b/crates/prune/Cargo.toml @@ -34,4 +34,4 @@ reth-stages = { path = "../stages", features = ["test-utils"] } # misc -assert_matches = "1.5.0" +assert_matches.workspace = true diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 8570e610f..d08e7c8fa 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -34,4 +34,4 @@ reth-rlp.workspace = true reth-interfaces = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-payload-builder = { workspace = true, features = ["test-utils"] } -assert_matches = "1.5.0" +assert_matches.workspace = true diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 44760ecd9..33c2fdde6 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -70,6 +70,6 @@ futures.workspace = true [dev-dependencies] jsonrpsee = { workspace = true, features = ["client"] } -assert_matches = "1.5.0" +assert_matches.workspace = true tempfile = "3.5.0" reth-interfaces = { workspace = true, features = ["test-utils"] } diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 01c3b78d7..a1770fa76 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -61,7 +61,7 @@ reth-trie = { path = "../trie", features = ["test-utils"] } itertools.workspace = true tokio = { workspace = true, features = ["rt", "sync", "macros"] } -assert_matches = "1.5.0" +assert_matches.workspace = true rand.workspace = true paste = "1.0" diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 6fe07013b..cf688f352 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -526,8 +526,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); let previous_stage_checkpoint = ExecutionCheckpoint { @@ -562,8 +562,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); let previous_stage_checkpoint = ExecutionCheckpoint { @@ -598,8 +598,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); let previous_checkpoint = StageCheckpoint { block_number: 1, stage_checkpoint: None }; @@ -632,8 +632,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); // insert pre state @@ -742,8 +742,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); // variables @@ -820,8 +820,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); provider.commit().unwrap(); // variables diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index ccb6fb960..5887fb803 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -95,7 +95,7 @@ impl AccountHashingStage { let blocks = random_block_range(&mut rng, opts.blocks.clone(), H256::zero(), opts.txs); for block in blocks { - provider.insert_block(block, None).unwrap(); + provider.insert_block(block, None, None).unwrap(); } let mut accounts = random_eoa_account_range(&mut rng, opts.accounts); { diff --git a/crates/stages/src/stages/mod.rs b/crates/stages/src/stages/mod.rs index 87e045aee..fbe9ff00e 100644 --- a/crates/stages/src/stages/mod.rs +++ b/crates/stages/src/stages/mod.rs @@ -81,8 +81,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - provider.insert_block(genesis, None).unwrap(); - provider.insert_block(block.clone(), None).unwrap(); + provider.insert_block(genesis, None, None).unwrap(); + provider.insert_block(block.clone(), None, None).unwrap(); // Fill with bogus blocks to respect PruneMode distance. let mut head = block.hash; @@ -90,7 +90,7 @@ mod tests { for block_number in 2..=tip { let nblock = random_block(&mut rng, block_number, Some(head), Some(0), Some(0)); head = nblock.hash; - provider.insert_block(nblock, None).unwrap(); + provider.insert_block(nblock, None, None).unwrap(); } provider.commit().unwrap(); diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index f4bcd553a..ae4d67ca8 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -76,7 +76,7 @@ serde_json.workspace = true paste = "1.0" -assert_matches = "1.5.0" +assert_matches.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["git", "gitcl"] } diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 3bd4bc57d..bcff000a0 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -40,6 +40,7 @@ reth-rlp.workspace = true reth-trie = { path = "../../trie", features = ["test-utils"] } parking_lot.workspace = true tempfile = "3.3" +assert_matches.workspace = true [features] test-utils = ["reth-rlp"] diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index b45417909..45529c9f0 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -393,12 +393,16 @@ impl PruneCheckpointReader for ProviderFactory { #[cfg(test)] mod tests { use super::ProviderFactory; - use crate::{BlockHashReader, BlockNumReader}; + use crate::{BlockHashReader, BlockNumReader, BlockWriter, TransactionsProvider}; + use assert_matches::assert_matches; use reth_db::{ test_utils::{create_test_rw_db, ERROR_TEMPDIR}, DatabaseEnv, }; - use reth_primitives::{ChainSpecBuilder, H256}; + use reth_primitives::{ + hex_literal::hex, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, H256, + }; + use reth_rlp::Decodable; use std::sync::Arc; #[test] @@ -449,4 +453,36 @@ mod tests { provider_rw.block_hash(0).unwrap(); provider.block_hash(0).unwrap(); } + + #[test] + fn insert_block_with_prune_modes() { + let chain_spec = ChainSpecBuilder::mainnet().build(); + let db = create_test_rw_db(); + let factory = ProviderFactory::new(db, Arc::new(chain_spec)); + + let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); + let block = SealedBlock::decode(&mut block_rlp).unwrap(); + + { + let provider = factory.provider_rw().unwrap(); + assert_matches!(provider.insert_block(block.clone(), None, None), Ok(_)); + assert_matches!(provider.transaction_id(block.body[0].hash), Ok(Some(0))); + } + + { + let provider = factory.provider_rw().unwrap(); + assert_matches!( + provider.insert_block( + block.clone(), + None, + Some(&PruneModes { + transaction_lookup: Some(PruneMode::Full), + ..PruneModes::none() + }) + ), + Ok(_) + ); + assert_matches!(provider.transaction_id(block.body[0].hash), Ok(None)); + } + } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 85c497cc4..f63b256b3 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -31,10 +31,10 @@ use reth_primitives::{ stage::{StageCheckpoint, StageId}, trie::Nibbles, Account, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, - ChainInfo, ChainSpec, Hardfork, Head, Header, PruneCheckpoint, PrunePart, Receipt, SealedBlock, - SealedBlockWithSenders, SealedHeader, StorageEntry, TransactionMeta, TransactionSigned, - TransactionSignedEcRecovered, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, H256, - U256, + ChainInfo, ChainSpec, Hardfork, Head, Header, PruneCheckpoint, PruneModes, PrunePart, Receipt, + SealedBlock, SealedBlockWithSenders, SealedHeader, StorageEntry, TransactionMeta, + TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, TxHash, TxNumber, + Withdrawal, H256, U256, }; use reth_revm_primitives::{ config::revm_spec, @@ -1871,6 +1871,7 @@ impl<'this, TX: DbTxMut<'this> + DbTx<'this>> BlockWriter for DatabaseProvider<' &self, block: SealedBlock, senders: Option>, + prune_modes: Option<&PruneModes>, ) -> Result { let block_number = block.number; self.tx.put::(block.number, block.hash())?; @@ -1922,7 +1923,14 @@ impl<'this, TX: DbTxMut<'this> + DbTx<'this>> BlockWriter for DatabaseProvider<' let hash = transaction.hash(); self.tx.put::(next_tx_num, sender)?; self.tx.put::(next_tx_num, transaction.into())?; - self.tx.put::(hash, next_tx_num)?; + + if prune_modes + .and_then(|modes| modes.transaction_lookup) + .filter(|prune_mode| prune_mode.is_full()) + .is_none() + { + self.tx.put::(hash, next_tx_num)?; + } next_tx_num += 1; } @@ -1949,6 +1957,7 @@ impl<'this, TX: DbTxMut<'this> + DbTx<'this>> BlockWriter for DatabaseProvider<' &self, blocks: Vec, state: PostState, + prune_modes: Option<&PruneModes>, ) -> Result<()> { if blocks.is_empty() { return Ok(()) @@ -1966,7 +1975,7 @@ impl<'this, TX: DbTxMut<'this> + DbTx<'this>> BlockWriter for DatabaseProvider<' // Insert the blocks for block in blocks { let (block, senders) = block.into_components(); - self.insert_block(block, Some(senders))?; + self.insert_block(block, Some(senders), prune_modes)?; } // Write state and changesets to the database. diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 182d7c579..c78070161 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -7,7 +7,8 @@ use reth_db::models::StoredBlockBodyIndices; use reth_interfaces::Result; use reth_primitives::{ Address, Block, BlockHashOrNumber, BlockId, BlockNumber, BlockNumberOrTag, BlockWithSenders, - ChainSpec, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, H256, + ChainSpec, Header, PruneModes, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, + H256, }; use std::ops::RangeInclusive; @@ -242,6 +243,7 @@ pub trait BlockWriter: Send + Sync { &self, block: SealedBlock, senders: Option>, + prune_modes: Option<&PruneModes>, ) -> Result; /// Appends a batch of sealed blocks to the blockchain, including sender information, and @@ -254,6 +256,7 @@ pub trait BlockWriter: Send + Sync { /// /// - `blocks`: Vector of `SealedBlockWithSenders` instances to append. /// - `state`: Post-state information to update after appending. + /// - `prune_modes`: Optional pruning configuration. /// /// # Returns /// @@ -263,5 +266,6 @@ pub trait BlockWriter: Send + Sync { &self, blocks: Vec, state: PostState, + prune_modes: Option<&PruneModes>, ) -> Result<()>; } diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index a5ef62b83..0ac9379d2 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -55,7 +55,7 @@ paste = "1.0" rand = "0.8" proptest.workspace = true criterion = "0.5" -assert_matches = "1.5" +assert_matches.workspace = true [features] default = ["serde"] diff --git a/examples/rpc-db.rs b/examples/rpc-db.rs index 5ff672f82..86f890299 100644 --- a/examples/rpc-db.rs +++ b/examples/rpc-db.rs @@ -48,6 +48,7 @@ async fn main() -> eyre::Result<()> { externals, canon_state_notification_sender, tree_config, + None, )?); BlockchainProvider::new(factory, tree)? diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index d63124581..d240885e7 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -81,6 +81,7 @@ impl Case for BlockchainTestCase { provider.insert_block( SealedBlock::new(case.genesis_block_header.clone().into(), BlockBody::default()), None, + None, )?; case.pre.write_to_db(provider.tx_ref())?; @@ -88,7 +89,7 @@ impl Case for BlockchainTestCase { for block in case.blocks.iter() { let decoded = SealedBlock::decode(&mut block.rlp.as_ref())?; last_block = Some(decoded.number); - provider.insert_block(decoded, None)?; + provider.insert_block(decoded, None, None)?; } // Call execution stage