jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(trie): move trie updates to reth-trie-common (#12863)

Browse Source
This commit is contained in:
Roman Krasiuk
2024-11-26 13:09:12 +01:00
committed by GitHub
parent 26fc701814
commit 2dc92880ba
17 changed files with 113 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -7573,6 +7573,7 @@ dependencies = [
"reth-primitives",
"reth-primitives-traits",
"reth-trie",
"reth-trie-common",
"revm",
"serde",
"serde_with",
@ -9349,7 +9350,6 @@ dependencies = [
"alloy-rlp",
"alloy-trie",
"auto_impl",
"bincode",
"criterion",
"itertools 0.13.0",
"metrics",
@ -9363,9 +9363,7 @@ dependencies = [
"reth-storage-errors",
"reth-trie-common",
"revm",
"serde",
"serde_json",
"serde_with",
"tracing",
"triehash",
]
@ -9380,6 +9378,7 @@ dependencies = [
"alloy-rlp",
"alloy-trie",
"arbitrary",
"bincode",
"bytes",
"criterion",
"derive_more 1.0.0",
@ -9394,6 +9393,7 @@ dependencies = [
"revm-primitives",
"serde",
"serde_json",
"serde_with",
]
[[package]]

2
crates/engine/invalid-block-hooks/Cargo.toml
View File

@ -20,7 +20,7 @@ reth-provider.workspace = true
reth-revm = { workspace = true, features = ["serde"] }
reth-rpc-api = { workspace = true, features = ["client"] }
reth-tracing.workspace = true
reth-trie = { workspace = true, features = ["serde"] }
reth-trie.workspace = true
# alloy
alloy-primitives.workspace = true

12
crates/evm/execution-types/Cargo.toml
View File

@ -12,9 +12,10 @@ workspace = true
[dependencies]
reth-primitives.workspace = true
reth-execution-errors.workspace = true
reth-trie.workspace = true
reth-primitives-traits.workspace = true
reth-execution-errors.workspace = true
reth-trie-common.workspace = true
reth-trie.workspace = true
revm.workspace = true
@ -36,17 +37,18 @@ default = ["std"]
optimism = ["reth-primitives/optimism", "revm/optimism"]
serde = [
"dep:serde",
"reth-trie/serde",
"rand/serde",
"revm/serde",
"alloy-eips/serde",
"alloy-primitives/serde",
"rand/serde",
"reth-primitives-traits/serde",
"reth-trie-common/serde",
"reth-trie/serde",
]
serde-bincode-compat = [
"reth-primitives/serde-bincode-compat",
"reth-primitives-traits/serde-bincode-compat",
"reth-trie/serde-bincode-compat",
"reth-trie-common/serde-bincode-compat",
"serde_with",
"alloy-eips/serde-bincode-compat",
]

10
crates/evm/execution-types/src/chain.rs
View File

@ -11,7 +11,7 @@ use reth_primitives::{
TransactionSignedEcRecovered,
};
use reth_primitives_traits::NodePrimitives;
use reth_trie::updates::TrieUpdates;
use reth_trie_common::updates::TrieUpdates;
use revm::db::BundleState;
/// A chain of blocks and their final state.
@ -513,16 +513,14 @@ pub enum ChainSplit<N: NodePrimitives = reth_primitives::EthPrimitives> {
/// Bincode-compatible [`Chain`] serde implementation.
#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
pub(super) mod serde_bincode_compat {
use std::collections::BTreeMap;
use crate::ExecutionOutcome;
use alloc::borrow::Cow;
use alloy_primitives::BlockNumber;
use reth_primitives::serde_bincode_compat::SealedBlockWithSenders;
use reth_trie::serde_bincode_compat::updates::TrieUpdates;
use reth_trie_common::serde_bincode_compat::updates::TrieUpdates;
use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
use serde_with::{DeserializeAs, SerializeAs};
use crate::ExecutionOutcome;
use std::collections::BTreeMap;
/// Bincode-compatible [`super::Chain`] serde implementation.
///

5
crates/primitives/Cargo.toml
View File

@ -158,11 +158,12 @@ test-utils = [
"arbitrary",
]
serde-bincode-compat = [
"serde_with",
"alloy-eips/serde-bincode-compat",
"alloy-consensus/serde-bincode-compat",
"op-alloy-consensus?/serde-bincode-compat",
"reth-primitives-traits/serde-bincode-compat",
"serde_with",
"alloy-eips/serde-bincode-compat",
"reth-trie-common/serde-bincode-compat",
]
[[bench]]

2
crates/revm/Cargo.toml
View File

@ -55,9 +55,9 @@ test-utils = [
]
serde = [
"revm/serde",
"reth-trie?/serde",
"alloy-eips/serde",
"alloy-primitives/serde",
"alloy-consensus/serde",
"reth-primitives-traits/serde",
"reth-trie?/serde",
]

14
crates/storage/provider/Cargo.toml
View File

@ -98,21 +98,21 @@ optimism = [
"revm/optimism",
]
serde = [
"reth-execution-types/serde",
"reth-trie-db/serde",
"reth-trie/serde",
"alloy-consensus/serde",
"alloy-eips/serde",
"alloy-primitives/serde",
"alloy-rpc-types-engine/serde",
"dashmap/serde",
"notify/serde",
"parking_lot/serde",
"rand/serde",
"alloy-primitives/serde",
"alloy-consensus/serde",
"alloy-eips/serde",
"alloy-rpc-types-engine/serde",
"revm/serde",
"reth-codecs/serde",
"reth-optimism-primitives?/serde",
"reth-primitives-traits/serde",
"reth-execution-types/serde",
"reth-trie-db/serde",
"reth-trie/serde",
]
test-utils = [
"reth-db/test-utils",

9
crates/trie/common/Cargo.toml
View File

@ -31,6 +31,9 @@ nybbles = { workspace = true, features = ["rlp"] }
# `serde` feature
serde = { workspace = true, optional = true }
# `serde-bincode-compat` feature
serde_with = { workspace = true, optional = true }
# `test-utils` feature
hash-db = { version = "=0.15.2", optional = true }
plain_hasher = { version = "0.2", optional = true }
@ -46,6 +49,7 @@ arbitrary = { workspace = true, features = ["derive"] }
proptest.workspace = true
proptest-arbitrary-interop.workspace = true
criterion.workspace = true
bincode.workspace = true
[features]
serde = [
@ -59,6 +63,11 @@ serde = [
"reth-primitives-traits/serde",
"reth-codecs/serde"
]
serde-bincode-compat = [
"serde_with",
"reth-primitives-traits/serde-bincode-compat",
"alloy-consensus/serde-bincode-compat"
]
test-utils = [
"dep:plain_hasher",
"dep:hash-db",

14
crates/trie/common/src/lib.rs
View File

@ -37,5 +37,19 @@ pub use proofs::*;
pub mod root;
/// Buffer for trie updates.
pub mod updates;
/// Bincode-compatible serde implementations for trie types.
///
/// `bincode` crate allows for more efficient serialization of trie types, because it allows
/// non-string map keys.
///
/// Read more: <https://github.com/paradigmxyz/reth/issues/11370>
#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
pub mod serde_bincode_compat {
pub use super::updates::serde_bincode_compat as updates;
}
/// Re-export
pub use alloy_trie::{nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, EMPTY_ROOT_HASH};

78
crates/trie/trie/src/updates.rs → crates/trie/common/src/updates.rs
View File

@ -1,4 +1,4 @@
use crate::{walker::TrieWalker, BranchNodeCompact, HashBuilder, Nibbles};
use crate::{BranchNodeCompact, HashBuilder, Nibbles};
use alloy_primitives::B256;
use std::collections::{HashMap, HashSet};
@ -78,20 +78,19 @@ impl TrieUpdates {
}
/// Finalize state trie updates.
pub fn finalize<C>(
pub fn finalize(
&mut self,
walker: TrieWalker<C>,
hash_builder: HashBuilder,
removed_keys: HashSet<Nibbles>,
destroyed_accounts: HashSet<B256>,
) {
// Retrieve deleted keys from trie walker.
let (_, removed_node_keys) = walker.split();
self.removed_nodes.extend(exclude_empty(removed_node_keys));
// Retrieve updated nodes from hash builder.
let (_, updated_nodes) = hash_builder.split();
self.account_nodes.extend(exclude_empty_from_pair(updated_nodes));
// Add deleted node paths.
self.removed_nodes.extend(exclude_empty(removed_keys));
// Add deleted storage tries for destroyed accounts.
for destroyed in destroyed_accounts {
self.storage_tries.entry(destroyed).or_default().set_deleted(true);
@ -201,14 +200,13 @@ impl StorageTrieUpdates {
}
/// Finalize storage trie updates for by taking updates from walker and hash builder.
pub fn finalize<C>(&mut self, walker: TrieWalker<C>, hash_builder: HashBuilder) {
// Retrieve deleted keys from trie walker.
let (_, removed_keys) = walker.split();
self.removed_nodes.extend(exclude_empty(removed_keys));
pub fn finalize(&mut self, hash_builder: HashBuilder, removed_keys: HashSet<Nibbles>) {
// Retrieve updated nodes from hash builder.
let (_, updated_nodes) = hash_builder.split();
self.storage_nodes.extend(exclude_empty_from_pair(updated_nodes));
// Add deleted node paths.
self.removed_nodes.extend(exclude_empty(removed_keys));
}
/// Convert storage trie updates into [`StorageTrieUpdatesSorted`].
@ -229,10 +227,9 @@ impl StorageTrieUpdates {
/// This also sorts the set before serializing.
#[cfg(feature = "serde")]
mod serde_nibbles_set {
use std::collections::HashSet;
use reth_trie_common::Nibbles;
use crate::Nibbles;
use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
use std::collections::HashSet;
pub(super) fn serialize<S>(map: &HashSet<Nibbles>, serializer: S) -> Result<S::Ok, S::Error>
where
@ -266,15 +263,14 @@ mod serde_nibbles_set {
/// This also sorts the map's keys before encoding and serializing.
#[cfg(feature = "serde")]
mod serde_nibbles_map {
use std::{collections::HashMap, marker::PhantomData};
use crate::Nibbles;
use alloy_primitives::hex;
use reth_trie_common::Nibbles;
use serde::{
de::{Error, MapAccess, Visitor},
ser::SerializeMap,
Deserialize, Deserializer, Serialize, Serializer,
};
use std::{collections::HashMap, marker::PhantomData};
pub(super) fn serialize<S, T>(
map: &HashMap<Nibbles, T>,
@ -340,9 +336,13 @@ mod serde_nibbles_map {
/// Sorted trie updates used for lookups and insertions.
#[derive(PartialEq, Eq, Clone, Default, Debug)]
pub struct TrieUpdatesSorted {
pub(crate) account_nodes: Vec<(Nibbles, BranchNodeCompact)>,
pub(crate) removed_nodes: HashSet<Nibbles>,
pub(crate) storage_tries: HashMap<B256, StorageTrieUpdatesSorted>,
/// Sorted collection of updated state nodes with corresponding paths.
pub account_nodes: Vec<(Nibbles, BranchNodeCompact)>,
/// The set of removed state node keys.
pub removed_nodes: HashSet<Nibbles>,
/// Storage tries storage stored by hashed address of the account
/// the trie belongs to.
pub storage_tries: HashMap<B256, StorageTrieUpdatesSorted>,
}
impl TrieUpdatesSorted {
@ -365,9 +365,12 @@ impl TrieUpdatesSorted {
/// Sorted trie updates used for lookups and insertions.
#[derive(PartialEq, Eq, Clone, Default, Debug)]
pub struct StorageTrieUpdatesSorted {
pub(crate) is_deleted: bool,
pub(crate) storage_nodes: Vec<(Nibbles, BranchNodeCompact)>,
pub(crate) removed_nodes: HashSet<Nibbles>,
/// Flag indicating whether the trie has been deleted/wiped.
pub is_deleted: bool,
/// Sorted collection of updated storage nodes with corresponding paths.
pub storage_nodes: Vec<(Nibbles, BranchNodeCompact)>,
/// The set of removed storage node keys.
pub removed_nodes: HashSet<Nibbles>,
}
impl StorageTrieUpdatesSorted {
@ -402,21 +405,20 @@ fn exclude_empty_from_pair<V>(
/// Bincode-compatible trie updates type serde implementations.
#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
pub mod serde_bincode_compat {
use crate::{BranchNodeCompact, Nibbles};
use alloy_primitives::B256;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_with::{DeserializeAs, SerializeAs};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
};
use alloy_primitives::B256;
use reth_trie_common::{BranchNodeCompact, Nibbles};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_with::{DeserializeAs, SerializeAs};
/// Bincode-compatible [`super::TrieUpdates`] serde implementation.
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_trie::{serde_bincode_compat, updates::TrieUpdates};
/// use reth_trie_common::{serde_bincode_compat, updates::TrieUpdates};
/// use serde::{Deserialize, Serialize};
/// use serde_with::serde_as;
///
@ -480,7 +482,7 @@ pub mod serde_bincode_compat {
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_trie::{serde_bincode_compat, updates::StorageTrieUpdates};
/// use reth_trie_common::{serde_bincode_compat, updates::StorageTrieUpdates};
/// use serde::{Deserialize, Serialize};
/// use serde_with::serde_as;
///
@ -541,12 +543,12 @@ pub mod serde_bincode_compat {
#[cfg(test)]
mod tests {
use crate::updates::StorageTrieUpdates;
use super::super::{serde_bincode_compat, TrieUpdates};
use crate::{
serde_bincode_compat,
updates::{StorageTrieUpdates, TrieUpdates},
BranchNodeCompact, Nibbles,
};
use alloy_primitives::B256;
use reth_trie_common::{BranchNodeCompact, Nibbles};
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
@ -555,7 +557,7 @@ pub mod serde_bincode_compat {
#[serde_as]
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
struct Data {
#[serde_as(as = "serde_bincode_compat::TrieUpdates")]
#[serde_as(as = "serde_bincode_compat::updates::TrieUpdates")]
trie_updates: TrieUpdates,
}
@ -588,7 +590,7 @@ pub mod serde_bincode_compat {
#[serde_as]
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
struct Data {
#[serde_as(as = "serde_bincode_compat::StorageTrieUpdates")]
#[serde_as(as = "serde_bincode_compat::updates::StorageTrieUpdates")]
trie_updates: StorageTrieUpdates,
}

12
crates/trie/db/Cargo.toml
View File

@ -67,16 +67,17 @@ similar-asserts.workspace = true
metrics = ["reth-metrics", "reth-trie/metrics", "dep:metrics"]
serde = [
"dep:serde",
"reth-provider/serde",
"reth-trie/serde",
"reth-trie-common/serde",
"similar-asserts/serde",
"revm/serde",
"alloy-consensus/serde",
"alloy-primitives/serde",
"revm/serde",
"similar-asserts/serde"
"reth-trie/serde",
"reth-trie-common/serde",
"reth-provider/serde",
]
test-utils = [
"triehash",
"revm/test-utils",
"reth-trie-common/test-utils",
"reth-chainspec/test-utils",
"reth-primitives/test-utils",
@ -84,5 +85,4 @@ test-utils = [
"reth-db-api/test-utils",
"reth-provider/test-utils",
"reth-trie/test-utils",
"revm/test-utils"
]

7
crates/trie/parallel/src/root.rs
View File

@ -193,11 +193,8 @@ where
let root = hash_builder.root();
trie_updates.finalize(
account_node_iter.walker,
hash_builder,
prefix_sets.destroyed_accounts,
);
let removed_keys = account_node_iter.walker.take_removed_keys();
trie_updates.finalize(hash_builder, removed_keys, prefix_sets.destroyed_accounts);
let stats = tracker.finish();

2
crates/trie/sparse/benches/root.rs
View File

@ -146,7 +146,7 @@ pub fn calculate_root_from_leaves_repeated(c: &mut Criterion) {
hb.root();
if storage_updates.peek().is_some() {
trie_updates.finalize(node_iter.walker, hb);
trie_updates.finalize(hb, node_iter.walker.take_removed_keys());
}
}
},

21
crates/trie/trie/Cargo.toml
View File

@ -42,12 +42,6 @@ metrics = { workspace = true, optional = true }
# `test-utils` feature
triehash = { version = "0.8", optional = true }
# `serde` feature
serde = { workspace = true, optional = true }
# `serde-bincode-compat` feature
serde_with = { workspace = true, optional = true }
[dev-dependencies]
# reth
reth-primitives = { workspace = true, features = ["test-utils", "arbitrary"] }
@ -61,28 +55,21 @@ proptest.workspace = true
proptest-arbitrary-interop.workspace = true
serde_json.workspace = true
criterion.workspace = true
bincode.workspace = true
[features]
metrics = ["reth-metrics", "dep:metrics"]
serde = [
"dep:serde",
"alloy-consensus/serde",
"alloy-primitives/serde",
"revm/serde",
"alloy-consensus/serde",
"alloy-trie/serde",
"revm/serde",
"reth-trie-common/serde"
]
serde-bincode-compat = [
"serde_with",
"reth-primitives/serde-bincode-compat",
"alloy-consensus/serde-bincode-compat"
]
test-utils = [
"triehash",
"reth-trie-common/test-utils",
"reth-primitives/test-utils",
"revm/test-utils",
"reth-primitives/test-utils",
"reth-trie-common/test-utils",
"reth-stages-types/test-utils"
]

14
crates/trie/trie/src/lib.rs
View File

@ -50,9 +50,6 @@ pub mod witness;
mod trie;
pub use trie::{StateRoot, StorageRoot};
/// Buffer for trie updates.
pub mod updates;
/// Utilities for state root checkpoint progress.
mod progress;
pub use progress::{IntermediateStateRootState, StateRootProgress};
@ -63,17 +60,6 @@ pub mod stats;
// re-export for convenience
pub use reth_trie_common::*;
/// Bincode-compatible serde implementations for trie types.
///
/// `bincode` crate allows for more efficient serialization of trie types, because it allows
/// non-string map keys.
///
/// Read more: <https://github.com/paradigmxyz/reth/issues/11370>
#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
pub mod serde_bincode_compat {
pub use super::updates::serde_bincode_compat as updates;
}
/// Trie calculation metrics.
#[cfg(feature = "metrics")]
pub mod metrics;

10
crates/trie/trie/src/trie.rs
View File

@ -258,11 +258,8 @@ where
let root = hash_builder.root();
trie_updates.finalize(
account_node_iter.walker,
hash_builder,
self.prefix_sets.destroyed_accounts,
);
let removed_keys = account_node_iter.walker.take_removed_keys();
trie_updates.finalize(hash_builder, removed_keys, self.prefix_sets.destroyed_accounts);
let stats = tracker.finish();
@ -434,7 +431,8 @@ where
let root = hash_builder.root();
let mut trie_updates = StorageTrieUpdates::default();
trie_updates.finalize(storage_node_iter.walker, hash_builder);
let removed_keys = storage_node_iter.walker.take_removed_keys();
trie_updates.finalize(hash_builder, removed_keys);
let stats = tracker.finish();

9
crates/trie/trie/src/walker.rs
View File

@ -58,8 +58,13 @@ impl<C> TrieWalker<C> {
/// Split the walker into stack and trie updates.
pub fn split(mut self) -> (Vec<CursorSubNode>, HashSet<Nibbles>) {
let keys = self.removed_keys.take();
(self.stack, keys.unwrap_or_default())
let keys = self.take_removed_keys();
(self.stack, keys)
}
/// Take removed keys from the walker.
pub fn take_removed_keys(&mut self) -> HashSet<Nibbles> {
self.removed_keys.take().unwrap_or_default()
}
/// Prints the current stack of trie nodes.