diff --git a/Cargo.lock b/Cargo.lock index 1d55ac9a9..a1fc1a92d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6646,6 +6646,7 @@ dependencies = [ "reth-metrics", "reth-nippy-jar", "reth-primitives", + "reth-primitives-traits", "reth-prune-types", "reth-stages-types", "reth-storage-errors", @@ -6681,6 +6682,7 @@ dependencies = [ "rand 0.8.5", "reth-codecs", "reth-primitives", + "reth-primitives-traits", "reth-prune-types", "reth-stages-types", "reth-storage-errors", @@ -6702,6 +6704,7 @@ dependencies = [ "reth-db-api", "reth-etl", "reth-primitives", + "reth-primitives-traits", "reth-provider", "reth-stages-types", "reth-trie", @@ -7743,7 +7746,6 @@ dependencies = [ "reth-static-file-types", "reth-trie-common", "revm-primitives", - "roaring", "secp256k1", "serde", "serde_json", @@ -7775,8 +7777,11 @@ dependencies = [ "rand 0.8.5", "reth-codecs", "revm-primitives", + "roaring", "serde", + "serde_json", "test-fuzz", + "thiserror-no-std", ] [[package]] diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index 97ed1a344..871d9ea31 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -24,6 +24,10 @@ alloy-rpc-types-eth = { workspace = true, optional = true } derive_more.workspace = true revm-primitives.workspace = true +# misc +thiserror-no-std = { workspace = true, default-features = false } +roaring = "0.10.2" + # required by reth-codecs modular-bitfield.workspace = true bytes.workspace = true @@ -40,10 +44,11 @@ proptest.workspace = true proptest-derive.workspace = true test-fuzz.workspace = true rand.workspace = true +serde_json.workspace = true [features] default = ["std"] -std = [] +std = ["thiserror-no-std/std"] test-utils = ["arbitrary"] arbitrary = [ "dep:arbitrary", diff --git a/crates/primitives/src/integer_list.rs b/crates/primitives-traits/src/integer_list.rs similarity index 100% rename from crates/primitives/src/integer_list.rs rename to crates/primitives-traits/src/integer_list.rs diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 0051f6c7d..3a352f292 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -21,6 +21,9 @@ pub mod constants; pub mod account; pub use account::Account; +mod integer_list; +pub use integer_list::IntegerList; + /// Common header types pub mod header; #[cfg(any(test, feature = "arbitrary", feature = "test-utils"))] diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index f5ae16277..37b67fe46 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -49,7 +49,6 @@ serde.workspace = true tempfile = { workspace = true, optional = true } thiserror-no-std = { workspace = true, default-features = false } zstd = { version = "0.13", features = ["experimental"], optional = true } -roaring = "0.10.2" # arbitrary utils arbitrary = { workspace = true, features = ["derive"], optional = true } @@ -123,6 +122,3 @@ name = "validate_blob_tx" required-features = ["arbitrary", "c-kzg"] harness = false -[[bench]] -name = "integer_list" -harness = false diff --git a/crates/primitives/benches/integer_list.rs b/crates/primitives/benches/integer_list.rs deleted file mode 100644 index 097280748..000000000 --- a/crates/primitives/benches/integer_list.rs +++ /dev/null @@ -1,244 +0,0 @@ -#![allow(missing_docs)] -use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; -use rand::prelude::*; - -pub fn new_pre_sorted(c: &mut Criterion) { - let mut group = c.benchmark_group("new_pre_sorted"); - - for delta in [1, 100, 1000, 10000] { - let integers_usize = generate_integers(2000, delta); - assert_eq!(integers_usize.len(), 2000); - - let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::>(); - assert_eq!(integers_u64.len(), 2000); - - group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| { - b.iter(|| elias_fano::IntegerList::new_pre_sorted(black_box(&integers_usize))); - }); - - group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| { - b.iter(|| reth_primitives::IntegerList::new_pre_sorted(black_box(&integers_u64))); - }); - } -} - -pub fn rank_select(c: &mut Criterion) { - let mut group = c.benchmark_group("rank + select"); - - for delta in [1, 100, 1000, 10000] { - let integers_usize = generate_integers(2000, delta); - assert_eq!(integers_usize.len(), 2000); - - let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::>(); - assert_eq!(integers_u64.len(), 2000); - - group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| { - b.iter_batched( - || { - let (index, element) = - integers_usize.iter().enumerate().choose(&mut thread_rng()).unwrap(); - (elias_fano::IntegerList::new_pre_sorted(&integers_usize).0, index, *element) - }, - |(list, index, element)| { - let list = list.enable_rank(); - list.rank(element); - list.select(index); - }, - BatchSize::PerIteration, - ); - }); - - group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| { - b.iter_batched( - || { - let (index, element) = - integers_u64.iter().enumerate().choose(&mut thread_rng()).unwrap(); - ( - reth_primitives::IntegerList::new_pre_sorted(&integers_u64), - index as u64, - *element, - ) - }, - |(list, index, element)| { - list.rank(element); - list.select(index); - }, - BatchSize::PerIteration, - ); - }); - } -} - -fn generate_integers(n: usize, delta: usize) -> Vec { - (0..n).fold(Vec::new(), |mut vec, _| { - vec.push(vec.last().map_or(0, |last| { - last + thread_rng().gen_range(delta - delta / 2..=delta + delta / 2) - })); - vec - }) -} - -criterion_group! { - name = benches; - config = Criterion::default(); - targets = new_pre_sorted, rank_select -} -criterion_main!(benches); - -/// Implementation from -/// adapted to work with `sucds = "0.8.1"` -#[allow(unused, unreachable_pub)] -mod elias_fano { - use derive_more::Deref; - use std::{fmt, ops::Deref}; - use sucds::{mii_sequences::EliasFano, Serializable}; - - #[derive(Clone, PartialEq, Eq, Default, Deref)] - pub struct IntegerList(pub EliasFano); - - impl fmt::Debug for IntegerList { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let vec: Vec = self.0.iter(0).collect(); - write!(f, "IntegerList {vec:?}") - } - } - - impl IntegerList { - /// Creates an `IntegerList` from a list of integers. `usize` is safe to use since - /// [`sucds::EliasFano`] restricts its compilation to 64bits. - /// - /// # Returns - /// - /// Returns an error if the list is empty or not pre-sorted. - pub fn new>(list: T) -> Result { - let mut builder = EliasFanoBuilder::new( - list.as_ref().iter().max().map_or(0, |max| max + 1), - list.as_ref().len(), - ) - .map_err(|err| EliasFanoError::InvalidInput(err.to_string()))?; - builder.extend(list.as_ref().iter().copied()); - Ok(Self(builder.build())) - } - - // Creates an IntegerList from a pre-sorted list of integers. `usize` is safe to use since - /// [`sucds::EliasFano`] restricts its compilation to 64bits. - /// - /// # Panics - /// - /// Panics if the list is empty or not pre-sorted. - pub fn new_pre_sorted>(list: T) -> Self { - Self::new(list).expect("IntegerList must be pre-sorted and non-empty.") - } - - /// Serializes a [`IntegerList`] into a sequence of bytes. - pub fn to_bytes(&self) -> Vec { - let mut vec = Vec::with_capacity(self.0.size_in_bytes()); - self.0.serialize_into(&mut vec).expect("not able to encode integer list."); - vec - } - - /// Serializes a [`IntegerList`] into a sequence of bytes. - pub fn to_mut_bytes(&self, buf: &mut B) { - let len = self.0.size_in_bytes(); - let mut vec = Vec::with_capacity(len); - self.0.serialize_into(&mut vec).unwrap(); - buf.put_slice(vec.as_slice()); - } - - /// Deserializes a sequence of bytes into a proper [`IntegerList`]. - pub fn from_bytes(data: &[u8]) -> Result { - Ok(Self( - EliasFano::deserialize_from(data).map_err(|_| EliasFanoError::FailedDeserialize)?, - )) - } - } - - macro_rules! impl_uint { - ($($w:tt),+) => { - $( - impl From> for IntegerList { - fn from(v: Vec<$w>) -> Self { - let v: Vec = v.iter().map(|v| *v as usize).collect(); - Self::new(v.as_slice()).expect("could not create list.") - } - } - )+ - }; - } - - impl_uint!(usize, u64, u32, u8, u16); - - impl Serialize for IntegerList { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let vec = self.0.iter(0).collect::>(); - let mut seq = serializer.serialize_seq(Some(self.len()))?; - for e in vec { - seq.serialize_element(&e)?; - } - seq.end() - } - } - - struct IntegerListVisitor; - impl<'de> Visitor<'de> for IntegerListVisitor { - type Value = IntegerList; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("a usize array") - } - - fn visit_seq(self, mut seq: E) -> Result - where - E: SeqAccess<'de>, - { - let mut list = Vec::new(); - while let Some(item) = seq.next_element()? { - list.push(item); - } - - IntegerList::new(list) - .map_err(|_| serde::de::Error::invalid_value(Unexpected::Seq, &self)) - } - } - - impl<'de> Deserialize<'de> for IntegerList { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_byte_buf(IntegerListVisitor) - } - } - - #[cfg(any(test, feature = "arbitrary"))] - use arbitrary::{Arbitrary, Unstructured}; - use serde::{ - de::{SeqAccess, Unexpected, Visitor}, - ser::SerializeSeq, - Deserialize, Deserializer, Serialize, Serializer, - }; - use sucds::mii_sequences::EliasFanoBuilder; - - #[cfg(any(test, feature = "arbitrary"))] - impl<'a> Arbitrary<'a> for IntegerList { - fn arbitrary(u: &mut Unstructured<'a>) -> Result { - let mut nums: Vec = Vec::arbitrary(u)?; - nums.sort(); - Self::new(&nums).map_err(|_| arbitrary::Error::IncorrectFormat) - } - } - - /// Primitives error type. - #[derive(Debug, thiserror_no_std::Error)] - pub enum EliasFanoError { - /// The provided input is invalid. - #[error("{0}")] - InvalidInput(String), - /// Failed to deserialize data into type. - #[error("failed to deserialize data into type")] - FailedDeserialize, - } -} diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index b965de167..e5b0f4a6d 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -34,7 +34,6 @@ pub mod eip4844; mod error; pub mod genesis; pub mod header; -mod integer_list; mod log; pub mod proofs; mod receipt; @@ -61,7 +60,6 @@ pub use constants::{ pub use error::{GotExpected, GotExpectedBoxed}; pub use genesis::{ChainConfig, Genesis, GenesisAccount}; pub use header::{Header, HeadersDirection, SealedHeader}; -pub use integer_list::IntegerList; pub use log::{logs_bloom, Log}; pub use receipt::{ gas_spent_by_transactions, Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts, diff --git a/crates/storage/db-api/Cargo.toml b/crates/storage/db-api/Cargo.toml index a3ecc56f8..1f55e6d7f 100644 --- a/crates/storage/db-api/Cargo.toml +++ b/crates/storage/db-api/Cargo.toml @@ -15,6 +15,7 @@ workspace = true # reth reth-codecs.workspace = true reth-primitives.workspace = true +reth-primitives-traits.workspace = true reth-prune-types.workspace = true reth-storage-errors.workspace = true reth-stages-types.workspace = true diff --git a/crates/storage/db-api/src/models/integer_list.rs b/crates/storage/db-api/src/models/integer_list.rs index e419a9435..f47605bf8 100644 --- a/crates/storage/db-api/src/models/integer_list.rs +++ b/crates/storage/db-api/src/models/integer_list.rs @@ -4,7 +4,7 @@ use crate::{ table::{Compress, Decompress}, DatabaseError, }; -use reth_primitives::IntegerList; +use reth_primitives_traits::IntegerList; impl Compress for IntegerList { type Compressed = Vec; diff --git a/crates/storage/db-common/Cargo.toml b/crates/storage/db-common/Cargo.toml index c6eebae87..3299bcc12 100644 --- a/crates/storage/db-common/Cargo.toml +++ b/crates/storage/db-common/Cargo.toml @@ -31,5 +31,8 @@ serde_json.workspace = true # tracing tracing.workspace = true +[dev-dependencies] +reth-primitives-traits.workspace = true + [lints] workspace = true diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index d8bf583df..3907efd58 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -534,8 +534,9 @@ mod tests { transaction::DbTx, }; use reth_primitives::{ - Genesis, IntegerList, GOERLI_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH, + Genesis, GOERLI_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH, }; + use reth_primitives_traits::IntegerList; use reth_provider::test_utils::create_test_provider_factory_with_chain_spec; fn collect_table_entries( diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 2bac4c107..b563e1d78 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -15,6 +15,7 @@ workspace = true # reth reth-db-api.workspace = true reth-primitives.workspace = true +reth-primitives-traits.workspace = true reth-fs-util.workspace = true reth-storage-errors.workspace = true reth-libmdbx = { workspace = true, optional = true, features = [ diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 4191d7aae..cc9f055e9 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -482,7 +482,8 @@ mod tests { table::{Encode, Table}, }; use reth_libmdbx::Error; - use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, B256, U256}; + use reth_primitives::{Account, Address, Header, StorageEntry, B256, U256}; + use reth_primitives_traits::IntegerList; use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation}; use std::str::FromStr; use tempfile::TempDir; diff --git a/crates/storage/db/src/tables/codecs/fuzz/inputs.rs b/crates/storage/db/src/tables/codecs/fuzz/inputs.rs index 533b6b692..2c944e158 100644 --- a/crates/storage/db/src/tables/codecs/fuzz/inputs.rs +++ b/crates/storage/db/src/tables/codecs/fuzz/inputs.rs @@ -1,6 +1,6 @@ //! Curates the input coming from the fuzzer for certain types. -use reth_primitives::IntegerList; +use reth_primitives_traits::IntegerList; use serde::{Deserialize, Serialize}; /// Makes sure that the list provided by the fuzzer is not empty and pre-sorted diff --git a/crates/storage/db/src/tables/codecs/fuzz/mod.rs b/crates/storage/db/src/tables/codecs/fuzz/mod.rs index 826f44d43..1d038bf7e 100644 --- a/crates/storage/db/src/tables/codecs/fuzz/mod.rs +++ b/crates/storage/db/src/tables/codecs/fuzz/mod.rs @@ -19,6 +19,9 @@ macro_rules! impl_fuzzer_with_input { #[allow(unused_imports)] use reth_primitives::*; + #[allow(unused_imports)] + use reth_primitives_traits::*; + #[allow(unused_imports)] use super::inputs::*; diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 367a7dd55..c968647a9 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -32,9 +32,10 @@ use reth_db_api::{ table::{Decode, DupSort, Encode, Table}, }; use reth_primitives::{ - Account, Address, BlockHash, BlockNumber, Bytecode, Header, IntegerList, Receipt, Requests, - StorageEntry, TransactionSignedNoHash, TxHash, TxNumber, B256, + Account, Address, BlockHash, BlockNumber, Bytecode, Header, Receipt, Requests, StorageEntry, + TransactionSignedNoHash, TxHash, TxNumber, B256, }; +use reth_primitives_traits::IntegerList; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::StageCheckpoint; use reth_trie_common::{StorageTrieEntry, StoredBranchNode, StoredNibbles, StoredNibblesSubKey};