jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: move IntegerList to reth-primitives-traits (#8948)

Browse Source
This commit is contained in:
joshieDo
2024-06-19 16:13:31 +02:00
committed by GitHub
parent a3fd112915
commit d0b241c0c2
16 changed files with 32 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Cargo.lock generated
View File

@ -6646,6 +6646,7 @@ dependencies = [
"reth-metrics",
"reth-nippy-jar",
"reth-primitives",
"reth-primitives-traits",
"reth-prune-types",
"reth-stages-types",
"reth-storage-errors",
@ -6681,6 +6682,7 @@ dependencies = [
"rand 0.8.5",
"reth-codecs",
"reth-primitives",
"reth-primitives-traits",
"reth-prune-types",
"reth-stages-types",
"reth-storage-errors",
@ -6702,6 +6704,7 @@ dependencies = [
"reth-db-api",
"reth-etl",
"reth-primitives",
"reth-primitives-traits",
"reth-provider",
"reth-stages-types",
"reth-trie",
@ -7743,7 +7746,6 @@ dependencies = [
"reth-static-file-types",
"reth-trie-common",
"revm-primitives",
"roaring",
"secp256k1",
"serde",
"serde_json",
@ -7775,8 +7777,11 @@ dependencies = [
"rand 0.8.5",
"reth-codecs",
"revm-primitives",
"roaring",
"serde",
"serde_json",
"test-fuzz",
"thiserror-no-std",
]
[[package]]

7
crates/primitives-traits/Cargo.toml
View File

@ -24,6 +24,10 @@ alloy-rpc-types-eth = { workspace = true, optional = true }
derive_more.workspace = true
revm-primitives.workspace = true
# misc
thiserror-no-std = { workspace = true, default-features = false }
roaring = "0.10.2"
# required by reth-codecs
modular-bitfield.workspace = true
bytes.workspace = true
@ -40,10 +44,11 @@ proptest.workspace = true
proptest-derive.workspace = true
test-fuzz.workspace = true
rand.workspace = true
serde_json.workspace = true
[features]
default = ["std"]
std = []
std = ["thiserror-no-std/std"]
test-utils = ["arbitrary"]
arbitrary = [
"dep:arbitrary",

0
crates/primitives/src/integer_list.rs → crates/primitives-traits/src/integer_list.rs
View File

3
crates/primitives-traits/src/lib.rs
View File

@ -21,6 +21,9 @@ pub mod constants;
pub mod account;
pub use account::Account;
mod integer_list;
pub use integer_list::IntegerList;
/// Common header types
pub mod header;
#[cfg(any(test, feature = "arbitrary", feature = "test-utils"))]

4
crates/primitives/Cargo.toml
View File

@ -49,7 +49,6 @@ serde.workspace = true
tempfile = { workspace = true, optional = true }
thiserror-no-std = { workspace = true, default-features = false }
zstd = { version = "0.13", features = ["experimental"], optional = true }
roaring = "0.10.2"
# arbitrary utils
arbitrary = { workspace = true, features = ["derive"], optional = true }
@ -123,6 +122,3 @@ name = "validate_blob_tx"
required-features = ["arbitrary", "c-kzg"]
harness = false
[[bench]]
name = "integer_list"
harness = false

244
crates/primitives/benches/integer_list.rs
View File

@ -1,244 +0,0 @@
#![allow(missing_docs)]
use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
use rand::prelude::*;
pub fn new_pre_sorted(c: &mut Criterion) {
let mut group = c.benchmark_group("new_pre_sorted");
for delta in [1, 100, 1000, 10000] {
let integers_usize = generate_integers(2000, delta);
assert_eq!(integers_usize.len(), 2000);
let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::<Vec<_>>();
assert_eq!(integers_u64.len(), 2000);
group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| {
b.iter(|| elias_fano::IntegerList::new_pre_sorted(black_box(&integers_usize)));
});
group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| {
b.iter(|| reth_primitives::IntegerList::new_pre_sorted(black_box(&integers_u64)));
});
}
}
pub fn rank_select(c: &mut Criterion) {
let mut group = c.benchmark_group("rank + select");
for delta in [1, 100, 1000, 10000] {
let integers_usize = generate_integers(2000, delta);
assert_eq!(integers_usize.len(), 2000);
let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::<Vec<_>>();
assert_eq!(integers_u64.len(), 2000);
group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| {
b.iter_batched(
|| {
let (index, element) =
integers_usize.iter().enumerate().choose(&mut thread_rng()).unwrap();
(elias_fano::IntegerList::new_pre_sorted(&integers_usize).0, index, *element)
},
|(list, index, element)| {
let list = list.enable_rank();
list.rank(element);
list.select(index);
},
BatchSize::PerIteration,
);
});
group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| {
b.iter_batched(
|| {
let (index, element) =
integers_u64.iter().enumerate().choose(&mut thread_rng()).unwrap();
(
reth_primitives::IntegerList::new_pre_sorted(&integers_u64),
index as u64,
*element,
)
},
|(list, index, element)| {
list.rank(element);
list.select(index);
},
BatchSize::PerIteration,
);
});
}
}
fn generate_integers(n: usize, delta: usize) -> Vec<usize> {
(0..n).fold(Vec::new(), |mut vec, _| {
vec.push(vec.last().map_or(0, |last| {
last + thread_rng().gen_range(delta - delta / 2..=delta + delta / 2)
}));
vec
})
}
criterion_group! {
name = benches;
config = Criterion::default();
targets = new_pre_sorted, rank_select
}
criterion_main!(benches);
/// Implementation from <https://github.com/paradigmxyz/reth/blob/cda5d4e7c53ccc898b7725eb5d3b46c35e4da7f8/crates/primitives/src/integer_list.rs>
/// adapted to work with `sucds = "0.8.1"`
#[allow(unused, unreachable_pub)]
mod elias_fano {
use derive_more::Deref;
use std::{fmt, ops::Deref};
use sucds::{mii_sequences::EliasFano, Serializable};
#[derive(Clone, PartialEq, Eq, Default, Deref)]
pub struct IntegerList(pub EliasFano);
impl fmt::Debug for IntegerList {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let vec: Vec<usize> = self.0.iter(0).collect();
write!(f, "IntegerList {vec:?}")
}
}
impl IntegerList {
/// Creates an `IntegerList` from a list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
///
/// # Returns
///
/// Returns an error if the list is empty or not pre-sorted.
pub fn new<T: AsRef<[usize]>>(list: T) -> Result<Self, EliasFanoError> {
let mut builder = EliasFanoBuilder::new(
list.as_ref().iter().max().map_or(0, |max| max + 1),
list.as_ref().len(),
)
.map_err(|err| EliasFanoError::InvalidInput(err.to_string()))?;
builder.extend(list.as_ref().iter().copied());
Ok(Self(builder.build()))
}
// Creates an IntegerList from a pre-sorted list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
///
/// # Panics
///
/// Panics if the list is empty or not pre-sorted.
pub fn new_pre_sorted<T: AsRef<[usize]>>(list: T) -> Self {
Self::new(list).expect("IntegerList must be pre-sorted and non-empty.")
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_bytes(&self) -> Vec<u8> {
let mut vec = Vec::with_capacity(self.0.size_in_bytes());
self.0.serialize_into(&mut vec).expect("not able to encode integer list.");
vec
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_mut_bytes<B: bytes::BufMut>(&self, buf: &mut B) {
let len = self.0.size_in_bytes();
let mut vec = Vec::with_capacity(len);
self.0.serialize_into(&mut vec).unwrap();
buf.put_slice(vec.as_slice());
}
/// Deserializes a sequence of bytes into a proper [`IntegerList`].
pub fn from_bytes(data: &[u8]) -> Result<Self, EliasFanoError> {
Ok(Self(
EliasFano::deserialize_from(data).map_err(|_| EliasFanoError::FailedDeserialize)?,
))
}
}
macro_rules! impl_uint {
($($w:tt),+) => {
$(
impl From<Vec<$w>> for IntegerList {
fn from(v: Vec<$w>) -> Self {
let v: Vec<usize> = v.iter().map(|v| *v as usize).collect();
Self::new(v.as_slice()).expect("could not create list.")
}
}
)+
};
}
impl_uint!(usize, u64, u32, u8, u16);
impl Serialize for IntegerList {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let vec = self.0.iter(0).collect::<Vec<usize>>();
let mut seq = serializer.serialize_seq(Some(self.len()))?;
for e in vec {
seq.serialize_element(&e)?;
}
seq.end()
}
}
struct IntegerListVisitor;
impl<'de> Visitor<'de> for IntegerListVisitor {
type Value = IntegerList;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a usize array")
}
fn visit_seq<E>(self, mut seq: E) -> Result<Self::Value, E::Error>
where
E: SeqAccess<'de>,
{
let mut list = Vec::new();
while let Some(item) = seq.next_element()? {
list.push(item);
}
IntegerList::new(list)
.map_err(|_| serde::de::Error::invalid_value(Unexpected::Seq, &self))
}
}
impl<'de> Deserialize<'de> for IntegerList {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_byte_buf(IntegerListVisitor)
}
}
#[cfg(any(test, feature = "arbitrary"))]
use arbitrary::{Arbitrary, Unstructured};
use serde::{
de::{SeqAccess, Unexpected, Visitor},
ser::SerializeSeq,
Deserialize, Deserializer, Serialize, Serializer,
};
use sucds::mii_sequences::EliasFanoBuilder;
#[cfg(any(test, feature = "arbitrary"))]
impl<'a> Arbitrary<'a> for IntegerList {
fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
let mut nums: Vec<usize> = Vec::arbitrary(u)?;
nums.sort();
Self::new(&nums).map_err(|_| arbitrary::Error::IncorrectFormat)
}
}
/// Primitives error type.
#[derive(Debug, thiserror_no_std::Error)]
pub enum EliasFanoError {
/// The provided input is invalid.
#[error("{0}")]
InvalidInput(String),
/// Failed to deserialize data into type.
#[error("failed to deserialize data into type")]
FailedDeserialize,
}
}

2
crates/primitives/src/lib.rs
View File

@ -34,7 +34,6 @@ pub mod eip4844;
mod error;
pub mod genesis;
pub mod header;
mod integer_list;
mod log;
pub mod proofs;
mod receipt;
@ -61,7 +60,6 @@ pub use constants::{
pub use error::{GotExpected, GotExpectedBoxed};
pub use genesis::{ChainConfig, Genesis, GenesisAccount};
pub use header::{Header, HeadersDirection, SealedHeader};
pub use integer_list::IntegerList;
pub use log::{logs_bloom, Log};
pub use receipt::{
gas_spent_by_transactions, Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts,

1
crates/storage/db-api/Cargo.toml
View File

@ -15,6 +15,7 @@ workspace = true
# reth
reth-codecs.workspace = true
reth-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-prune-types.workspace = true
reth-storage-errors.workspace = true
reth-stages-types.workspace = true

2
crates/storage/db-api/src/models/integer_list.rs
View File

@ -4,7 +4,7 @@ use crate::{
table::{Compress, Decompress},
DatabaseError,
};
use reth_primitives::IntegerList;
use reth_primitives_traits::IntegerList;
impl Compress for IntegerList {
type Compressed = Vec<u8>;

3
crates/storage/db-common/Cargo.toml
View File

@ -31,5 +31,8 @@ serde_json.workspace = true
# tracing
tracing.workspace = true
[dev-dependencies]
reth-primitives-traits.workspace = true
[lints]
workspace = true

3
crates/storage/db-common/src/init.rs
View File

@ -534,8 +534,9 @@ mod tests {
transaction::DbTx,
};
use reth_primitives::{
Genesis, IntegerList, GOERLI_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH,
Genesis, GOERLI_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH,
};
use reth_primitives_traits::IntegerList;
use reth_provider::test_utils::create_test_provider_factory_with_chain_spec;
fn collect_table_entries<DB, T>(

1
crates/storage/db/Cargo.toml
View File

@ -15,6 +15,7 @@ workspace = true
# reth
reth-db-api.workspace = true
reth-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-fs-util.workspace = true
reth-storage-errors.workspace = true
reth-libmdbx = { workspace = true, optional = true, features = [

3
crates/storage/db/src/implementation/mdbx/mod.rs
View File

@ -482,7 +482,8 @@ mod tests {
table::{Encode, Table},
};
use reth_libmdbx::Error;
use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, B256, U256};
use reth_primitives::{Account, Address, Header, StorageEntry, B256, U256};
use reth_primitives_traits::IntegerList;
use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation};
use std::str::FromStr;
use tempfile::TempDir;

2
crates/storage/db/src/tables/codecs/fuzz/inputs.rs
View File

@ -1,6 +1,6 @@
//! Curates the input coming from the fuzzer for certain types.
use reth_primitives::IntegerList;
use reth_primitives_traits::IntegerList;
use serde::{Deserialize, Serialize};
/// Makes sure that the list provided by the fuzzer is not empty and pre-sorted

3
crates/storage/db/src/tables/codecs/fuzz/mod.rs
View File

@ -19,6 +19,9 @@ macro_rules! impl_fuzzer_with_input {
#[allow(unused_imports)]
use reth_primitives::*;
#[allow(unused_imports)]
use reth_primitives_traits::*;
#[allow(unused_imports)]
use super::inputs::*;

5
crates/storage/db/src/tables/mod.rs
View File

@ -32,9 +32,10 @@ use reth_db_api::{
table::{Decode, DupSort, Encode, Table},
};
use reth_primitives::{
Account, Address, BlockHash, BlockNumber, Bytecode, Header, IntegerList, Receipt, Requests,
StorageEntry, TransactionSignedNoHash, TxHash, TxNumber, B256,
Account, Address, BlockHash, BlockNumber, Bytecode, Header, Receipt, Requests, StorageEntry,
TransactionSignedNoHash, TxHash, TxNumber, B256,
};
use reth_primitives_traits::IntegerList;
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::StageCheckpoint;
use reth_trie_common::{StorageTrieEntry, StoredBranchNode, StoredNibbles, StoredNibblesSubKey};