diff --git a/Cargo.lock b/Cargo.lock index d4bebd4c7..c7c8049ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3094,6 +3094,7 @@ dependencies = [ "eyre", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rlp", "thiserror", "tokio", @@ -3107,22 +3108,31 @@ version = "0.1.0" name = "reth-db" version = "0.1.0" dependencies = [ + "arbitrary", "async-trait", "bytes", "criterion", "eyre", + "futures", + "heapless", "iai", + "modular-bitfield", "page_size", "parity-scale-codec", "postcard", + "rand 0.8.5", + "reth-codecs", + "reth-db", "reth-interfaces", "reth-libmdbx", "reth-primitives", + "secp256k1", "serde", "tempfile", "test-fuzz", "thiserror", "tokio", + "tokio-stream", ] [[package]] @@ -3217,6 +3227,7 @@ dependencies = [ "reth-db", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rlp", "revm", "rlp", @@ -3335,6 +3346,7 @@ dependencies = [ "reth-eth-wire", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rlp", "reth-rlp-derive", "reth-tasks", @@ -3390,6 +3402,34 @@ dependencies = [ "triehash", ] +[[package]] +name = "reth-provider" +version = "0.1.0" +dependencies = [ + "arbitrary", + "async-trait", + "auto_impl", + "bytes", + "futures", + "heapless", + "hex-literal", + "modular-bitfield", + "parity-scale-codec", + "postcard", + "rand 0.8.5", + "reth-codecs", + "reth-db", + "reth-interfaces", + "reth-primitives", + "reth-rpc-types", + "secp256k1", + "serde", + "test-fuzz", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "reth-rlp" version = "0.1.2" @@ -3425,6 +3465,7 @@ dependencies = [ "jsonrpsee", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rpc-api", "reth-rpc-types", "reth-transaction-pool", @@ -3472,6 +3513,7 @@ dependencies = [ "reth-headers-downloaders", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rlp", "tempfile", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 47a517686..81ce5518e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "crates/common/rlp", "crates/common/rlp-derive", "crates/consensus", - "crates/db", "crates/executor", "crates/interfaces", "crates/net/p2p", @@ -21,11 +20,13 @@ members = [ "crates/net/bodies-downloaders", "crates/primitives", "crates/stages", + "crates/storage/codecs", + "crates/storage/db", + "crates/storage/libmdbx-rs", + "crates/storage/libmdbx-rs/mdbx-sys", + "crates/storage/provider", "crates/tracing", "crates/tasks", "crates/transaction-pool", - "crates/db", - "crates/libmdbx-rs", - "crates/libmdbx-rs/mdbx-sys", ] -default-members = ["bin/reth"] +default-members = ["bin/reth"] \ No newline at end of file diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index ee212fae5..5fde2b4b8 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -9,7 +9,7 @@ readme = "README.md" [dependencies] # reth reth-primitives = { path = "../../crates/primitives" } -reth-db = {path = "../../crates/db"} +reth-db = {path = "../../crates/storage/db", features = ["mdbx"]} reth-stages = {path = "../../crates/stages"} reth-interfaces = {path = "../../crates/interfaces"} reth-transaction-pool = {path = "../../crates/transaction-pool"} diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 72859a3d9..d53eecfeb 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -24,9 +24,9 @@ impl Command { let path = shellexpand::full(&self.db)?.into_owned(); let expanded_db_path = Path::new(&path); std::fs::create_dir_all(expanded_db_path)?; - let db = Arc::new(reth_db::kv::Env::::open( + let db = Arc::new(reth_db::mdbx::Env::::open( expanded_db_path, - reth_db::kv::EnvKind::RW, + reth_db::mdbx::EnvKind::RW, )?); info!("DB opened"); diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml index 36699076f..fad7e3750 100644 --- a/crates/consensus/Cargo.toml +++ b/crates/consensus/Cargo.toml @@ -10,6 +10,7 @@ readme = "README.md" # reth reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } +reth-provider = { path = "../storage/provider" } reth-rlp = {path = "../common/rlp"} # common diff --git a/crates/consensus/src/verification.rs b/crates/consensus/src/verification.rs index 2cec15a28..f02e555bb 100644 --- a/crates/consensus/src/verification.rs +++ b/crates/consensus/src/verification.rs @@ -1,14 +1,11 @@ //! ALl functions for verification of block use crate::{config, Config}; -use reth_interfaces::{ - consensus::Error, - provider::{AccountProvider, HeaderProvider}, - Result as RethResult, -}; +use reth_interfaces::{consensus::Error, Result as RethResult}; use reth_primitives::{ Account, BlockLocked, BlockNumber, SealedHeader, Transaction, TxEip1559, TxEip2930, TxLegacy, EMPTY_OMMER_ROOT, H256, U256, }; +use reth_provider::{AccountProvider, HeaderProvider}; use std::time::SystemTime; /// Validate header standalone diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs deleted file mode 100644 index 524643097..000000000 --- a/crates/db/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Module that interacts with MDBX. - -#![warn(missing_debug_implementations, missing_docs, unreachable_pub)] -#![deny(unused_must_use, rust_2018_idioms)] -#![doc(test( - no_crate_inject, - attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) -))] - -/// Rust bindings for [MDBX](https://libmdbx.dqdkfa.ru/). -pub mod mdbx { - pub use reth_libmdbx::*; -} - -pub mod kv; -mod utils; diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index c2009cc53..6ae2c9b14 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -11,6 +11,8 @@ readme = "README.md" reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } reth-rlp = { path = "../common/rlp" } +reth-db = { path = "../storage/db" } +reth-provider = { path = "../storage/provider" } reth-consensus = { path = "../consensus" } revm = "2.3" @@ -34,4 +36,4 @@ sha3 = { version = "0.10", default-features = false } [dev-dependencies] -reth-db = { path = "../db", features = ["test-utils"]} \ No newline at end of file +reth-db = { path = "../storage/db", features = ["test-utils"] } diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index af209cf82..54466f817 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -4,15 +4,13 @@ use crate::{ Config, }; use hashbrown::hash_map::Entry; -use reth_interfaces::{ - db::{models::AccountBeforeTx, tables, DbTxMut, Error as DbError}, - executor::Error, - provider::StateProvider, -}; +use reth_db::{models::AccountBeforeTx, tables, transaction::DbTxMut, Error as DbError}; +use reth_interfaces::executor::Error; use reth_primitives::{ bloom::logs_bloom, Account, Address, Bloom, Header, Log, Receipt, TransactionSignedEcRecovered, H256, U256, }; +use reth_provider::StateProvider; use revm::{ db::AccountState, Account as RevmAccount, AccountInfo, AnalysisKind, Bytecode, Database, EVM, }; @@ -414,17 +412,15 @@ mod tests { use crate::{config::SpecUpgrades, revm_wrap::State}; use reth_db::{ - kv::{test_utils, Env, EnvKind}, - mdbx::WriteMap, - }; - use reth_interfaces::{ - db::{Database, DbTx}, - provider::{AccountProvider, StateProvider}, + database::Database, + mdbx::{test_utils, Env, EnvKind, WriteMap}, + transaction::DbTx, }; use reth_primitives::{ hex_literal::hex, keccak256, Account, Address, BlockLocked, Bytes, StorageKey, H160, H256, U256, }; + use reth_provider::{AccountProvider, StateProvider}; use reth_rlp::Decodable; use super::*; diff --git a/crates/executor/src/revm_wrap.rs b/crates/executor/src/revm_wrap.rs index eb39d4221..3dc657563 100644 --- a/crates/executor/src/revm_wrap.rs +++ b/crates/executor/src/revm_wrap.rs @@ -1,8 +1,9 @@ -use reth_interfaces::{provider::StateProvider, Error}; +use reth_interfaces::Error; use reth_primitives::{ Account, Header, Transaction, TransactionKind, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, H160, H256, KECCAK_EMPTY, U256, }; +use reth_provider::StateProvider; use revm::{ db::{CacheDB, DatabaseRef}, BlockEnv, TransactTo, TxEnv, diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index 0a8d6415b..d23af22d0 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -7,7 +7,7 @@ repository = "https://github.com/paradigmxyz/reth" readme = "README.md" [dependencies] -reth-codecs = { path = "../codecs" } +reth-codecs = { path = "../storage/codecs" } reth-primitives = { path = "../primitives" } reth-rpc-types = { path = "../net/rpc-types" } async-trait = "0.1.57" @@ -32,7 +32,7 @@ secp256k1 = { version = "0.24.0", default-features = false, features = ["alloc", modular-bitfield = "0.11.2" [dev-dependencies] -reth-db = { path = "../db", features = ["test-utils"] } +reth-db = { path = "../storage/db", features = ["test-utils"] } test-fuzz = "3.0.4" tokio = { version = "1.21.2", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["sync"] } diff --git a/crates/interfaces/src/db/error.rs b/crates/interfaces/src/db.rs similarity index 94% rename from crates/interfaces/src/db/error.rs rename to crates/interfaces/src/db.rs index 4bc525b53..9d07343ff 100644 --- a/crates/interfaces/src/db/error.rs +++ b/crates/interfaces/src/db.rs @@ -1,4 +1,4 @@ -/// KV error type. They are using u32 to represent error code. +/// Database error type. They are using u32 to represent error code. #[derive(Debug, thiserror::Error, PartialEq, Eq, Clone)] pub enum Error { /// Failed to open database. diff --git a/crates/interfaces/src/db/mod.rs b/crates/interfaces/src/db/mod.rs deleted file mode 100644 index f2fe5783a..000000000 --- a/crates/interfaces/src/db/mod.rs +++ /dev/null @@ -1,289 +0,0 @@ -pub mod codecs; -mod error; -pub mod mock; -pub mod models; -mod table; -pub mod tables; - -use std::marker::PhantomData; - -pub use error::Error; -pub use table::*; - -// Sealed trait helper to prevent misuse of the API. -mod sealed { - pub trait Sealed: Sized {} - pub struct Bounds(T); - impl Sealed for Bounds {} -} -use sealed::{Bounds, Sealed}; - -/// Implements the GAT method from: -/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. -/// -/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers -pub trait DatabaseGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { - /// RO database transaction - type TX: DbTx<'a> + Send + Sync; - /// RW database transaction - type TXMut: DbTxMut<'a> + DbTx<'a> + Send + Sync; -} - -/// Main Database trait that spawns transactions to be executed. -pub trait Database: for<'a> DatabaseGAT<'a> { - /// Create read only transaction. - fn tx(&self) -> Result<>::TX, Error>; - - /// Create read write transaction only possible if database is open with write access. - fn tx_mut(&self) -> Result<>::TXMut, Error>; - - /// Takes a function and passes a read-only transaction into it, making sure it's closed in the - /// end of the execution. - fn view(&self, f: F) -> Result - where - F: Fn(&>::TX) -> T, - { - let tx = self.tx()?; - - let res = f(&tx); - tx.commit()?; - - Ok(res) - } - - /// Takes a function and passes a write-read transaction into it, making sure it's committed in - /// the end of the execution. - fn update(&self, f: F) -> Result - where - F: Fn(&>::TXMut) -> T, - { - let tx = self.tx_mut()?; - - let res = f(&tx); - tx.commit()?; - - Ok(res) - } -} - -/// Implements the GAT method from: -/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. -/// -/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers -pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { - /// Cursor GAT - type Cursor: DbCursorRO<'a, T> + Send + Sync; - /// DupCursor GAT - type DupCursor: DbDupCursorRO<'a, T> + DbCursorRO<'a, T> + Send + Sync; -} - -/// Implements the GAT method from: -/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. -/// -/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers -pub trait DbTxMutGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { - /// Cursor GAT - type CursorMut: DbCursorRW<'a, T> + DbCursorRO<'a, T> + Send + Sync; - /// DupCursor GAT - type DupCursorMut: DbDupCursorRW<'a, T> - + DbCursorRW<'a, T> - + DbDupCursorRO<'a, T> - + DbCursorRO<'a, T> - + Send - + Sync; -} - -/// Read only transaction -pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> { - /// Get value - fn get(&self, key: T::Key) -> Result, Error>; - /// Commit for read only transaction will consume and free transaction and allows - /// freeing of memory pages - fn commit(self) -> Result; - /// Iterate over read only values in table. - fn cursor(&self) -> Result<>::Cursor, Error>; - /// Iterate over read only values in dup sorted table. - fn cursor_dup(&self) -> Result<>::DupCursor, Error>; -} - -/// Read write transaction that allows writing to database -pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { - /// Put value to database - fn put(&self, key: T::Key, value: T::Value) -> Result<(), Error>; - /// Delete value from database - fn delete(&self, key: T::Key, value: Option) -> Result; - /// Clears database. - fn clear(&self) -> Result<(), Error>; - /// Cursor mut - fn cursor_mut(&self) -> Result<>::CursorMut, Error>; - /// DupCursor mut. - fn cursor_dup_mut( - &self, - ) -> Result<>::DupCursorMut, Error>; -} - -/// Alias type for a `(key, value)` result coming from a cursor. -pub type PairResult = Result::Key, ::Value)>, Error>; -/// Alias type for a `(key, value)` result coming from an iterator. -pub type IterPairResult = Option::Key, ::Value), Error>>; -/// Alias type for a value result coming from a cursor without its key. -pub type ValueOnlyResult = Result::Value>, Error>; - -/// Read only cursor over table. -pub trait DbCursorRO<'tx, T: Table> { - /// First item in table - fn first(&mut self) -> PairResult; - - /// Seeks for the exact `(key, value)` pair with `key`. - fn seek_exact(&mut self, key: T::Key) -> PairResult; - - /// Returns the next `(key, value)` pair. - #[allow(clippy::should_implement_trait)] - fn next(&mut self) -> PairResult; - - /// Returns the previous `(key, value)` pair. - fn prev(&mut self) -> PairResult; - - /// Returns the last `(key, value)` pair. - fn last(&mut self) -> PairResult; - - /// Returns the current `(key, value)` pair of the cursor. - fn current(&mut self) -> PairResult; - - /// Returns an iterator starting at a key greater or equal than `start_key`. - fn walk<'cursor>( - &'cursor mut self, - start_key: T::Key, - ) -> Result, Error> - where - Self: Sized; -} - -/// Read only curor over DupSort table. -pub trait DbDupCursorRO<'tx, T: DupSort> { - /// Seeks for a `(key, value)` pair greater or equal than `key`. - fn seek(&mut self, key: T::SubKey) -> PairResult; - - /// Returns the next `(key, value)` pair of a DUPSORT table. - fn next_dup(&mut self) -> PairResult; - - /// Returns the next `(key, value)` pair skipping the duplicates. - fn next_no_dup(&mut self) -> PairResult; - - /// Returns the next `value` of a duplicate `key`. - fn next_dup_val(&mut self) -> ValueOnlyResult; - - /// Returns an iterator starting at a key greater or equal than `start_key` of a DUPSORT - /// table. - fn walk_dup<'cursor>( - &'cursor mut self, - key: T::Key, - subkey: T::SubKey, - ) -> Result, Error> - where - Self: Sized; -} - -/// Read write cursor over table. -pub trait DbCursorRW<'tx, T: Table> { - /// Database operation that will update an existing row if a specified value already - /// exists in a table, and insert a new row if the specified value doesn't already exist - fn upsert(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; - - /// Database operation that will insert a row at a given key. If the key is already - /// present, the operation will result in an error. - fn insert(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; - - /// Append value to next cursor item. - /// - /// This is efficient for pre-sorted data. If the data is not pre-sorted, use [`insert`]. - fn append(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; - - /// Delete current value that cursor points to - fn delete_current(&mut self) -> Result<(), Error>; -} - -/// Read Write Cursor over DupSorted table. -pub trait DbDupCursorRW<'tx, T: DupSort> { - /// Append value to next cursor item - fn delete_current_duplicates(&mut self) -> Result<(), Error>; - - /// Append duplicate value. - /// - /// This is efficient for pre-sorted data. If the data is not pre-sorted, use [`insert`]. - fn append_dup(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; -} - -/// Provides an iterator to `Cursor` when handling `Table`. -/// -/// Reason why we have two lifetimes is to distinguish between `'cursor` lifetime -/// and inherited `'tx` lifetime. If there is only one, rust would short circle -/// the Cursor lifetime and it wouldn't be possible to use Walker. -pub struct Walker<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> { - /// Cursor to be used to walk through the table. - pub cursor: &'cursor mut CURSOR, - /// `(key, value)` where to start the walk. - pub start: IterPairResult, - /// Phantom data for 'tx. As it is only used for `DbCursorRO`. - pub _tx_phantom: PhantomData<&'tx T>, -} - -impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator - for Walker<'cursor, 'tx, T, CURSOR> -{ - type Item = Result<(T::Key, T::Value), Error>; - fn next(&mut self) -> Option { - let start = self.start.take(); - if start.is_some() { - return start - } - - self.cursor.next().transpose() - } -} - -/// Provides an iterator to `Cursor` when handling a `DupSort` table. -/// -/// Reason why we have two lifetimes is to distinguish between `'cursor` lifetime -/// and inherited `'tx` lifetime. If there is only one, rust would short circle -/// the Cursor lifetime and it wouldn't be possible to use Walker. -pub struct DupWalker<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> { - /// Cursor to be used to walk through the table. - pub cursor: &'cursor mut CURSOR, - /// Value where to start the walk. - pub start: IterPairResult, - /// Phantom data for 'tx. As it is only used for `DbDupCursorRO`. - pub _tx_phantom: PhantomData<&'tx T>, -} - -impl<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> std::iter::Iterator - for DupWalker<'cursor, 'tx, T, CURSOR> -{ - type Item = Result<(T::Key, T::Value), Error>; - fn next(&mut self) -> Option { - let start = self.start.take(); - if start.is_some() { - return start - } - self.cursor.next_dup().transpose() - } -} - -#[macro_export] -/// Implements the [`arbitrary::Arbitrary`] trait for types with fixed array types. -macro_rules! impl_fixed_arbitrary { - ($name:tt, $size:tt) => { - #[cfg(any(test, feature = "arbitrary"))] - use arbitrary::{Arbitrary, Unstructured}; - - #[cfg(any(test, feature = "arbitrary"))] - impl<'a> Arbitrary<'a> for $name { - fn arbitrary(u: &mut Unstructured<'a>) -> Result { - let mut buffer = vec![0; $size]; - u.fill_buffer(buffer.as_mut_slice())?; - - Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat) - } - } - }; -} diff --git a/crates/interfaces/src/lib.rs b/crates/interfaces/src/lib.rs index d21fbc074..0ec6c0001 100644 --- a/crates/interfaces/src/lib.rs +++ b/crates/interfaces/src/lib.rs @@ -13,11 +13,12 @@ pub mod executor; /// Consensus traits. pub mod consensus; -/// Database traits. -pub mod db; -/// Traits that provide chain access. +/// Provider error pub mod provider; +/// Database error +pub mod db; + /// P2P traits. pub mod p2p; diff --git a/crates/interfaces/src/provider/error.rs b/crates/interfaces/src/provider.rs similarity index 80% rename from crates/interfaces/src/provider/error.rs rename to crates/interfaces/src/provider.rs index 99a5167ad..5ae00420e 100644 --- a/crates/interfaces/src/provider/error.rs +++ b/crates/interfaces/src/provider.rs @@ -1,7 +1,5 @@ use reth_primitives::{BlockHash, BlockNumber}; -use crate::db::models::BlockNumHash; - /// KV error type. They are using u32 to represent error code. #[allow(missing_docs)] #[derive(Debug, thiserror::Error, PartialEq, Eq, Clone)] @@ -12,6 +10,6 @@ pub enum Error { BlockTxNumberNotExists { block_hash: BlockHash }, #[error("Block hash {block_hash:?} does not exists in Headers table")] BlockHashNotExist { block_hash: BlockHash }, - #[error("Block body not exists {block_num_hash:?}")] - BlockBodyNotExist { block_num_hash: BlockNumHash }, + #[error("Block body not exists #{block_number} {block_hash}")] + BlockBodyNotExist { block_number: BlockNumber, block_hash: BlockHash }, } diff --git a/crates/interfaces/src/provider/mod.rs b/crates/interfaces/src/provider/mod.rs deleted file mode 100644 index 27d46fc28..000000000 --- a/crates/interfaces/src/provider/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod block; -pub mod db_provider; -mod error; -mod state; - -pub use block::{ - get_cumulative_tx_count_by_hash, insert_canonical_block, BlockProvider, ChainInfo, - HeaderProvider, -}; -pub use db_provider::{self as db, ProviderImpl}; -pub use error::Error; -pub use state::{AccountProvider, StateProvider, StateProviderFactory}; diff --git a/crates/interfaces/src/test_utils/mod.rs b/crates/interfaces/src/test_utils/mod.rs index 415fccb74..79a68b77b 100644 --- a/crates/interfaces/src/test_utils/mod.rs +++ b/crates/interfaces/src/test_utils/mod.rs @@ -1,10 +1,8 @@ -mod api; mod bodies; mod headers; /// Generators for different data structures like block headers, block bodies and ranges of those. pub mod generators; -pub use api::TestApi; pub use bodies::*; pub use headers::*; diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 47989fda9..69dac7634 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -20,6 +20,7 @@ reth-rlp = { path = "../../common/rlp" } reth-rlp-derive = { path = "../../common/rlp-derive" } reth-tasks = { path = "../../tasks" } reth-transaction-pool = { path = "../../transaction-pool" } +reth-provider = { path = "../../storage/provider"} # async/futures futures = "0.3" @@ -47,6 +48,7 @@ secp256k1 = { version = "0.24", features = [ [dev-dependencies] # reth reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } +reth-provider = { path = "../../storage/provider", features = ["test-utils"] } reth-tracing = { path = "../../tracing" } rand = "0.8" diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 8adec9bb8..56ad31dc3 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -6,11 +6,9 @@ use reth_eth_wire::{ BlockBodies, BlockBody, BlockHeaders, GetBlockBodies, GetBlockHeaders, GetNodeData, GetReceipts, NodeData, Receipts, }; -use reth_interfaces::{ - p2p::error::RequestResult, - provider::{BlockProvider, HeaderProvider}, -}; +use reth_interfaces::p2p::error::RequestResult; use reth_primitives::{BlockHashOrNumber, Header, HeadersDirection, PeerId}; +use reth_provider::{BlockProvider, HeaderProvider}; use std::{ borrow::Borrow, future::Future, diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index c5400af97..dbfa1b43b 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -37,8 +37,8 @@ use reth_eth_wire::{ capability::{Capabilities, CapabilityMessage}, DisconnectReason, }; -use reth_interfaces::provider::BlockProvider; use reth_primitives::{PeerId, H256}; +use reth_provider::BlockProvider; use std::{ net::SocketAddr, pin::Pin, diff --git a/crates/net/network/src/state.rs b/crates/net/network/src/state.rs index a2acc65e8..7f14bba9e 100644 --- a/crates/net/network/src/state.rs +++ b/crates/net/network/src/state.rs @@ -14,8 +14,8 @@ use crate::{ use reth_eth_wire::{ capability::Capabilities, BlockHashNumber, DisconnectReason, NewBlockHashes, Status, }; -use reth_interfaces::provider::BlockProvider; use reth_primitives::{PeerId, H256}; +use reth_provider::BlockProvider; use std::{ collections::{HashMap, VecDeque}, net::SocketAddr, diff --git a/crates/net/network/src/swarm.rs b/crates/net/network/src/swarm.rs index 2ee2d52ba..ff13e55e1 100644 --- a/crates/net/network/src/swarm.rs +++ b/crates/net/network/src/swarm.rs @@ -10,8 +10,8 @@ use reth_eth_wire::{ capability::{Capabilities, CapabilityMessage}, error::EthStreamError, }; -use reth_interfaces::provider::BlockProvider; use reth_primitives::PeerId; +use reth_provider::BlockProvider; use std::{ io, net::SocketAddr, diff --git a/crates/net/network/tests/it/connect.rs b/crates/net/network/tests/it/connect.rs index 775d39c08..ad4b1ed99 100644 --- a/crates/net/network/tests/it/connect.rs +++ b/crates/net/network/tests/it/connect.rs @@ -3,8 +3,8 @@ use super::testnet::Testnet; use futures::StreamExt; use reth_discv4::{bootnodes::mainnet_nodes, Discv4Config}; -use reth_interfaces::test_utils::TestApi; use reth_network::{NetworkConfig, NetworkEvent, NetworkManager}; +use reth_provider::test_utils::TestApi; use secp256k1::SecretKey; use std::{collections::HashSet, sync::Arc}; diff --git a/crates/net/network/tests/it/testnet.rs b/crates/net/network/tests/it/testnet.rs index 7797def48..fc21a8363 100644 --- a/crates/net/network/tests/it/testnet.rs +++ b/crates/net/network/tests/it/testnet.rs @@ -3,10 +3,6 @@ use futures::{FutureExt, StreamExt}; use parking_lot::Mutex; use pin_project::pin_project; -use reth_interfaces::{ - provider::{BlockProvider, ChainInfo, HeaderProvider}, - test_utils::TestApi, -}; use reth_network::{ error::NetworkError, eth_requests::EthRequestHandler, NetworkConfig, NetworkEvent, NetworkHandle, NetworkManager, @@ -15,6 +11,7 @@ use reth_primitives::{ rpc::{BlockId, BlockNumber}, Block, BlockHash, Header, PeerId, H256, U256, }; +use reth_provider::{test_utils::TestApi, BlockProvider, ChainInfo, HeaderProvider}; use secp256k1::SecretKey; use std::{ collections::HashMap, diff --git a/crates/net/rpc/Cargo.toml b/crates/net/rpc/Cargo.toml index 28606969d..25c90838e 100644 --- a/crates/net/rpc/Cargo.toml +++ b/crates/net/rpc/Cargo.toml @@ -14,6 +14,7 @@ reth-interfaces = { path = "../../interfaces" } reth-primitives = { path = "../../primitives" } reth-rpc-api = { path = "../rpc-api" } reth-rpc-types = { path = "../rpc-types" } +reth-provider = { path = "../../storage/provider"} reth-transaction-pool = { path = "../../transaction-pool" } # rpc diff --git a/crates/net/rpc/src/eth/api/mod.rs b/crates/net/rpc/src/eth/api/mod.rs index 416004ec0..33628ab3e 100644 --- a/crates/net/rpc/src/eth/api/mod.rs +++ b/crates/net/rpc/src/eth/api/mod.rs @@ -1,10 +1,8 @@ //! Provides everything related to `eth_` namespace -use reth_interfaces::{ - provider::{BlockProvider, StateProviderFactory}, - Result, -}; +use reth_interfaces::Result; use reth_primitives::{Transaction, U256, U64}; +use reth_provider::{BlockProvider, StateProviderFactory}; use reth_transaction_pool::TransactionPool; use std::sync::Arc; diff --git a/crates/net/rpc/src/eth/api/server.rs b/crates/net/rpc/src/eth/api/server.rs index 0c9ea39bf..fd33c4ee4 100644 --- a/crates/net/rpc/src/eth/api/server.rs +++ b/crates/net/rpc/src/eth/api/server.rs @@ -3,11 +3,11 @@ use crate::{eth::api::EthApi, result::ToRpcResult}; use jsonrpsee::core::RpcResult as Result; -use reth_interfaces::provider::{BlockProvider, StateProviderFactory}; use reth_primitives::{ rpc::{transaction::eip2930::AccessListWithGasUsed, BlockId}, Address, BlockNumber, Bytes, Transaction, H256, H64, U256, U64, }; +use reth_provider::{BlockProvider, StateProviderFactory}; use reth_rpc_api::EthApiServer; use reth_rpc_types::{ CallRequest, EIP1186AccountProofResponse, FeeHistory, Index, RichBlock, SyncStatus, diff --git a/crates/net/rpc/src/eth/pubsub.rs b/crates/net/rpc/src/eth/pubsub.rs index 8bb3d5ac7..afdfbfab2 100644 --- a/crates/net/rpc/src/eth/pubsub.rs +++ b/crates/net/rpc/src/eth/pubsub.rs @@ -1,7 +1,7 @@ //! `eth_` PubSub RPC handler implementation use jsonrpsee::{types::SubscriptionResult, SubscriptionSink}; -use reth_interfaces::provider::BlockProvider; +use reth_provider::BlockProvider; use reth_rpc_api::EthPubSubApiServer; use reth_rpc_types::pubsub::{Kind, Params}; use reth_transaction_pool::TransactionPool; diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 5a85d8daf..5d50524c0 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -14,7 +14,7 @@ reth-rlp = { path = "../common/rlp", features = [ "derive", "ethereum-types", ] } -reth-codecs = { version = "0.1.0", path = "../codecs" } +reth-codecs = { version = "0.1.0", path = "../storage/codecs" } # ethereum ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false } diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 080f5c449..3882051f6 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -11,9 +11,10 @@ description = "Staged syncing primitives used in reth." # reth libs reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } -reth-db = { path = "../db" } reth-executor = { path = "../executor" } reth-rlp = { path = "../common/rlp" } +reth-db = { path = "../storage/db" } +reth-provider = { path = "../storage/provider"} #async tokio = { version = "1.21.2", features = ["sync"] } @@ -29,9 +30,11 @@ itertools = "0.10.5" rayon = "1.6.0" [dev-dependencies] -reth-db = { path = "../db", features = ["test-utils"] } +# reth +reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] } reth-interfaces = { path = "../interfaces", features = ["test-utils"] } reth-bodies-downloaders = { path = "../net/bodies-downloaders" } + # TODO(onbjerg): We only need this for [BlockBody] reth-eth-wire = { path = "../net/eth-wire" } reth-headers-downloaders = { path = "../net/headers-downloaders" } diff --git a/crates/stages/src/db.rs b/crates/stages/src/db.rs index 9f2fa4897..bb278b947 100644 --- a/crates/stages/src/db.rs +++ b/crates/stages/src/db.rs @@ -3,9 +3,14 @@ use std::{ ops::{Deref, DerefMut}, }; -use reth_interfaces::db::{ +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW}, + database::{Database, DatabaseGAT}, models::{BlockNumHash, NumTransactions}, - tables, Database, DatabaseGAT, DbCursorRO, DbCursorRW, DbTx, DbTxMut, Error, Table, + table::Table, + tables, + transaction::{DbTx, DbTxMut}, + Error, }; use reth_primitives::{BlockHash, BlockNumber, TxNumber}; diff --git a/crates/stages/src/id.rs b/crates/stages/src/id.rs index 818962715..3c0ec3484 100644 --- a/crates/stages/src/id.rs +++ b/crates/stages/src/id.rs @@ -1,5 +1,9 @@ use metrics::counter; -use reth_interfaces::db::{tables::SyncStage, DbTx, DbTxMut, Error as DbError}; +use reth_db::{ + tables::SyncStage, + transaction::{DbTx, DbTxMut}, + Error as DbError, +}; use reth_primitives::BlockNumber; use std::fmt::Display; diff --git a/crates/stages/src/pipeline.rs b/crates/stages/src/pipeline.rs index a020d4772..38f01e0cb 100644 --- a/crates/stages/src/pipeline.rs +++ b/crates/stages/src/pipeline.rs @@ -2,7 +2,7 @@ use crate::{ db::StageDB, error::*, util::opt::MaybeSender, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, }; -use reth_interfaces::db::{Database, DbTx}; +use reth_db::{database::Database, transaction::DbTx}; use reth_primitives::BlockNumber; use std::{ fmt::{Debug, Formatter}, @@ -365,10 +365,7 @@ impl QueuedStage { mod tests { use super::*; use crate::{StageId, UnwindOutput}; - use reth_db::{ - kv::{test_utils, Env, EnvKind}, - mdbx::{self, WriteMap}, - }; + use reth_db::mdbx::{self, test_utils, Env, EnvKind, WriteMap}; use reth_interfaces::consensus; use tokio::sync::mpsc::channel; use tokio_stream::{wrappers::ReceiverStream, StreamExt}; diff --git a/crates/stages/src/stage.rs b/crates/stages/src/stage.rs index 68bb1fdf9..bc455c740 100644 --- a/crates/stages/src/stage.rs +++ b/crates/stages/src/stage.rs @@ -1,6 +1,6 @@ use crate::{db::StageDB, error::StageError, id::StageId}; use async_trait::async_trait; -use reth_interfaces::db::Database; +use reth_db::database::Database; use reth_primitives::BlockNumber; /// Stage execution input, see [Stage::execute]. diff --git a/crates/stages/src/stages/bodies.rs b/crates/stages/src/stages/bodies.rs index b499e4f91..941a15603 100644 --- a/crates/stages/src/stages/bodies.rs +++ b/crates/stages/src/stages/bodies.rs @@ -3,14 +3,14 @@ use crate::{ UnwindInput, UnwindOutput, }; use futures_util::TryStreamExt; -use reth_interfaces::{ - consensus::Consensus, - db::{ - models::StoredBlockOmmers, tables, Database, DatabaseGAT, DbCursorRO, DbCursorRW, DbTx, - DbTxMut, - }, - p2p::bodies::downloader::BodyDownloader, +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW}, + database::{Database, DatabaseGAT}, + models::StoredBlockOmmers, + tables, + transaction::{DbTx, DbTxMut}, }; +use reth_interfaces::{consensus::Consensus, p2p::bodies::downloader::BodyDownloader}; use reth_primitives::{ proofs::{EMPTY_LIST_HASH, EMPTY_ROOT}, BlockLocked, BlockNumber, SealedHeader, H256, @@ -477,12 +477,14 @@ mod tests { ExecInput, ExecOutput, UnwindInput, }; use assert_matches::assert_matches; + use reth_db::{ + cursor::DbCursorRO, + models::{BlockNumHash, NumTransactions, StoredBlockOmmers}, + tables, + transaction::{DbTx, DbTxMut}, + }; use reth_eth_wire::BlockBody; use reth_interfaces::{ - db::{ - models::{BlockNumHash, NumTransactions, StoredBlockOmmers}, - tables, DbCursorRO, DbTx, DbTxMut, - }, p2p::{ bodies::{ client::BodiesClient, diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 5bd8d120d..eae4a8dbe 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -2,20 +2,21 @@ use crate::{ db::StageDB, DatabaseIntegrityError, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput, }; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + database::Database, + models::{BlockNumHash, TxNumberAddress}, + tables, + transaction::{DbTx, DbTxMut}, +}; use reth_executor::{ config::SpecUpgrades, executor::AccountChangeSet, revm_wrap::{State, SubState}, Config, }; -use reth_interfaces::{ - db::{ - models::{BlockNumHash, TxNumberAddress}, - tables, Database, DbCursorRO, DbCursorRW, DbDupCursorRO, DbTx, DbTxMut, - }, - provider::db::StateProviderImplRefLatest, -}; use reth_primitives::{Address, StorageEntry, TransactionSignedEcRecovered, H256, U256}; +use reth_provider::StateProviderImplRefLatest; use std::{fmt::Debug, ops::DerefMut}; const EXECUTION: StageId = StageId("Execution"); @@ -423,12 +424,9 @@ mod tests { use std::ops::Deref; use super::*; - use reth_db::{ - kv::{test_utils::create_test_db, EnvKind}, - mdbx::WriteMap, - }; - use reth_interfaces::provider::insert_canonical_block; + use reth_db::mdbx::{test_utils::create_test_db, EnvKind, WriteMap}; use reth_primitives::{hex_literal::hex, keccak256, Account, BlockLocked, H160, U256}; + use reth_provider::insert_canonical_block; use reth_rlp::Decodable; #[tokio::test] diff --git a/crates/stages/src/stages/headers.rs b/crates/stages/src/stages/headers.rs index e44a6dc31..9b5b0532d 100644 --- a/crates/stages/src/stages/headers.rs +++ b/crates/stages/src/stages/headers.rs @@ -3,9 +3,15 @@ use crate::{ UnwindInput, UnwindOutput, }; use futures_util::StreamExt; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW}, + database::Database, + models::blocks::BlockNumHash, + tables, + transaction::{DbTx, DbTxMut}, +}; use reth_interfaces::{ consensus::{Consensus, ForkchoiceState}, - db::{models::blocks::BlockNumHash, tables, Database, DbCursorRO, DbCursorRW, DbTx, DbTxMut}, p2p::headers::{ client::HeadersClient, downloader::{ensure_parent, HeaderDownloader}, @@ -342,9 +348,9 @@ mod tests { }, ExecInput, ExecOutput, UnwindInput, }; + use reth_db::{models::blocks::BlockNumHash, tables, transaction::DbTx}; use reth_headers_downloaders::linear::{LinearDownloadBuilder, LinearDownloader}; use reth_interfaces::{ - db::{models::blocks::BlockNumHash, tables, DbTx}, p2p::headers::downloader::HeaderDownloader, test_utils::{ generators::{random_header, random_header_range}, diff --git a/crates/stages/src/stages/senders.rs b/crates/stages/src/stages/senders.rs index 6a5003857..2e29a6186 100644 --- a/crates/stages/src/stages/senders.rs +++ b/crates/stages/src/stages/senders.rs @@ -3,7 +3,13 @@ use crate::{ }; use itertools::Itertools; use rayon::prelude::*; -use reth_interfaces::db::{self, tables, Database, DbCursorRO, DbCursorRW, DbTx, DbTxMut}; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW}, + database::Database, + tables, + transaction::{DbTx, DbTxMut}, + Error as DbError, +}; use reth_primitives::TxNumber; use std::fmt::Debug; use thiserror::Error; @@ -79,7 +85,7 @@ impl Stage for SendersStage { // Iterate over transactions in chunks for chunk in &entries.chunks(self.batch_size) { - let transactions = chunk.collect::, db::Error>>()?; + let transactions = chunk.collect::, DbError>>()?; // Recover signers for the chunk in parallel let recovered = transactions .into_par_iter() diff --git a/crates/stages/src/test_utils/runner.rs b/crates/stages/src/test_utils/runner.rs index dd55fbb5d..01723b6d4 100644 --- a/crates/stages/src/test_utils/runner.rs +++ b/crates/stages/src/test_utils/runner.rs @@ -1,4 +1,4 @@ -use reth_db::{kv::Env, mdbx::WriteMap}; +use reth_db::mdbx::{Env, WriteMap}; use std::borrow::Borrow; use tokio::sync::oneshot; diff --git a/crates/stages/src/test_utils/test_db.rs b/crates/stages/src/test_utils/test_db.rs index 3392cdb87..d3ec1424b 100644 --- a/crates/stages/src/test_utils/test_db.rs +++ b/crates/stages/src/test_utils/test_db.rs @@ -1,9 +1,11 @@ use reth_db::{ - kv::{test_utils::create_test_db, tx::Tx, Env, EnvKind}, - mdbx::{WriteMap, RW}, -}; -use reth_interfaces::db::{ - self, models::BlockNumHash, tables, DbCursorRO, DbCursorRW, DbTx, DbTxMut, Table, + cursor::{DbCursorRO, DbCursorRW}, + mdbx::{test_utils::create_test_db, tx::Tx, Env, EnvKind, WriteMap, RW}, + models::BlockNumHash, + table::Table, + tables, + transaction::{DbTx, DbTxMut}, + Error as DbError, }; use reth_primitives::{BlockNumber, SealedHeader, U256}; use std::{borrow::Borrow, sync::Arc}; @@ -40,9 +42,9 @@ impl TestStageDB { } /// Invoke a callback with transaction committing it afterwards - pub(crate) fn commit(&self, f: F) -> Result<(), db::Error> + pub(crate) fn commit(&self, f: F) -> Result<(), DbError> where - F: FnOnce(&mut Tx<'_, RW, WriteMap>) -> Result<(), db::Error>, + F: FnOnce(&mut Tx<'_, RW, WriteMap>) -> Result<(), DbError>, { let mut db = self.inner(); f(&mut db)?; @@ -51,15 +53,15 @@ impl TestStageDB { } /// Invoke a callback with a read transaction - pub(crate) fn query(&self, f: F) -> Result + pub(crate) fn query(&self, f: F) -> Result where - F: FnOnce(&Tx<'_, RW, WriteMap>) -> Result, + F: FnOnce(&Tx<'_, RW, WriteMap>) -> Result, { f(&self.inner()) } /// Check if the table is empty - pub(crate) fn table_is_empty(&self) -> Result { + pub(crate) fn table_is_empty(&self) -> Result { self.query(|tx| { let last = tx.cursor::()?.last()?; Ok(last.is_none()) @@ -74,7 +76,7 @@ impl TestStageDB { /// db.map_put::(&items, |item| item)?; /// ``` #[allow(dead_code)] - pub(crate) fn map_put(&self, values: &[S], mut map: F) -> Result<(), db::Error> + pub(crate) fn map_put(&self, values: &[S], mut map: F) -> Result<(), DbError> where T: Table, S: Clone, @@ -102,7 +104,7 @@ impl TestStageDB { &self, values: &[S], mut transform: F, - ) -> Result<(), db::Error> + ) -> Result<(), DbError> where T: Table, ::Value: Clone, @@ -126,7 +128,7 @@ impl TestStageDB { &self, num: u64, mut selector: F, - ) -> Result<(), db::Error> + ) -> Result<(), DbError> where T: Table, F: FnMut(T::Key) -> BlockNumber, @@ -146,7 +148,7 @@ impl TestStageDB { &self, num: u64, mut selector: F, - ) -> Result<(), db::Error> + ) -> Result<(), DbError> where T: Table, F: FnMut(T::Value) -> BlockNumber, @@ -162,7 +164,7 @@ impl TestStageDB { /// Insert ordered collection of [SealedHeader] into the corresponding tables /// that are supposed to be populated by the headers stage. - pub(crate) fn insert_headers<'a, I>(&self, headers: I) -> Result<(), db::Error> + pub(crate) fn insert_headers<'a, I>(&self, headers: I) -> Result<(), DbError> where I: Iterator, { diff --git a/crates/codecs/Cargo.toml b/crates/storage/codecs/Cargo.toml similarity index 100% rename from crates/codecs/Cargo.toml rename to crates/storage/codecs/Cargo.toml diff --git a/crates/codecs/README.md b/crates/storage/codecs/README.md similarity index 92% rename from crates/codecs/README.md rename to crates/storage/codecs/README.md index 7a5fc6028..8b7e979e3 100644 --- a/crates/codecs/README.md +++ b/crates/storage/codecs/README.md @@ -4,7 +4,7 @@ This crate allows to easily configure different codecs for different purposes (b Examples: -- [`Header` struct](../primitives/src/header.rs) +- [`Header` struct](../../primitives/src/header.rs) - [DB usage](../db/src/kv/codecs/scale.rs) ### Features diff --git a/crates/codecs/derive/Cargo.toml b/crates/storage/codecs/derive/Cargo.toml similarity index 100% rename from crates/codecs/derive/Cargo.toml rename to crates/storage/codecs/derive/Cargo.toml diff --git a/crates/codecs/derive/src/compact/enums.rs b/crates/storage/codecs/derive/src/compact/enums.rs similarity index 100% rename from crates/codecs/derive/src/compact/enums.rs rename to crates/storage/codecs/derive/src/compact/enums.rs diff --git a/crates/codecs/derive/src/compact/flags.rs b/crates/storage/codecs/derive/src/compact/flags.rs similarity index 100% rename from crates/codecs/derive/src/compact/flags.rs rename to crates/storage/codecs/derive/src/compact/flags.rs diff --git a/crates/codecs/derive/src/compact/generator.rs b/crates/storage/codecs/derive/src/compact/generator.rs similarity index 100% rename from crates/codecs/derive/src/compact/generator.rs rename to crates/storage/codecs/derive/src/compact/generator.rs diff --git a/crates/codecs/derive/src/compact/mod.rs b/crates/storage/codecs/derive/src/compact/mod.rs similarity index 100% rename from crates/codecs/derive/src/compact/mod.rs rename to crates/storage/codecs/derive/src/compact/mod.rs diff --git a/crates/codecs/derive/src/compact/structs.rs b/crates/storage/codecs/derive/src/compact/structs.rs similarity index 100% rename from crates/codecs/derive/src/compact/structs.rs rename to crates/storage/codecs/derive/src/compact/structs.rs diff --git a/crates/codecs/derive/src/lib.rs b/crates/storage/codecs/derive/src/lib.rs similarity index 100% rename from crates/codecs/derive/src/lib.rs rename to crates/storage/codecs/derive/src/lib.rs diff --git a/crates/codecs/src/lib.rs b/crates/storage/codecs/src/lib.rs similarity index 100% rename from crates/codecs/src/lib.rs rename to crates/storage/codecs/src/lib.rs diff --git a/crates/db/Cargo.toml b/crates/storage/db/Cargo.toml similarity index 55% rename from crates/db/Cargo.toml rename to crates/storage/db/Cargo.toml index 5b70f8ec9..eff42b2be 100644 --- a/crates/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -9,14 +9,22 @@ description = "Staged syncing primitives used in reth." [dependencies] # reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-libmdbx = { path = "../libmdbx-rs" } +reth-primitives = { path = "../../primitives" } +reth-interfaces = { path = "../../interfaces" } +reth-codecs = { path = "../codecs" } +reth-libmdbx = { path = "../libmdbx-rs", optional = true } # codecs serde = { version = "1.0.*", default-features = false } postcard = { version = "1.0.2", features = ["alloc"] } +heapless = "0.7.16" parity-scale-codec = { version = "3.2.1", features = ["bytes"] } +futures = "0.3.25" +tokio-stream = "0.1.11" +rand = "0.8.5" +arbitrary = { version = "1.1.7", features = ["derive"], optional = true } +secp256k1 = { version = "0.24.0", default-features = false, features = ["alloc", "recovery", "rand"], optional = true } +modular-bitfield = "0.11.2" # misc bytes = "1.2.1" @@ -25,19 +33,24 @@ thiserror = "1.0.37" tempfile = { version = "3.3.0", optional = true } eyre = "0.6.8" + [dev-dependencies] tempfile = "3.3.0" test-fuzz = "3.0.4" criterion = "0.4.0" iai = "0.1.1" tokio = { version = "1.21.2", features = ["full"] } +arbitrary = { version = "1.1.7", features = ["derive"]} +reth-db = { path = ".", features = ["test-utils","bench"]} -reth-interfaces = { path = "../interfaces",features=["bench"] } +reth-interfaces = { path = "../../interfaces",features=["bench"] } async-trait = "0.1.58" [features] +default = ["mdbx"] test-utils = ["tempfile"] bench-postcard = ["bench"] +mdbx = ["reth-libmdbx"] bench = [] [[bench]] diff --git a/crates/db/benches/README.md b/crates/storage/db/benches/README.md similarity index 100% rename from crates/db/benches/README.md rename to crates/storage/db/benches/README.md diff --git a/crates/db/benches/encoding_crit.rs b/crates/storage/db/benches/encoding_crit.rs similarity index 82% rename from crates/db/benches/encoding_crit.rs rename to crates/storage/db/benches/encoding_crit.rs index 76588291b..a3a8e653d 100644 --- a/crates/db/benches/encoding_crit.rs +++ b/crates/storage/db/benches/encoding_crit.rs @@ -8,9 +8,9 @@ macro_rules! impl_criterion_encoding_benchmark { c.bench_function(stringify!($name), |b| { b.iter(|| { let encoded_size = - reth_interfaces::db::codecs::fuzz::IntegerList::encode_and_decode( - black_box(reth_primitives::IntegerList::default()), - ) + reth_db::tables::codecs::fuzz::IntegerList::encode_and_decode(black_box( + reth_primitives::IntegerList::default(), + )) .0; if size == 0 { diff --git a/crates/db/benches/encoding_iai.rs b/crates/storage/db/benches/encoding_iai.rs similarity index 82% rename from crates/db/benches/encoding_iai.rs rename to crates/storage/db/benches/encoding_iai.rs index cfa30ff78..163061cef 100644 --- a/crates/db/benches/encoding_iai.rs +++ b/crates/storage/db/benches/encoding_iai.rs @@ -1,11 +1,11 @@ use iai::{black_box, main}; -use reth_interfaces::db; +use reth_db::tables::codecs; /// Benchmarks the encoding and decoding of `Header` using iai. macro_rules! impl_iai_encoding_benchmark { ($name:tt) => { fn $name() { - db::codecs::fuzz::IntegerList::encode_and_decode(black_box( + codecs::fuzz::IntegerList::encode_and_decode(black_box( reth_primitives::IntegerList::default(), )); } diff --git a/crates/storage/db/src/abstraction/common.rs b/crates/storage/db/src/abstraction/common.rs new file mode 100644 index 000000000..2e990e99e --- /dev/null +++ b/crates/storage/db/src/abstraction/common.rs @@ -0,0 +1,16 @@ +/// Alias type for a `(key, value)` result coming from a cursor. +pub type PairResult = Result::Key, ::Value)>, Error>; +/// Alias type for a `(key, value)` result coming from an iterator. +pub type IterPairResult = Option::Key, ::Value), Error>>; +/// Alias type for a value result coming from a cursor without its key. +pub type ValueOnlyResult = Result::Value>, Error>; + +use crate::{abstraction::table::*, Error}; + +// Sealed trait helper to prevent misuse of the API. +mod sealed { + pub trait Sealed: Sized {} + pub struct Bounds(T); + impl Sealed for Bounds {} +} +pub(crate) use sealed::{Bounds, Sealed}; diff --git a/crates/storage/db/src/abstraction/cursor.rs b/crates/storage/db/src/abstraction/cursor.rs new file mode 100644 index 000000000..a0a6a1947 --- /dev/null +++ b/crates/storage/db/src/abstraction/cursor.rs @@ -0,0 +1,147 @@ +use std::marker::PhantomData; + +use crate::{ + common::{IterPairResult, PairResult, ValueOnlyResult}, + table::{DupSort, Table}, + Error, +}; + +/// Read only cursor over table. +pub trait DbCursorRO<'tx, T: Table> { + /// First item in table + fn first(&mut self) -> PairResult; + + /// Seeks for the exact `(key, value)` pair with `key`. + fn seek_exact(&mut self, key: T::Key) -> PairResult; + + /// Returns the next `(key, value)` pair. + #[allow(clippy::should_implement_trait)] + fn next(&mut self) -> PairResult; + + /// Returns the previous `(key, value)` pair. + fn prev(&mut self) -> PairResult; + + /// Returns the last `(key, value)` pair. + fn last(&mut self) -> PairResult; + + /// Returns the current `(key, value)` pair of the cursor. + fn current(&mut self) -> PairResult; + + /// Returns an iterator starting at a key greater or equal than `start_key`. + fn walk<'cursor>( + &'cursor mut self, + start_key: T::Key, + ) -> Result, Error> + where + Self: Sized; +} + +/// Read only curor over DupSort table. +pub trait DbDupCursorRO<'tx, T: DupSort> { + /// Seeks for a `(key, value)` pair greater or equal than `key`. + fn seek(&mut self, key: T::SubKey) -> PairResult; + + /// Returns the next `(key, value)` pair of a DUPSORT table. + fn next_dup(&mut self) -> PairResult; + + /// Returns the next `(key, value)` pair skipping the duplicates. + fn next_no_dup(&mut self) -> PairResult; + + /// Returns the next `value` of a duplicate `key`. + fn next_dup_val(&mut self) -> ValueOnlyResult; + + /// Returns an iterator starting at a key greater or equal than `start_key` of a DUPSORT + /// table. + fn walk_dup<'cursor>( + &'cursor mut self, + key: T::Key, + subkey: T::SubKey, + ) -> Result, Error> + where + Self: Sized; +} + +/// Read write cursor over table. +pub trait DbCursorRW<'tx, T: Table> { + /// Database operation that will update an existing row if a specified value already + /// exists in a table, and insert a new row if the specified value doesn't already exist + fn upsert(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; + + /// Database operation that will insert a row at a given key. If the key is already + /// present, the operation will result in an error. + fn insert(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; + + /// Append value to next cursor item. + /// + /// This is efficient for pre-sorted data. If the data is not pre-sorted, use [`insert`]. + fn append(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; + + /// Delete current value that cursor points to + fn delete_current(&mut self) -> Result<(), Error>; +} + +/// Read Write Cursor over DupSorted table. +pub trait DbDupCursorRW<'tx, T: DupSort> { + /// Append value to next cursor item + fn delete_current_duplicates(&mut self) -> Result<(), Error>; + + /// Append duplicate value. + /// + /// This is efficient for pre-sorted data. If the data is not pre-sorted, use [`insert`]. + fn append_dup(&mut self, key: T::Key, value: T::Value) -> Result<(), Error>; +} + +/// Provides an iterator to `Cursor` when handling `Table`. +/// +/// Reason why we have two lifetimes is to distinguish between `'cursor` lifetime +/// and inherited `'tx` lifetime. If there is only one, rust would short circle +/// the Cursor lifetime and it wouldn't be possible to use Walker. +pub struct Walker<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> { + /// Cursor to be used to walk through the table. + pub cursor: &'cursor mut CURSOR, + /// `(key, value)` where to start the walk. + pub start: IterPairResult, + /// Phantom data for 'tx. As it is only used for `DbCursorRO`. + pub _tx_phantom: PhantomData<&'tx T>, +} + +impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator + for Walker<'cursor, 'tx, T, CURSOR> +{ + type Item = Result<(T::Key, T::Value), Error>; + fn next(&mut self) -> Option { + let start = self.start.take(); + if start.is_some() { + return start + } + + self.cursor.next().transpose() + } +} + +/// Provides an iterator to `Cursor` when handling a `DupSort` table. +/// +/// Reason why we have two lifetimes is to distinguish between `'cursor` lifetime +/// and inherited `'tx` lifetime. If there is only one, rust would short circle +/// the Cursor lifetime and it wouldn't be possible to use Walker. +pub struct DupWalker<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> { + /// Cursor to be used to walk through the table. + pub cursor: &'cursor mut CURSOR, + /// Value where to start the walk. + pub start: IterPairResult, + /// Phantom data for 'tx. As it is only used for `DbDupCursorRO`. + pub _tx_phantom: PhantomData<&'tx T>, +} + +impl<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> std::iter::Iterator + for DupWalker<'cursor, 'tx, T, CURSOR> +{ + type Item = Result<(T::Key, T::Value), Error>; + fn next(&mut self) -> Option { + let start = self.start.take(); + if start.is_some() { + return start + } + self.cursor.next_dup().transpose() + } +} diff --git a/crates/storage/db/src/abstraction/database.rs b/crates/storage/db/src/abstraction/database.rs new file mode 100644 index 000000000..89a602de8 --- /dev/null +++ b/crates/storage/db/src/abstraction/database.rs @@ -0,0 +1,53 @@ +use crate::{ + common::{Bounds, Sealed}, + transaction::{DbTx, DbTxMut}, + Error, +}; + +/// Implements the GAT method from: +/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. +/// +/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers +pub trait DatabaseGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { + /// RO database transaction + type TX: DbTx<'a> + Send + Sync; + /// RW database transaction + type TXMut: DbTxMut<'a> + DbTx<'a> + Send + Sync; +} + +/// Main Database trait that spawns transactions to be executed. +pub trait Database: for<'a> DatabaseGAT<'a> { + /// Create read only transaction. + fn tx(&self) -> Result<>::TX, Error>; + + /// Create read write transaction only possible if database is open with write access. + fn tx_mut(&self) -> Result<>::TXMut, Error>; + + /// Takes a function and passes a read-only transaction into it, making sure it's closed in the + /// end of the execution. + fn view(&self, f: F) -> Result + where + F: Fn(&>::TX) -> T, + { + let tx = self.tx()?; + + let res = f(&tx); + tx.commit()?; + + Ok(res) + } + + /// Takes a function and passes a write-read transaction into it, making sure it's committed in + /// the end of the execution. + fn update(&self, f: F) -> Result + where + F: Fn(&>::TXMut) -> T, + { + let tx = self.tx_mut()?; + + let res = f(&tx); + tx.commit()?; + + Ok(res) + } +} diff --git a/crates/interfaces/src/db/mock.rs b/crates/storage/db/src/abstraction/mock.rs similarity index 57% rename from crates/interfaces/src/db/mock.rs rename to crates/storage/db/src/abstraction/mock.rs index e25a3176b..e07a71eee 100644 --- a/crates/interfaces/src/db/mock.rs +++ b/crates/storage/db/src/abstraction/mock.rs @@ -1,9 +1,13 @@ //! Mock database use std::collections::BTreeMap; -use super::{ - Database, DatabaseGAT, DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DbTx, DbTxGAT, - DbTxMut, DbTxMutGAT, DupSort, DupWalker, Error, Table, Walker, +use crate::{ + common::{PairResult, ValueOnlyResult}, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, Walker}, + database::{Database, DatabaseGAT}, + table::{DupSort, Table}, + transaction::{DbTx, DbTxGAT, DbTxMut, DbTxMutGAT}, + Error, }; /// Mock database used for testing with inner BTreeMap structure @@ -15,11 +19,11 @@ pub struct DatabaseMock { } impl Database for DatabaseMock { - fn tx(&self) -> Result<>::TX, super::Error> { + fn tx(&self) -> Result<>::TX, Error> { Ok(TxMock::default()) } - fn tx_mut(&self) -> Result<>::TXMut, super::Error> { + fn tx_mut(&self) -> Result<>::TXMut, Error> { Ok(TxMock::default()) } } @@ -48,51 +52,43 @@ impl<'a> DbTxMutGAT<'a> for TxMock { } impl<'a> DbTx<'a> for TxMock { - fn get(&self, _key: T::Key) -> Result, super::Error> { + fn get(&self, _key: T::Key) -> Result, Error> { todo!() } - fn commit(self) -> Result { + fn commit(self) -> Result { todo!() } - fn cursor(&self) -> Result<>::Cursor, super::Error> { + fn cursor(&self) -> Result<>::Cursor, Error> { todo!() } - fn cursor_dup( - &self, - ) -> Result<>::DupCursor, super::Error> { + fn cursor_dup(&self) -> Result<>::DupCursor, Error> { todo!() } } impl<'a> DbTxMut<'a> for TxMock { - fn put(&self, _key: T::Key, _value: T::Value) -> Result<(), super::Error> { + fn put(&self, _key: T::Key, _value: T::Value) -> Result<(), Error> { todo!() } - fn delete( + fn delete(&self, _key: T::Key, _value: Option) -> Result { + todo!() + } + + fn cursor_mut(&self) -> Result<>::CursorMut, Error> { + todo!() + } + + fn cursor_dup_mut( &self, - _key: T::Key, - _value: Option, - ) -> Result { + ) -> Result<>::DupCursorMut, Error> { todo!() } - fn cursor_mut( - &self, - ) -> Result<>::CursorMut, super::Error> { - todo!() - } - - fn cursor_dup_mut( - &self, - ) -> Result<>::DupCursorMut, super::Error> { - todo!() - } - - fn clear(&self) -> Result<(), super::Error> { + fn clear(&self) -> Result<(), Error> { todo!() } } @@ -103,27 +99,27 @@ pub struct CursorMock { } impl<'tx, T: Table> DbCursorRO<'tx, T> for CursorMock { - fn first(&mut self) -> super::PairResult { + fn first(&mut self) -> PairResult { todo!() } - fn seek_exact(&mut self, _key: T::Key) -> super::PairResult { + fn seek_exact(&mut self, _key: T::Key) -> PairResult { todo!() } - fn next(&mut self) -> super::PairResult { + fn next(&mut self) -> PairResult { todo!() } - fn prev(&mut self) -> super::PairResult { + fn prev(&mut self) -> PairResult { todo!() } - fn last(&mut self) -> super::PairResult { + fn last(&mut self) -> PairResult { todo!() } - fn current(&mut self) -> super::PairResult { + fn current(&mut self) -> PairResult { todo!() } @@ -139,19 +135,19 @@ impl<'tx, T: Table> DbCursorRO<'tx, T> for CursorMock { } impl<'tx, T: DupSort> DbDupCursorRO<'tx, T> for CursorMock { - fn next_dup(&mut self) -> super::PairResult { + fn next_dup(&mut self) -> PairResult { todo!() } - fn seek(&mut self, _key: T::SubKey) -> super::PairResult { + fn seek(&mut self, _key: T::SubKey) -> PairResult { todo!() } - fn next_no_dup(&mut self) -> super::PairResult { + fn next_no_dup(&mut self) -> PairResult { todo!() } - fn next_dup_val(&mut self) -> super::ValueOnlyResult { + fn next_dup_val(&mut self) -> ValueOnlyResult { todo!() } @@ -172,7 +168,7 @@ impl<'tx, T: Table> DbCursorRW<'tx, T> for CursorMock { &mut self, _key: ::Key, _value: ::Value, - ) -> Result<(), super::Error> { + ) -> Result<(), Error> { todo!() } @@ -180,7 +176,7 @@ impl<'tx, T: Table> DbCursorRW<'tx, T> for CursorMock { &mut self, _key: ::Key, _value: ::Value, - ) -> Result<(), super::Error> { + ) -> Result<(), Error> { todo!() } @@ -188,21 +184,21 @@ impl<'tx, T: Table> DbCursorRW<'tx, T> for CursorMock { &mut self, _key: ::Key, _value: ::Value, - ) -> Result<(), super::Error> { + ) -> Result<(), Error> { todo!() } - fn delete_current(&mut self) -> Result<(), super::Error> { + fn delete_current(&mut self) -> Result<(), Error> { todo!() } } impl<'tx, T: DupSort> DbDupCursorRW<'tx, T> for CursorMock { - fn delete_current_duplicates(&mut self) -> Result<(), super::Error> { + fn delete_current_duplicates(&mut self) -> Result<(), Error> { todo!() } - fn append_dup(&mut self, _key: ::Key, _value: ::Value) -> Result<(), super::Error> { + fn append_dup(&mut self, _key: ::Key, _value: ::Value) -> Result<(), Error> { todo!() } } diff --git a/crates/storage/db/src/abstraction/mod.rs b/crates/storage/db/src/abstraction/mod.rs new file mode 100644 index 000000000..9acdba41f --- /dev/null +++ b/crates/storage/db/src/abstraction/mod.rs @@ -0,0 +1,12 @@ +/// Commont return types +pub mod common; +/// Cursor database traits. +pub mod cursor; +/// Database traits. +pub mod database; +/// mock +pub mod mock; +/// Table traits +pub mod table; +/// Transaction database traits. +pub mod transaction; diff --git a/crates/interfaces/src/db/table.rs b/crates/storage/db/src/abstraction/table.rs similarity index 99% rename from crates/interfaces/src/db/table.rs rename to crates/storage/db/src/abstraction/table.rs index 076380f42..deba47b5b 100644 --- a/crates/interfaces/src/db/table.rs +++ b/crates/storage/db/src/abstraction/table.rs @@ -1,4 +1,4 @@ -use super::Error; +use crate::Error; use bytes::Bytes; use std::{ fmt::Debug, diff --git a/crates/storage/db/src/abstraction/transaction.rs b/crates/storage/db/src/abstraction/transaction.rs new file mode 100644 index 000000000..d318f7eb2 --- /dev/null +++ b/crates/storage/db/src/abstraction/transaction.rs @@ -0,0 +1,62 @@ +use crate::{ + common::{Bounds, Sealed}, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + table::{DupSort, Table}, + Error, +}; + +/// Implements the GAT method from: +/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. +/// +/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers +pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { + /// Cursor GAT + type Cursor: DbCursorRO<'a, T> + Send + Sync; + /// DupCursor GAT + type DupCursor: DbDupCursorRO<'a, T> + DbCursorRO<'a, T> + Send + Sync; +} + +/// Implements the GAT method from: +/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. +/// +/// Sealed trait which cannot be implemented by 3rd parties, exposed only for implementers +pub trait DbTxMutGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { + /// Cursor GAT + type CursorMut: DbCursorRW<'a, T> + DbCursorRO<'a, T> + Send + Sync; + /// DupCursor GAT + type DupCursorMut: DbDupCursorRW<'a, T> + + DbCursorRW<'a, T> + + DbDupCursorRO<'a, T> + + DbCursorRO<'a, T> + + Send + + Sync; +} + +/// Read only transaction +pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> { + /// Get value + fn get(&self, key: T::Key) -> Result, Error>; + /// Commit for read only transaction will consume and free transaction and allows + /// freeing of memory pages + fn commit(self) -> Result; + /// Iterate over read only values in table. + fn cursor(&self) -> Result<>::Cursor, Error>; + /// Iterate over read only values in dup sorted table. + fn cursor_dup(&self) -> Result<>::DupCursor, Error>; +} + +/// Read write transaction that allows writing to database +pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { + /// Put value to database + fn put(&self, key: T::Key, value: T::Value) -> Result<(), Error>; + /// Delete value from database + fn delete(&self, key: T::Key, value: Option) -> Result; + /// Clears database. + fn clear(&self) -> Result<(), Error>; + /// Cursor mut + fn cursor_mut(&self) -> Result<>::CursorMut, Error>; + /// DupCursor mut. + fn cursor_dup_mut( + &self, + ) -> Result<>::DupCursorMut, Error>; +} diff --git a/crates/db/src/kv/cursor.rs b/crates/storage/db/src/implementation/mdbx/cursor.rs similarity index 88% rename from crates/db/src/kv/cursor.rs rename to crates/storage/db/src/implementation/mdbx/cursor.rs index 2cb4a3cf8..74a4a7479 100644 --- a/crates/db/src/kv/cursor.rs +++ b/crates/storage/db/src/implementation/mdbx/cursor.rs @@ -2,10 +2,11 @@ use std::{borrow::Cow, marker::PhantomData}; -use crate::utils::*; -use reth_interfaces::db::{ - Compress, DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupSort, DupWalker, Encode, - Error, Table, Walker, +use crate::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, Walker}, + table::{Compress, DupSort, Encode, Table}, + tables::utils::*, + Error, }; use reth_libmdbx::{self, TransactionKind, WriteFlags, RO, RW}; @@ -41,27 +42,27 @@ macro_rules! decode { } impl<'tx, K: TransactionKind, T: Table> DbCursorRO<'tx, T> for Cursor<'tx, K, T> { - fn first(&mut self) -> reth_interfaces::db::PairResult { + fn first(&mut self) -> PairResult { decode!(self.inner.first()) } - fn seek_exact(&mut self, key: ::Key) -> reth_interfaces::db::PairResult { + fn seek_exact(&mut self, key: ::Key) -> PairResult { decode!(self.inner.set_key(key.encode().as_ref())) } - fn next(&mut self) -> reth_interfaces::db::PairResult { + fn next(&mut self) -> PairResult { decode!(self.inner.next()) } - fn prev(&mut self) -> reth_interfaces::db::PairResult { + fn prev(&mut self) -> PairResult { decode!(self.inner.prev()) } - fn last(&mut self) -> reth_interfaces::db::PairResult { + fn last(&mut self) -> PairResult { decode!(self.inner.last()) } - fn current(&mut self) -> reth_interfaces::db::PairResult { + fn current(&mut self) -> PairResult { decode!(self.inner.get_current()) } @@ -83,7 +84,7 @@ impl<'tx, K: TransactionKind, T: Table> DbCursorRO<'tx, T> for Cursor<'tx, K, T> } impl<'tx, K: TransactionKind, T: DupSort> DbDupCursorRO<'tx, T> for Cursor<'tx, K, T> { - fn seek(&mut self, key: ::SubKey) -> reth_interfaces::db::PairResult { + fn seek(&mut self, key: ::SubKey) -> PairResult { decode!(self.inner.set_range(key.encode().as_ref())) } diff --git a/crates/db/src/kv/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs similarity index 94% rename from crates/db/src/kv/mod.rs rename to crates/storage/db/src/implementation/mdbx/mod.rs index cc49dff94..5f9704742 100644 --- a/crates/db/src/kv/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -1,9 +1,10 @@ //! Module that interacts with MDBX. -use crate::utils::default_page_size; -use reth_interfaces::db::{ +use crate::{ + database::{Database, DatabaseGAT}, tables::{TableType, TABLES}, - Database, DatabaseGAT, Error, + utils::default_page_size, + Error, }; use reth_libmdbx::{ DatabaseFlags, Environment, EnvironmentFlags, EnvironmentKind, Geometry, Mode, PageSize, @@ -138,16 +139,13 @@ pub mod test_utils { #[cfg(test)] mod tests { use super::{test_utils, Env, EnvKind}; - use reth_interfaces::{ - db::{ - self, - models::ShardedKey, - tables::{ - AccountHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState, - }, - Database, DbCursorRO, DbCursorRW, DbDupCursorRO, DbTx, DbTxMut, - }, - provider::{ProviderImpl, StateProviderFactory}, + use crate::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + database::Database, + models::ShardedKey, + tables::{AccountHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState}, + transaction::{DbTx, DbTxMut}, + Error, }; use reth_libmdbx::{NoWriteMap, WriteMap}; use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, H256, U256}; @@ -261,7 +259,7 @@ mod tests { assert_eq!(cursor.current(), Ok(Some((key_to_insert, H256::zero())))); // INSERT (failure) - assert_eq!(cursor.insert(key_to_insert, H256::zero()), Err(db::Error::Write(4294936497))); + assert_eq!(cursor.insert(key_to_insert, H256::zero()), Err(Error::Write(4294936497))); assert_eq!(cursor.current(), Ok(Some((key_to_insert, H256::zero())))); } @@ -282,7 +280,7 @@ mod tests { let tx = db.tx_mut().expect(ERROR_INIT_TX); let mut cursor = tx.cursor_mut::().unwrap(); cursor.seek_exact(1).unwrap(); - assert_eq!(cursor.append(key_to_append, H256::zero()), Err(db::Error::Write(4294936878))); + assert_eq!(cursor.append(key_to_append, H256::zero()), Err(Error::Write(4294936878))); assert_eq!(cursor.current(), Ok(Some((5, H256::zero())))); // the end of table } @@ -396,11 +394,4 @@ mod tests { assert_eq!(list200, list); } } - - #[test] - fn common_history_provider() { - let db = test_utils::create_test_db::(EnvKind::RW); - let provider = ProviderImpl::new(db); - let _ = provider.latest(); - } } diff --git a/crates/db/src/kv/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs similarity index 95% rename from crates/db/src/kv/tx.rs rename to crates/storage/db/src/implementation/mdbx/tx.rs index 5ac5a58a0..f7f825628 100644 --- a/crates/db/src/kv/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -1,8 +1,11 @@ //! Transaction wrapper for libmdbx-sys. -use crate::{kv::cursor::Cursor, utils::decode_one}; -use reth_interfaces::db::{ - Compress, DbTx, DbTxGAT, DbTxMut, DbTxMutGAT, DupSort, Encode, Error, Table, +use super::cursor::Cursor; +use crate::{ + table::{Compress, DupSort, Encode, Table}, + tables::utils::decode_one, + transaction::{DbTx, DbTxGAT, DbTxMut, DbTxMutGAT}, + Error, }; use reth_libmdbx::{EnvironmentKind, Transaction, TransactionKind, WriteFlags, RW}; use std::marker::PhantomData; diff --git a/crates/storage/db/src/implementation/mod.rs b/crates/storage/db/src/implementation/mod.rs new file mode 100644 index 000000000..7585c2385 --- /dev/null +++ b/crates/storage/db/src/implementation/mod.rs @@ -0,0 +1,2 @@ +#[cfg(feature = "mdbx")] +pub(crate) mod mdbx; diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs new file mode 100644 index 000000000..65e3b7c13 --- /dev/null +++ b/crates/storage/db/src/lib.rs @@ -0,0 +1,26 @@ +//! Rust database abstraction and concrete database implementations. + +#![warn(missing_docs, unreachable_pub)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] + +/// Abstracted part of database, containing traits for transactions and cursors. +pub mod abstraction; + +mod implementation; +pub mod tables; +mod utils; + +#[cfg(feature = "mdbx")] +/// Bindings for [MDBX](https://libmdbx.dqdkfa.ru/). +pub mod mdbx { + pub use crate::implementation::mdbx::*; + pub use reth_libmdbx::*; +} + +pub use abstraction::*; +pub use reth_interfaces::db::Error; +pub use tables::*; diff --git a/crates/interfaces/src/db/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs similarity index 95% rename from crates/interfaces/src/db/codecs/compact.rs rename to crates/storage/db/src/tables/codecs/compact.rs index 49b8ed076..a346ce23e 100644 --- a/crates/interfaces/src/db/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -1,6 +1,7 @@ -use crate::db::{ - models::{accounts::AccountBeforeTx, StoredBlockOmmers}, - Compress, Decompress, Error, +use crate::{ + table::{Compress, Decompress}, + tables::models::*, + Error, }; use reth_codecs::{main_codec, Compact}; use reth_primitives::*; diff --git a/crates/interfaces/src/db/codecs/fuzz/inputs.rs b/crates/storage/db/src/tables/codecs/fuzz/inputs.rs similarity index 100% rename from crates/interfaces/src/db/codecs/fuzz/inputs.rs rename to crates/storage/db/src/tables/codecs/fuzz/inputs.rs diff --git a/crates/interfaces/src/db/codecs/fuzz/mod.rs b/crates/storage/db/src/tables/codecs/fuzz/mod.rs similarity index 97% rename from crates/interfaces/src/db/codecs/fuzz/mod.rs rename to crates/storage/db/src/tables/codecs/fuzz/mod.rs index f18870f36..4c34a0cce 100644 --- a/crates/interfaces/src/db/codecs/fuzz/mod.rs +++ b/crates/storage/db/src/tables/codecs/fuzz/mod.rs @@ -14,7 +14,7 @@ macro_rules! impl_fuzzer_with_input { #[allow(non_snake_case)] #[cfg(any(test, feature = "bench"))] pub mod $name { - use crate::db::table; + use crate::table; #[allow(unused_imports)] use reth_primitives::*; @@ -23,7 +23,7 @@ macro_rules! impl_fuzzer_with_input { use super::inputs::*; #[allow(unused_imports)] - use crate::db::models::*; + use crate::tables::models::*; /// Encodes and decodes table types returning its encoded size and the decoded object. /// This method is used for benchmarking, so its parameter should be the actual type that is being tested. diff --git a/crates/interfaces/src/db/codecs/mod.rs b/crates/storage/db/src/tables/codecs/mod.rs similarity index 79% rename from crates/interfaces/src/db/codecs/mod.rs rename to crates/storage/db/src/tables/codecs/mod.rs index fe3d07df4..39716669d 100644 --- a/crates/interfaces/src/db/codecs/mod.rs +++ b/crates/storage/db/src/tables/codecs/mod.rs @@ -4,6 +4,6 @@ mod compact; pub use compact::CompactU256; pub mod fuzz; + mod postcard; -#[cfg(not(feature = "bench-postcard"))] mod scale; diff --git a/crates/interfaces/src/db/codecs/postcard.rs b/crates/storage/db/src/tables/codecs/postcard.rs similarity index 80% rename from crates/interfaces/src/db/codecs/postcard.rs rename to crates/storage/db/src/tables/codecs/postcard.rs index 4a2e66801..8800bdc12 100644 --- a/crates/interfaces/src/db/codecs/postcard.rs +++ b/crates/storage/db/src/tables/codecs/postcard.rs @@ -1,6 +1,9 @@ #![allow(unused)] -use crate::db::{Decode, Encode, Error}; +use crate::{ + table::{Decode, Encode}, + Error, +}; use postcard::{from_bytes, to_allocvec, to_vec}; use reth_primitives::*; @@ -26,7 +29,7 @@ macro_rules! impl_postcard { } impl Decode for $name { - fn decode>(value: B) -> Result { + fn decode>(value: B) -> Result { from_bytes(&value.into()).map_err(|e| Error::Decode(e.into())) } } @@ -36,5 +39,5 @@ macro_rules! impl_postcard { type VecU8 = Vec; -#[cfg(feature = "bench-postcard")] -impl_postcard!(VecU8, Receipt, H256, U256, H160, u8, u16, u64, Header, Account, Log, TxType); +//#[cfg(feature = "bench-postcard")] +//impl_postcard!(VecU8, Receipt, H256, U256, H160, u8, u16, u64, Header, Account, Log, TxType); diff --git a/crates/interfaces/src/db/codecs/scale.rs b/crates/storage/db/src/tables/codecs/scale.rs similarity index 94% rename from crates/interfaces/src/db/codecs/scale.rs rename to crates/storage/db/src/tables/codecs/scale.rs index 4210c2953..121e4526f 100644 --- a/crates/interfaces/src/db/codecs/scale.rs +++ b/crates/storage/db/src/tables/codecs/scale.rs @@ -1,4 +1,7 @@ -use crate::db::{Compress, Decompress, Error}; +use crate::{ + table::{Compress, Decompress}, + Error, +}; use parity_scale_codec::decode_from_bytes; use reth_primitives::*; diff --git a/crates/interfaces/src/db/tables.rs b/crates/storage/db/src/tables/mod.rs similarity index 93% rename from crates/interfaces/src/db/tables.rs rename to crates/storage/db/src/tables/mod.rs index a5f00be60..a7cfe5495 100644 --- a/crates/interfaces/src/db/tables.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -1,13 +1,20 @@ -//! Declaration of all Database tables. +//! Table and data structures -use crate::db::{ - codecs::CompactU256, - models::{ - accounts::{AccountBeforeTx, TxNumberAddress}, - blocks::{BlockNumHash, HeaderHash, NumTransactions, StoredBlockOmmers}, - ShardedKey, +pub mod codecs; +pub mod models; +pub mod utils; + +/// Declaration of all Database tables. +use crate::{ + table::DupSort, + tables::{ + codecs::CompactU256, + models::{ + accounts::{AccountBeforeTx, TxNumberAddress}, + blocks::{HeaderHash, NumTransactions, StoredBlockOmmers}, + BlockNumHash, ShardedKey, + }, }, - DupSort, }; use reth_primitives::{ Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry, @@ -57,7 +64,7 @@ macro_rules! table { #[derive(Clone, Copy, Debug, Default)] pub struct $table_name; - impl $crate::db::table::Table for $table_name { + impl $crate::table::Table for $table_name { const NAME: &'static str = $table_name::const_name(); type Key = $key; type Value = $value; @@ -171,10 +178,10 @@ table!( /// /// ``` /// use reth_primitives::{Address, IntegerList}; - /// use reth_interfaces::db::{DbTx, DbTxMut, DbCursorRO, Database, models::ShardedKey, tables::AccountHistory}; - /// use reth_db::{kv::{EnvKind, Env, test_utils}, mdbx::WriteMap}; + /// use reth_db::{transaction::{DbTxMut,DbTx}, mdbx::{EnvKind, Env, test_utils,WriteMap}, cursor::DbCursorRO,database::Database, tables::{AccountHistory,models::ShardedKey}}; /// use std::{str::FromStr,sync::Arc}; /// + /// /// fn main() { /// let db: Arc> = test_utils::create_test_db(EnvKind::RW); /// let account = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047").unwrap(); diff --git a/crates/interfaces/src/db/models/accounts.rs b/crates/storage/db/src/tables/models/accounts.rs similarity index 97% rename from crates/interfaces/src/db/models/accounts.rs rename to crates/storage/db/src/tables/models/accounts.rs index 70361a1c8..fd199ebda 100644 --- a/crates/interfaces/src/db/models/accounts.rs +++ b/crates/storage/db/src/tables/models/accounts.rs @@ -1,11 +1,9 @@ //! Account related models and types. use crate::{ - db::{ - table::{Decode, Encode}, - Error, - }, impl_fixed_arbitrary, + table::{Decode, Encode}, + Error, }; use bytes::Bytes; use reth_codecs::{main_codec, Compact}; diff --git a/crates/interfaces/src/db/models/blocks.rs b/crates/storage/db/src/tables/models/blocks.rs similarity index 98% rename from crates/interfaces/src/db/models/blocks.rs rename to crates/storage/db/src/tables/models/blocks.rs index a57d4eb32..57a509172 100644 --- a/crates/interfaces/src/db/models/blocks.rs +++ b/crates/storage/db/src/tables/models/blocks.rs @@ -1,11 +1,9 @@ //! Block related models and types. use crate::{ - db::{ - table::{Decode, Encode}, - Error, - }, impl_fixed_arbitrary, + table::{Decode, Encode}, + Error, }; use bytes::Bytes; use reth_codecs::{main_codec, Compact}; diff --git a/crates/interfaces/src/db/models/integer_list.rs b/crates/storage/db/src/tables/models/integer_list.rs similarity index 93% rename from crates/interfaces/src/db/models/integer_list.rs rename to crates/storage/db/src/tables/models/integer_list.rs index ad740cac0..f43345685 100644 --- a/crates/interfaces/src/db/models/integer_list.rs +++ b/crates/storage/db/src/tables/models/integer_list.rs @@ -1,8 +1,8 @@ //! Implements [`Compress`] and [`Decompress`] for [`IntegerList`] -use crate::db::{ - error::Error, +use crate::{ table::{Compress, Decompress}, + Error, }; use bytes::Bytes; use reth_primitives::IntegerList; diff --git a/crates/interfaces/src/db/models/mod.rs b/crates/storage/db/src/tables/models/mod.rs similarity index 99% rename from crates/interfaces/src/db/models/mod.rs rename to crates/storage/db/src/tables/models/mod.rs index a89c3edeb..e775a1b71 100644 --- a/crates/interfaces/src/db/models/mod.rs +++ b/crates/storage/db/src/tables/models/mod.rs @@ -10,7 +10,7 @@ pub use blocks::*; use reth_primitives::{Address, H256}; pub use sharded_key::ShardedKey; -use crate::db::{ +use crate::{ table::{Decode, Encode}, Error, }; diff --git a/crates/interfaces/src/db/models/sharded_key.rs b/crates/storage/db/src/tables/models/sharded_key.rs similarity index 98% rename from crates/interfaces/src/db/models/sharded_key.rs rename to crates/storage/db/src/tables/models/sharded_key.rs index 63cab5550..518bebf9e 100644 --- a/crates/interfaces/src/db/models/sharded_key.rs +++ b/crates/storage/db/src/tables/models/sharded_key.rs @@ -1,6 +1,6 @@ //! Sharded key -use crate::db::{ +use crate::{ table::{Decode, Encode}, Error, }; diff --git a/crates/db/src/utils.rs b/crates/storage/db/src/tables/utils.rs similarity index 53% rename from crates/db/src/utils.rs rename to crates/storage/db/src/tables/utils.rs index 80aa400e8..198356507 100644 --- a/crates/db/src/utils.rs +++ b/crates/storage/db/src/tables/utils.rs @@ -1,22 +1,28 @@ -//! Utils crate for `db`. - -//suse crate::kv::Error; +//! Small database table utilities and helper functions +use crate::{ + table::{Decode, Decompress, Table}, + Error, +}; use bytes::Bytes; -use reth_interfaces::db::{Decode, Decompress, Error, Table}; use std::borrow::Cow; -/// Returns the default page size that can be used in this OS. -pub(crate) fn default_page_size() -> usize { - let os_page_size = page_size::get(); +#[macro_export] +/// Implements the [`arbitrary::Arbitrary`] trait for types with fixed array types. +macro_rules! impl_fixed_arbitrary { + ($name:tt, $size:tt) => { + #[cfg(any(test, feature = "arbitrary"))] + use arbitrary::{Arbitrary, Unstructured}; - // source: https://gitflic.ru/project/erthink/libmdbx/blob?file=mdbx.h#line-num-821 - let libmdbx_max_page_size = 0x10000; + #[cfg(any(test, feature = "arbitrary"))] + impl<'a> Arbitrary<'a> for $name { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + let mut buffer = vec![0; $size]; + u.fill_buffer(buffer.as_mut_slice())?; - // May lead to errors if it's reduced further because of the potential size of the - // data. - let min_page_size = 4096; - - os_page_size.clamp(min_page_size, libmdbx_max_page_size) + Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat) + } + } + }; } /// Helper function to decode a `(key, value)` pair. diff --git a/crates/storage/db/src/utils.rs b/crates/storage/db/src/utils.rs new file mode 100644 index 000000000..4115b52df --- /dev/null +++ b/crates/storage/db/src/utils.rs @@ -0,0 +1,15 @@ +//! Utils crate for `db`. + +/// Returns the default page size that can be used in this OS. +pub(crate) fn default_page_size() -> usize { + let os_page_size = page_size::get(); + + // source: https://gitflic.ru/project/erthink/libmdbx/blob?file=mdbx.h#line-num-821 + let libmdbx_max_page_size = 0x10000; + + // May lead to errors if it's reduced further because of the potential size of the + // data. + let min_page_size = 4096; + + os_page_size.clamp(min_page_size, libmdbx_max_page_size) +} diff --git a/crates/libmdbx-rs/Cargo.lock b/crates/storage/libmdbx-rs/Cargo.lock similarity index 100% rename from crates/libmdbx-rs/Cargo.lock rename to crates/storage/libmdbx-rs/Cargo.lock diff --git a/crates/libmdbx-rs/Cargo.toml b/crates/storage/libmdbx-rs/Cargo.toml similarity index 100% rename from crates/libmdbx-rs/Cargo.toml rename to crates/storage/libmdbx-rs/Cargo.toml diff --git a/crates/libmdbx-rs/LICENSE b/crates/storage/libmdbx-rs/LICENSE similarity index 100% rename from crates/libmdbx-rs/LICENSE rename to crates/storage/libmdbx-rs/LICENSE diff --git a/crates/libmdbx-rs/README.md b/crates/storage/libmdbx-rs/README.md similarity index 100% rename from crates/libmdbx-rs/README.md rename to crates/storage/libmdbx-rs/README.md diff --git a/crates/libmdbx-rs/benches/cursor.rs b/crates/storage/libmdbx-rs/benches/cursor.rs similarity index 100% rename from crates/libmdbx-rs/benches/cursor.rs rename to crates/storage/libmdbx-rs/benches/cursor.rs diff --git a/crates/libmdbx-rs/benches/transaction.rs b/crates/storage/libmdbx-rs/benches/transaction.rs similarity index 100% rename from crates/libmdbx-rs/benches/transaction.rs rename to crates/storage/libmdbx-rs/benches/transaction.rs diff --git a/crates/libmdbx-rs/benches/utils.rs b/crates/storage/libmdbx-rs/benches/utils.rs similarity index 100% rename from crates/libmdbx-rs/benches/utils.rs rename to crates/storage/libmdbx-rs/benches/utils.rs diff --git a/crates/libmdbx-rs/mdbx-sys/Cargo.toml b/crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/Cargo.toml rename to crates/storage/libmdbx-rs/mdbx-sys/Cargo.toml diff --git a/crates/libmdbx-rs/mdbx-sys/build.rs b/crates/storage/libmdbx-rs/mdbx-sys/build.rs similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/build.rs rename to crates/storage/libmdbx-rs/mdbx-sys/build.rs diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/ChangeLog.md b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/ChangeLog.md similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/ChangeLog.md rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/ChangeLog.md diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/LICENSE b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/LICENSE rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/Makefile b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/Makefile rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/README.md b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/README.md similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/README.md rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/README.md diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/config.h.in b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/config.h.in rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c diff --git a/crates/libmdbx-rs/mdbx-sys/libmdbx/ntdll.def b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/ntdll.def similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/libmdbx/ntdll.def rename to crates/storage/libmdbx-rs/mdbx-sys/libmdbx/ntdll.def diff --git a/crates/libmdbx-rs/mdbx-sys/src/lib.rs b/crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs similarity index 100% rename from crates/libmdbx-rs/mdbx-sys/src/lib.rs rename to crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs diff --git a/crates/libmdbx-rs/src/codec.rs b/crates/storage/libmdbx-rs/src/codec.rs similarity index 100% rename from crates/libmdbx-rs/src/codec.rs rename to crates/storage/libmdbx-rs/src/codec.rs diff --git a/crates/libmdbx-rs/src/cursor.rs b/crates/storage/libmdbx-rs/src/cursor.rs similarity index 100% rename from crates/libmdbx-rs/src/cursor.rs rename to crates/storage/libmdbx-rs/src/cursor.rs diff --git a/crates/libmdbx-rs/src/database.rs b/crates/storage/libmdbx-rs/src/database.rs similarity index 100% rename from crates/libmdbx-rs/src/database.rs rename to crates/storage/libmdbx-rs/src/database.rs diff --git a/crates/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs similarity index 100% rename from crates/libmdbx-rs/src/environment.rs rename to crates/storage/libmdbx-rs/src/environment.rs diff --git a/crates/libmdbx-rs/src/error.rs b/crates/storage/libmdbx-rs/src/error.rs similarity index 100% rename from crates/libmdbx-rs/src/error.rs rename to crates/storage/libmdbx-rs/src/error.rs diff --git a/crates/libmdbx-rs/src/flags.rs b/crates/storage/libmdbx-rs/src/flags.rs similarity index 100% rename from crates/libmdbx-rs/src/flags.rs rename to crates/storage/libmdbx-rs/src/flags.rs diff --git a/crates/libmdbx-rs/src/lib.rs b/crates/storage/libmdbx-rs/src/lib.rs similarity index 100% rename from crates/libmdbx-rs/src/lib.rs rename to crates/storage/libmdbx-rs/src/lib.rs diff --git a/crates/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs similarity index 100% rename from crates/libmdbx-rs/src/transaction.rs rename to crates/storage/libmdbx-rs/src/transaction.rs diff --git a/crates/libmdbx-rs/tests/cursor.rs b/crates/storage/libmdbx-rs/tests/cursor.rs similarity index 100% rename from crates/libmdbx-rs/tests/cursor.rs rename to crates/storage/libmdbx-rs/tests/cursor.rs diff --git a/crates/libmdbx-rs/tests/environment.rs b/crates/storage/libmdbx-rs/tests/environment.rs similarity index 100% rename from crates/libmdbx-rs/tests/environment.rs rename to crates/storage/libmdbx-rs/tests/environment.rs diff --git a/crates/libmdbx-rs/tests/transaction.rs b/crates/storage/libmdbx-rs/tests/transaction.rs similarity index 100% rename from crates/libmdbx-rs/tests/transaction.rs rename to crates/storage/libmdbx-rs/tests/transaction.rs diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml new file mode 100644 index 000000000..601a40bc0 --- /dev/null +++ b/crates/storage/provider/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "reth-provider" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" +description = "Reth storage provider." + +[dependencies] +reth-codecs = { path = "../codecs" } +reth-primitives = { path = "../../primitives" } +reth-interfaces = { path = "../../interfaces" } +reth-rpc-types = { path = "../../net/rpc-types" } +reth-db = { path = "../db" } +async-trait = "0.1.57" +thiserror = "1.0.37" +auto_impl = "1.0" +tokio = { version = "1.21.2", features = ["sync"] } +bytes = "1.2" + +# codecs +serde = { version = "1.0.*", default-features = false } +postcard = { version = "1.0.2", features = ["alloc"] } +heapless = "0.7.16" +parity-scale-codec = { version = "3.2.1", features = ["bytes"] } +futures = "0.3.25" +tokio-stream = "0.1.11" +rand = "0.8.5" +arbitrary = { version = "1.1.7", features = ["derive"], optional = true } +secp256k1 = { version = "0.24.0", default-features = false, features = ["alloc", "recovery", "rand"], optional = true } +modular-bitfield = "0.11.2" + +[dev-dependencies] +reth-db = { path = "../db", features = ["test-utils"] } +test-fuzz = "3.0.4" +tokio = { version = "1.21.2", features = ["full"] } +tokio-stream = { version = "0.1.11", features = ["sync"] } +arbitrary = { version = "1.1.7", features = ["derive"]} +hex-literal = "0.3" +secp256k1 = { version = "0.24.0", default-features = false, features = ["alloc", "recovery", "rand"] } + +[features] +bench = [] +test-utils = ["tokio-stream/sync", "secp256k1"] diff --git a/crates/interfaces/src/provider/block.rs b/crates/storage/provider/src/block.rs similarity index 93% rename from crates/interfaces/src/provider/block.rs rename to crates/storage/provider/src/block.rs index a2a23f561..b3d30ccf1 100644 --- a/crates/interfaces/src/provider/block.rs +++ b/crates/storage/provider/src/block.rs @@ -1,12 +1,10 @@ -use crate::{ - db::{ - models::{BlockNumHash, StoredBlockOmmers}, - tables, DbTx, DbTxMut, - }, - provider::Error as ProviderError, - Result, -}; use auto_impl::auto_impl; +use reth_db::{ + models::{BlockNumHash, StoredBlockOmmers}, + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_interfaces::{provider::Error as ProviderError, Result}; use reth_primitives::{ rpc::{BlockId, BlockNumber}, Block, BlockHash, BlockHashOrNumber, BlockLocked, Header, H256, U256, @@ -120,8 +118,13 @@ pub fn get_cumulative_tx_count_by_hash<'a, TX: DbTxMut<'a> + DbTx<'a>>( let block_num_hash = BlockNumHash((block_number, block_hash)); - tx.get::(block_num_hash)? - .ok_or_else(|| ProviderError::BlockBodyNotExist { block_num_hash }.into()) + tx.get::(block_num_hash)?.ok_or_else(|| { + ProviderError::BlockBodyNotExist { + block_number: block_num_hash.number(), + block_hash: block_num_hash.hash(), + } + .into() + }) } /// Fill block to database. Useful for tests. diff --git a/crates/interfaces/src/provider/db_provider.rs b/crates/storage/provider/src/db_provider.rs similarity index 57% rename from crates/interfaces/src/provider/db_provider.rs rename to crates/storage/provider/src/db_provider.rs index 08cd0b344..6e838ffc6 100644 --- a/crates/interfaces/src/provider/db_provider.rs +++ b/crates/storage/provider/src/db_provider.rs @@ -10,7 +10,7 @@ pub use storage::{ StateProviderImplRefLatest, }; -use crate::db::Database; +use reth_db::database::Database; /// Provider pub struct ProviderImpl { @@ -24,3 +24,18 @@ impl ProviderImpl { Self { db } } } + +#[cfg(test)] +mod tests { + use crate::StateProviderFactory; + + use super::ProviderImpl; + use reth_db::mdbx::{test_utils::create_test_db, EnvKind, WriteMap}; + + #[test] + fn common_history_provider() { + let db = create_test_db::(EnvKind::RW); + let provider = ProviderImpl::new(db); + let _ = provider.latest(); + } +} diff --git a/crates/interfaces/src/provider/db_provider/block.rs b/crates/storage/provider/src/db_provider/block.rs similarity index 65% rename from crates/interfaces/src/provider/db_provider/block.rs rename to crates/storage/provider/src/db_provider/block.rs index 9290e7136..1de611e37 100644 --- a/crates/interfaces/src/provider/db_provider/block.rs +++ b/crates/storage/provider/src/db_provider/block.rs @@ -1,15 +1,14 @@ -use crate::{ - db::{tables, Database, DbTx}, - provider::{HeaderProvider, ProviderImpl}, -}; +use crate::{HeaderProvider, ProviderImpl}; +use reth_db::{database::Database, tables, transaction::DbTx}; +use reth_interfaces::Result; use reth_primitives::{BlockNumber, Header}; impl HeaderProvider for ProviderImpl { - fn header(&self, block_hash: &reth_primitives::BlockHash) -> crate::Result> { + fn header(&self, block_hash: &reth_primitives::BlockHash) -> Result> { self.db.view(|tx| tx.get::((0, *block_hash).into()))?.map_err(Into::into) } - fn header_by_number(&self, num: BlockNumber) -> crate::Result> { + fn header_by_number(&self, num: BlockNumber) -> Result> { if let Some(hash) = self.db.view(|tx| tx.get::(num))?? { self.header(&hash) } else { diff --git a/crates/interfaces/src/provider/db_provider/storage.rs b/crates/storage/provider/src/db_provider/storage.rs similarity index 97% rename from crates/interfaces/src/provider/db_provider/storage.rs rename to crates/storage/provider/src/db_provider/storage.rs index 57c2e299a..84060aa4a 100644 --- a/crates/interfaces/src/provider/db_provider/storage.rs +++ b/crates/storage/provider/src/db_provider/storage.rs @@ -1,9 +1,13 @@ use super::ProviderImpl; -use crate::{ - db::{tables, Database, DatabaseGAT, DbCursorRO, DbDupCursorRO, DbTx}, - provider::{AccountProvider, Error, StateProvider, StateProviderFactory}, - Result, +use crate::{AccountProvider, Error, StateProvider, StateProviderFactory}; +use reth_db::{ + cursor::{DbCursorRO, DbDupCursorRO}, + database::{Database, DatabaseGAT}, + tables, + transaction::DbTx, }; +use reth_interfaces::Result; + use reth_primitives::{ Account, Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxNumber, H256, U256, }; diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs new file mode 100644 index 000000000..523fa34d9 --- /dev/null +++ b/crates/storage/provider/src/lib.rs @@ -0,0 +1,27 @@ +#![warn(missing_docs, unreachable_pub)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] + +//! + +mod block; +pub mod db_provider; +mod state; + +#[cfg(any(test, feature = "test-utils"))] +/// Common test helpers for mocking the Provider. +pub mod test_utils; + +pub use block::{ + get_cumulative_tx_count_by_hash, insert_canonical_block, BlockProvider, ChainInfo, + HeaderProvider, +}; +pub use db_provider::{ + self as db, ProviderImpl, StateProviderImplHistory, StateProviderImplLatest, + StateProviderImplRefHistory, StateProviderImplRefLatest, +}; +pub use reth_interfaces::provider::Error; +pub use state::{AccountProvider, StateProvider, StateProviderFactory}; diff --git a/crates/interfaces/src/provider/state.rs b/crates/storage/provider/src/state.rs similarity index 97% rename from crates/interfaces/src/provider/state.rs rename to crates/storage/provider/src/state.rs index e69893101..a9758216d 100644 --- a/crates/interfaces/src/provider/state.rs +++ b/crates/storage/provider/src/state.rs @@ -1,4 +1,4 @@ -use crate::Result; +use reth_interfaces::Result; use reth_primitives::{ Account, Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, H256, U256, }; diff --git a/crates/interfaces/src/test_utils/api.rs b/crates/storage/provider/src/test_utils/api.rs similarity index 52% rename from crates/interfaces/src/test_utils/api.rs rename to crates/storage/provider/src/test_utils/api.rs index dc6247605..6637ab7d7 100644 --- a/crates/interfaces/src/test_utils/api.rs +++ b/crates/storage/provider/src/test_utils/api.rs @@ -1,7 +1,5 @@ -use crate::{ - provider, - provider::{BlockProvider, HeaderProvider}, -}; +use crate::{BlockProvider, ChainInfo, HeaderProvider}; +use reth_interfaces::Result; use reth_primitives::{rpc::BlockId, Block, BlockHash, BlockNumber, Header, H256, U256}; /// Supports various api interfaces for testing purposes. @@ -11,8 +9,8 @@ pub struct TestApi; /// Noop implementation for testing purposes impl BlockProvider for TestApi { - fn chain_info(&self) -> crate::Result { - Ok(provider::ChainInfo { + fn chain_info(&self) -> Result { + Ok(ChainInfo { best_hash: Default::default(), best_number: 0, last_finalized: None, @@ -20,25 +18,25 @@ impl BlockProvider for TestApi { }) } - fn block(&self, _id: BlockId) -> crate::Result> { + fn block(&self, _id: BlockId) -> Result> { Ok(None) } - fn block_number(&self, _hash: H256) -> crate::Result> { + fn block_number(&self, _hash: H256) -> Result> { Ok(None) } - fn block_hash(&self, _number: U256) -> crate::Result> { + fn block_hash(&self, _number: U256) -> Result> { Ok(None) } } impl HeaderProvider for TestApi { - fn header(&self, _block_hash: &BlockHash) -> crate::Result> { + fn header(&self, _block_hash: &BlockHash) -> Result> { Ok(None) } - fn header_by_number(&self, _num: u64) -> crate::Result> { + fn header_by_number(&self, _num: u64) -> Result> { Ok(None) } } diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs new file mode 100644 index 000000000..7b837ea3a --- /dev/null +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -0,0 +1,3 @@ +mod api; + +pub use api::TestApi; diff --git a/docs/repo/layout.md b/docs/repo/layout.md index c9ba6500c..3db49207a 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -22,9 +22,9 @@ These crates define primitive types or algorithms such as RLP. These crates are related to the database. -- [`codecs`](../../crates/codecs): Different storage codecs. -- [`libmdbx-rs`](../../crates/libmdbx-rs): Rust bindings for [libmdbx](https://libmdbx.dqdkfa.ru). A fork of an earlier Apache-licensed version of [libmdbx-rs][libmdbx-rs]. -- [`db`](../../crates/db): Strongly typed database bindings to LibMDBX containing read/write access to Ethereum state and historical data (transactions, blocks etc.) +- [`codecs`](../../crates/storage/codecs): Different storage codecs. +- [`libmdbx-rs`](../../crates/storage/libmdbx-rs): Rust bindings for [libmdbx](https://libmdbx.dqdkfa.ru). A fork of an earlier Apache-licensed version of [libmdbx-rs][libmdbx-rs]. +- [`db`](../../crates/storage/db): Strongly typed database bindings to LibMDBX containing read/write access to Ethereum state and historical data (transactions, blocks etc.) ### Networking