From 6bf3ca320abf7d54312cfad361f5242e3bfd130f Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Sat, 17 Aug 2024 10:25:34 -0700 Subject: [PATCH] feat(db): models crate (#10314) Co-authored-by: Miguel Oliveira --- Cargo.lock | 18 ++++- Cargo.toml | 2 + crates/storage/db-api/Cargo.toml | 3 +- crates/storage/db-api/src/models/accounts.rs | 50 +----------- crates/storage/db-api/src/models/blocks.rs | 78 +----------------- crates/storage/db-api/src/models/mod.rs | 1 + crates/storage/db-models/Cargo.toml | 46 +++++++++++ crates/storage/db-models/src/accounts.rs | 51 ++++++++++++ crates/storage/db-models/src/blocks.rs | 85 ++++++++++++++++++++ crates/storage/db-models/src/lib.rs | 9 +++ crates/storage/db/src/tables/mod.rs | 4 +- crates/storage/storage-api/Cargo.toml | 2 +- crates/storage/storage-api/src/account.rs | 2 +- crates/storage/storage-api/src/block.rs | 2 +- 14 files changed, 220 insertions(+), 133 deletions(-) create mode 100644 crates/storage/db-models/Cargo.toml create mode 100644 crates/storage/db-models/src/accounts.rs create mode 100644 crates/storage/db-models/src/blocks.rs create mode 100644 crates/storage/db-models/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 731e76834..a8abcf077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6762,6 +6762,7 @@ dependencies = [ "proptest-arbitrary-interop", "rand 0.8.5", "reth-codecs", + "reth-db-models", "reth-primitives", "reth-primitives-traits", "reth-prune-types", @@ -6798,6 +6799,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-db-models" +version = "1.0.5" +dependencies = [ + "arbitrary", + "bytes", + "modular-bitfield", + "proptest", + "proptest-arbitrary-interop", + "reth-codecs", + "reth-primitives", + "serde", + "test-fuzz", +] + [[package]] name = "reth-discv4" version = "1.0.5" @@ -8665,7 +8681,7 @@ version = "1.0.5" dependencies = [ "auto_impl", "reth-chainspec", - "reth-db-api", + "reth-db-models", "reth-execution-types", "reth-primitives", "reth-prune-types", diff --git a/Cargo.toml b/Cargo.toml index 722d3e493..4e8a30e04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,7 @@ members = [ "crates/storage/codecs/derive/", "crates/storage/db-api/", "crates/storage/db-common", + "crates/storage/db-models/", "crates/storage/db/", "crates/storage/errors/", "crates/storage/libmdbx-rs/", @@ -289,6 +290,7 @@ reth-consensus-debug-client = { path = "crates/consensus/debug-client" } reth-db = { path = "crates/storage/db", default-features = false } reth-db-api = { path = "crates/storage/db-api" } reth-db-common = { path = "crates/storage/db-common" } +reth-db-models = { path = "crates/storage/db-models"} reth-discv4 = { path = "crates/net/discv4" } reth-discv5 = { path = "crates/net/discv5" } reth-dns-discovery = { path = "crates/net/dns" } diff --git a/crates/storage/db-api/Cargo.toml b/crates/storage/db-api/Cargo.toml index 54ec5bf42..8d9476ce6 100644 --- a/crates/storage/db-api/Cargo.toml +++ b/crates/storage/db-api/Cargo.toml @@ -14,11 +14,12 @@ workspace = true [dependencies] # reth reth-codecs.workspace = true +reth-db-models.workspace = true reth-primitives = { workspace = true, features = ["reth-codec"] } reth-primitives-traits.workspace = true reth-prune-types.workspace = true -reth-storage-errors.workspace = true reth-stages-types.workspace = true +reth-storage-errors.workspace = true reth-trie-common.workspace = true # codecs diff --git a/crates/storage/db-api/src/models/accounts.rs b/crates/storage/db-api/src/models/accounts.rs index fd3c6ee38..b71864c2f 100644 --- a/crates/storage/db-api/src/models/accounts.rs +++ b/crates/storage/db-api/src/models/accounts.rs @@ -7,57 +7,9 @@ use crate::{ table::{Decode, Encode}, DatabaseError, }; -use reth_codecs::{add_arbitrary_tests, Compact}; -use reth_primitives::{Account, Address, BlockNumber, Buf, StorageKey}; +use reth_primitives::{Address, BlockNumber, StorageKey}; use serde::{Deserialize, Serialize}; -/// Account as it is saved in the database. -/// -/// [`Address`] is the subkey. -#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)] -#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] -#[add_arbitrary_tests(compact)] -pub struct AccountBeforeTx { - /// Address for the account. Acts as `DupSort::SubKey`. - pub address: Address, - /// Account state before the transaction. - pub info: Option, -} - -// NOTE: Removing reth_codec and manually encode subkey -// and compress second part of the value. If we have compression -// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey -impl Compact for AccountBeforeTx { - fn to_compact(&self, buf: &mut B) -> usize - where - B: bytes::BufMut + AsMut<[u8]>, - { - // for now put full bytes and later compress it. - buf.put_slice(self.address.as_slice()); - - let mut acc_len = 0; - if let Some(account) = self.info { - acc_len = account.to_compact(buf); - } - acc_len + 20 - } - - fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { - let address = Address::from_slice(&buf[..20]); - buf.advance(20); - - let info = if len - 20 > 0 { - let (acc, advanced_buf) = Account::from_compact(buf, len - 20); - buf = advanced_buf; - Some(acc) - } else { - None - }; - - (Self { address, info }, buf) - } -} - /// [`BlockNumber`] concatenated with [`Address`]. /// /// Since it's used as a key, it isn't compressed when encoding it. diff --git a/crates/storage/db-api/src/models/blocks.rs b/crates/storage/db-api/src/models/blocks.rs index 6e6a7d2c7..052254be7 100644 --- a/crates/storage/db-api/src/models/blocks.rs +++ b/crates/storage/db-api/src/models/blocks.rs @@ -1,71 +1,8 @@ //! Block related models and types. use reth_codecs::{add_arbitrary_tests, Compact}; -use reth_primitives::{Header, TxNumber, Withdrawals, B256}; +use reth_primitives::{Header, Withdrawals, B256}; use serde::{Deserialize, Serialize}; -use std::ops::Range; - -/// Total number of transactions. -pub type NumTransactions = u64; - -/// The storage of the block body indices. -/// -/// It has the pointer to the transaction Number of the first -/// transaction in the block and the total number of transactions. -#[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize, Compact)] -#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] -#[add_arbitrary_tests(compact)] -pub struct StoredBlockBodyIndices { - /// The number of the first transaction in this block - /// - /// Note: If the block is empty, this is the number of the first transaction - /// in the next non-empty block. - pub first_tx_num: TxNumber, - /// The total number of transactions in the block - /// - /// NOTE: Number of transitions is equal to number of transactions with - /// additional transition for block change if block has block reward or withdrawal. - pub tx_count: NumTransactions, -} - -impl StoredBlockBodyIndices { - /// Return the range of transaction ids for this block. - pub const fn tx_num_range(&self) -> Range { - self.first_tx_num..self.first_tx_num + self.tx_count - } - - /// Return the index of last transaction in this block unless the block - /// is empty in which case it refers to the last transaction in a previous - /// non-empty block - pub const fn last_tx_num(&self) -> TxNumber { - self.first_tx_num.saturating_add(self.tx_count).saturating_sub(1) - } - - /// First transaction index. - /// - /// Caution: If the block is empty, this is the number of the first transaction - /// in the next non-empty block. - pub const fn first_tx_num(&self) -> TxNumber { - self.first_tx_num - } - - /// Return the index of the next transaction after this block. - pub const fn next_tx_num(&self) -> TxNumber { - self.first_tx_num + self.tx_count - } - - /// Return a flag whether the block is empty - pub const fn is_empty(&self) -> bool { - self.tx_count == 0 - } - - /// Return number of transaction inside block - /// - /// NOTE: This is not the same as the number of transitions. - pub const fn tx_count(&self) -> NumTransactions { - self.tx_count - } -} /// The storage representation of a block's ommers. /// @@ -105,17 +42,4 @@ mod tests { StoredBlockOmmers::decompress::>(ommer.compress()).unwrap() ); } - - #[test] - fn block_indices() { - let first_tx_num = 10; - let tx_count = 6; - let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count }; - - assert_eq!(block_indices.first_tx_num(), first_tx_num); - assert_eq!(block_indices.last_tx_num(), first_tx_num + tx_count - 1); - assert_eq!(block_indices.next_tx_num(), first_tx_num + tx_count); - assert_eq!(block_indices.tx_count(), tx_count); - assert_eq!(block_indices.tx_num_range(), first_tx_num..first_tx_num + tx_count); - } } diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index e8300b264..c177ca80b 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -21,6 +21,7 @@ pub mod storage_sharded_key; pub use accounts::*; pub use blocks::*; pub use client_version::ClientVersion; +pub use reth_db_models::{AccountBeforeTx, StoredBlockBodyIndices}; pub use sharded_key::ShardedKey; /// Macro that implements [`Encode`] and [`Decode`] for uint types. diff --git a/crates/storage/db-models/Cargo.toml b/crates/storage/db-models/Cargo.toml new file mode 100644 index 000000000..6f0ae3ffc --- /dev/null +++ b/crates/storage/db-models/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "reth-db-models" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Database models used in storage module." + +[lints] +workspace = true + +[dependencies] +# reth +reth-codecs.workspace = true +reth-primitives = { workspace = true, features = ["reth-codec"] } + +# codecs +modular-bitfield.workspace = true +serde = { workspace = true, default-features = false } + +# misc +bytes.workspace = true + +# arbitrary utils +arbitrary = { workspace = true, features = ["derive"], optional = true } +proptest = { workspace = true, optional = true } + +[dev-dependencies] +# reth +reth-primitives = { workspace = true, features = ["arbitrary"] } +reth-codecs.workspace = true + +arbitrary = { workspace = true, features = ["derive"] } +proptest-arbitrary-interop.workspace = true +proptest.workspace = true +test-fuzz.workspace = true + +[features] +test-utils = ["arbitrary"] +arbitrary = [ + "reth-primitives/arbitrary", + "dep:arbitrary", + "dep:proptest", +] diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs new file mode 100644 index 000000000..bc84ea0fc --- /dev/null +++ b/crates/storage/db-models/src/accounts.rs @@ -0,0 +1,51 @@ +use reth_codecs::{add_arbitrary_tests, Compact}; +use serde::Serialize; + +use reth_primitives::{Account, Address, Buf}; + +/// Account as it is saved in the database. +/// +/// [`Address`] is the subkey. +#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] +#[add_arbitrary_tests(compact)] +pub struct AccountBeforeTx { + /// Address for the account. Acts as `DupSort::SubKey`. + pub address: Address, + /// Account state before the transaction. + pub info: Option, +} + +// NOTE: Removing reth_codec and manually encode subkey +// and compress second part of the value. If we have compression +// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey +impl Compact for AccountBeforeTx { + fn to_compact(&self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + // for now put full bytes and later compress it. + buf.put_slice(self.address.as_slice()); + + let mut acc_len = 0; + if let Some(account) = self.info { + acc_len = account.to_compact(buf); + } + acc_len + 20 + } + + fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { + let address = Address::from_slice(&buf[..20]); + buf.advance(20); + + let info = if len - 20 > 0 { + let (acc, advanced_buf) = Account::from_compact(buf, len - 20); + buf = advanced_buf; + Some(acc) + } else { + None + }; + + (Self { address, info }, buf) + } +} diff --git a/crates/storage/db-models/src/blocks.rs b/crates/storage/db-models/src/blocks.rs new file mode 100644 index 000000000..c993e58da --- /dev/null +++ b/crates/storage/db-models/src/blocks.rs @@ -0,0 +1,85 @@ +use std::ops::Range; + +use reth_codecs::{add_arbitrary_tests, Compact}; +use reth_primitives::TxNumber; +use serde::{Deserialize, Serialize}; + +/// Total number of transactions. +pub type NumTransactions = u64; + +/// The storage of the block body indices. +/// +/// It has the pointer to the transaction Number of the first +/// transaction in the block and the total number of transactions. +#[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize, Compact)] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] +#[add_arbitrary_tests(compact)] +pub struct StoredBlockBodyIndices { + /// The number of the first transaction in this block + /// + /// Note: If the block is empty, this is the number of the first transaction + /// in the next non-empty block. + pub first_tx_num: TxNumber, + /// The total number of transactions in the block + /// + /// NOTE: Number of transitions is equal to number of transactions with + /// additional transition for block change if block has block reward or withdrawal. + pub tx_count: NumTransactions, +} + +impl StoredBlockBodyIndices { + /// Return the range of transaction ids for this block. + pub const fn tx_num_range(&self) -> Range { + self.first_tx_num..self.first_tx_num + self.tx_count + } + + /// Return the index of last transaction in this block unless the block + /// is empty in which case it refers to the last transaction in a previous + /// non-empty block + pub const fn last_tx_num(&self) -> TxNumber { + self.first_tx_num.saturating_add(self.tx_count).saturating_sub(1) + } + + /// First transaction index. + /// + /// Caution: If the block is empty, this is the number of the first transaction + /// in the next non-empty block. + pub const fn first_tx_num(&self) -> TxNumber { + self.first_tx_num + } + + /// Return the index of the next transaction after this block. + pub const fn next_tx_num(&self) -> TxNumber { + self.first_tx_num + self.tx_count + } + + /// Return a flag whether the block is empty + pub const fn is_empty(&self) -> bool { + self.tx_count == 0 + } + + /// Return number of transaction inside block + /// + /// NOTE: This is not the same as the number of transitions. + pub const fn tx_count(&self) -> NumTransactions { + self.tx_count + } +} + +#[cfg(test)] +mod tests { + use crate::StoredBlockBodyIndices; + + #[test] + fn block_indices() { + let first_tx_num = 10; + let tx_count = 6; + let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count }; + + assert_eq!(block_indices.first_tx_num(), first_tx_num); + assert_eq!(block_indices.last_tx_num(), first_tx_num + tx_count - 1); + assert_eq!(block_indices.next_tx_num(), first_tx_num + tx_count); + assert_eq!(block_indices.tx_count(), tx_count); + assert_eq!(block_indices.tx_num_range(), first_tx_num..first_tx_num + tx_count); + } +} diff --git a/crates/storage/db-models/src/lib.rs b/crates/storage/db-models/src/lib.rs new file mode 100644 index 000000000..55a210269 --- /dev/null +++ b/crates/storage/db-models/src/lib.rs @@ -0,0 +1,9 @@ +//! Models used in storage module + +/// Accounts +pub mod accounts; +pub use accounts::AccountBeforeTx; + +/// Blocks +pub mod blocks; +pub use blocks::StoredBlockBodyIndices; diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index b44957f40..cff4b8009 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -21,11 +21,11 @@ pub(crate) mod utils; use reth_db_api::{ models::{ - accounts::{AccountBeforeTx, BlockNumberAddress}, + accounts::BlockNumberAddress, blocks::{HeaderHash, StoredBlockOmmers}, client_version::ClientVersion, storage_sharded_key::StorageShardedKey, - CompactU256, ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals, + AccountBeforeTx, CompactU256, ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals, }, table::{Decode, DupSort, Encode, Table}, }; diff --git a/crates/storage/storage-api/Cargo.toml b/crates/storage/storage-api/Cargo.toml index 36a0c2a13..e9edf9471 100644 --- a/crates/storage/storage-api/Cargo.toml +++ b/crates/storage/storage-api/Cargo.toml @@ -14,8 +14,8 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true +reth-db-models.workspace = true reth-execution-types.workspace = true -reth-db-api.workspace = true reth-primitives.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true diff --git a/crates/storage/storage-api/src/account.rs b/crates/storage/storage-api/src/account.rs index 1257f07b4..8f77894ec 100644 --- a/crates/storage/storage-api/src/account.rs +++ b/crates/storage/storage-api/src/account.rs @@ -1,5 +1,5 @@ use auto_impl::auto_impl; -use reth_db_api::models::AccountBeforeTx; +use reth_db_models::AccountBeforeTx; use reth_primitives::{Account, Address, BlockNumber}; use reth_storage_errors::provider::ProviderResult; use std::{ diff --git a/crates/storage/storage-api/src/block.rs b/crates/storage/storage-api/src/block.rs index 569a1e7ff..d6d0366a1 100644 --- a/crates/storage/storage-api/src/block.rs +++ b/crates/storage/storage-api/src/block.rs @@ -2,7 +2,7 @@ use crate::{ BlockIdReader, BlockNumReader, HeaderProvider, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, TransactionVariant, TransactionsProvider, WithdrawalsProvider, }; -use reth_db_api::models::StoredBlockBodyIndices; +use reth_db_models::StoredBlockBodyIndices; use reth_primitives::{ Block, BlockHashOrNumber, BlockId, BlockNumber, BlockNumberOrTag, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, B256,