feat(db): models crate (#10314)

Co-authored-by: Miguel Oliveira <migueloliveiradev@gmail.com>

Cargo.lock

@@ -6762,6 +6762,7 @@ dependencies = [
  "proptest-arbitrary-interop",
  "rand 0.8.5",
  "reth-codecs",
+ "reth-db-models",
  "reth-primitives",
  "reth-primitives-traits",
  "reth-prune-types",
@@ -6798,6 +6799,21 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "reth-db-models"
+version = "1.0.5"
+dependencies = [
+ "arbitrary",
+ "bytes",
+ "modular-bitfield",
+ "proptest",
+ "proptest-arbitrary-interop",
+ "reth-codecs",
+ "reth-primitives",
+ "serde",
+ "test-fuzz",
+]
+
 [[package]]
 name = "reth-discv4"
 version = "1.0.5"
@@ -8665,7 +8681,7 @@ version = "1.0.5"
 dependencies = [
  "auto_impl",
  "reth-chainspec",
- "reth-db-api",
+ "reth-db-models",
  "reth-execution-types",
  "reth-primitives",
  "reth-prune-types",

Cargo.toml

@@ -104,6 +104,7 @@ members = [
     "crates/storage/codecs/derive/",
     "crates/storage/db-api/",
     "crates/storage/db-common",
+    "crates/storage/db-models/",
     "crates/storage/db/",
     "crates/storage/errors/",
     "crates/storage/libmdbx-rs/",
@@ -289,6 +290,7 @@ reth-consensus-debug-client = { path = "crates/consensus/debug-client" }
 reth-db = { path = "crates/storage/db", default-features = false }
 reth-db-api = { path = "crates/storage/db-api" }
 reth-db-common = { path = "crates/storage/db-common" }
+reth-db-models = { path = "crates/storage/db-models"}
 reth-discv4 = { path = "crates/net/discv4" }
 reth-discv5 = { path = "crates/net/discv5" }
 reth-dns-discovery = { path = "crates/net/dns" }

crates/storage/db-api/Cargo.toml

@@ -14,11 +14,12 @@ workspace = true
 [dependencies]
 # reth
 reth-codecs.workspace = true
+reth-db-models.workspace = true
 reth-primitives = { workspace = true, features = ["reth-codec"] }
 reth-primitives-traits.workspace = true
 reth-prune-types.workspace = true
-reth-storage-errors.workspace = true
 reth-stages-types.workspace = true
+reth-storage-errors.workspace = true
 reth-trie-common.workspace = true
 
 # codecs

crates/storage/db-api/src/models/accounts.rs

@@ -7,57 +7,9 @@ use crate::{
     table::{Decode, Encode},
     DatabaseError,
 };
-use reth_codecs::{add_arbitrary_tests, Compact};
-use reth_primitives::{Account, Address, BlockNumber, Buf, StorageKey};
+use reth_primitives::{Address, BlockNumber, StorageKey};
 use serde::{Deserialize, Serialize};
 
-/// Account as it is saved in the database.
-///
-/// [`Address`] is the subkey.
-#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)]
-#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
-#[add_arbitrary_tests(compact)]
-pub struct AccountBeforeTx {
-    /// Address for the account. Acts as `DupSort::SubKey`.
-    pub address: Address,
-    /// Account state before the transaction.
-    pub info: Option<Account>,
-}
-
-// NOTE: Removing reth_codec and manually encode subkey
-// and compress second part of the value. If we have compression
-// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
-impl Compact for AccountBeforeTx {
-    fn to_compact<B>(&self, buf: &mut B) -> usize
-    where
-        B: bytes::BufMut + AsMut<[u8]>,
-    {
-        // for now put full bytes and later compress it.
-        buf.put_slice(self.address.as_slice());
-
-        let mut acc_len = 0;
-        if let Some(account) = self.info {
-            acc_len = account.to_compact(buf);
-        }
-        acc_len + 20
-    }
-
-    fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
-        let address = Address::from_slice(&buf[..20]);
-        buf.advance(20);
-
-        let info = if len - 20 > 0 {
-            let (acc, advanced_buf) = Account::from_compact(buf, len - 20);
-            buf = advanced_buf;
-            Some(acc)
-        } else {
-            None
-        };
-
-        (Self { address, info }, buf)
-    }
-}
-
 /// [`BlockNumber`] concatenated with [`Address`].
 ///
 /// Since it's used as a key, it isn't compressed when encoding it.

crates/storage/db-api/src/models/blocks.rs

@@ -1,71 +1,8 @@
 //! Block related models and types.
 
 use reth_codecs::{add_arbitrary_tests, Compact};
-use reth_primitives::{Header, TxNumber, Withdrawals, B256};
+use reth_primitives::{Header, Withdrawals, B256};
 use serde::{Deserialize, Serialize};
-use std::ops::Range;
-
-/// Total number of transactions.
-pub type NumTransactions = u64;
-
-/// The storage of the block body indices.
-///
-/// It has the pointer to the transaction Number of the first
-/// transaction in the block and the total number of transactions.
-#[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize, Compact)]
-#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
-#[add_arbitrary_tests(compact)]
-pub struct StoredBlockBodyIndices {
-    /// The number of the first transaction in this block
-    ///
-    /// Note: If the block is empty, this is the number of the first transaction
-    /// in the next non-empty block.
-    pub first_tx_num: TxNumber,
-    /// The total number of transactions in the block
-    ///
-    /// NOTE: Number of transitions is equal to number of transactions with
-    /// additional transition for block change if block has block reward or withdrawal.
-    pub tx_count: NumTransactions,
-}
-
-impl StoredBlockBodyIndices {
-    /// Return the range of transaction ids for this block.
-    pub const fn tx_num_range(&self) -> Range<TxNumber> {
-        self.first_tx_num..self.first_tx_num + self.tx_count
-    }
-
-    /// Return the index of last transaction in this block unless the block
-    /// is empty in which case it refers to the last transaction in a previous
-    /// non-empty block
-    pub const fn last_tx_num(&self) -> TxNumber {
-        self.first_tx_num.saturating_add(self.tx_count).saturating_sub(1)
-    }
-
-    /// First transaction index.
-    ///
-    /// Caution: If the block is empty, this is the number of the first transaction
-    /// in the next non-empty block.
-    pub const fn first_tx_num(&self) -> TxNumber {
-        self.first_tx_num
-    }
-
-    /// Return the index of the next transaction after this block.
-    pub const fn next_tx_num(&self) -> TxNumber {
-        self.first_tx_num + self.tx_count
-    }
-
-    /// Return a flag whether the block is empty
-    pub const fn is_empty(&self) -> bool {
-        self.tx_count == 0
-    }
-
-    /// Return number of transaction inside block
-    ///
-    /// NOTE: This is not the same as the number of transitions.
-    pub const fn tx_count(&self) -> NumTransactions {
-        self.tx_count
-    }
-}
 
 /// The storage representation of a block's ommers.
 ///
@@ -105,17 +42,4 @@ mod tests {
             StoredBlockOmmers::decompress::<Vec<_>>(ommer.compress()).unwrap()
         );
     }
-
-    #[test]
-    fn block_indices() {
-        let first_tx_num = 10;
-        let tx_count = 6;
-        let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count };
-
-        assert_eq!(block_indices.first_tx_num(), first_tx_num);
-        assert_eq!(block_indices.last_tx_num(), first_tx_num + tx_count - 1);
-        assert_eq!(block_indices.next_tx_num(), first_tx_num + tx_count);
-        assert_eq!(block_indices.tx_count(), tx_count);
-        assert_eq!(block_indices.tx_num_range(), first_tx_num..first_tx_num + tx_count);
-    }
 }

crates/storage/db-api/src/models/mod.rs

@@ -21,6 +21,7 @@ pub mod storage_sharded_key;
 pub use accounts::*;
 pub use blocks::*;
 pub use client_version::ClientVersion;
+pub use reth_db_models::{AccountBeforeTx, StoredBlockBodyIndices};
 pub use sharded_key::ShardedKey;
 
 /// Macro that implements [`Encode`] and [`Decode`] for uint types.

crates/storage/db-models/Cargo.toml

@@ -0,0 +1,46 @@
+[package]
+name = "reth-db-models"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Database models used in storage module."
+
+[lints]
+workspace = true
+
+[dependencies]
+# reth
+reth-codecs.workspace = true
+reth-primitives = { workspace = true, features = ["reth-codec"] }
+
+# codecs
+modular-bitfield.workspace = true
+serde = { workspace = true, default-features = false }
+
+# misc
+bytes.workspace = true
+
+# arbitrary utils
+arbitrary = { workspace = true, features = ["derive"], optional = true }
+proptest = { workspace = true, optional = true }
+
+[dev-dependencies]
+# reth
+reth-primitives = { workspace = true, features = ["arbitrary"] }
+reth-codecs.workspace = true
+
+arbitrary = { workspace = true, features = ["derive"] }
+proptest-arbitrary-interop.workspace = true
+proptest.workspace = true
+test-fuzz.workspace = true
+
+[features]
+test-utils = ["arbitrary"]
+arbitrary = [
+    "reth-primitives/arbitrary",
+    "dep:arbitrary",
+    "dep:proptest",
+]

crates/storage/db-models/src/accounts.rs

@@ -0,0 +1,51 @@
+use reth_codecs::{add_arbitrary_tests, Compact};
+use serde::Serialize;
+
+use reth_primitives::{Account, Address, Buf};
+
+/// Account as it is saved in the database.
+///
+/// [`Address`] is the subkey.
+#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)]
+#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
+#[add_arbitrary_tests(compact)]
+pub struct AccountBeforeTx {
+    /// Address for the account. Acts as `DupSort::SubKey`.
+    pub address: Address,
+    /// Account state before the transaction.
+    pub info: Option<Account>,
+}
+
+// NOTE: Removing reth_codec and manually encode subkey
+// and compress second part of the value. If we have compression
+// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
+impl Compact for AccountBeforeTx {
+    fn to_compact<B>(&self, buf: &mut B) -> usize
+    where
+        B: bytes::BufMut + AsMut<[u8]>,
+    {
+        // for now put full bytes and later compress it.
+        buf.put_slice(self.address.as_slice());
+
+        let mut acc_len = 0;
+        if let Some(account) = self.info {
+            acc_len = account.to_compact(buf);
+        }
+        acc_len + 20
+    }
+
+    fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
+        let address = Address::from_slice(&buf[..20]);
+        buf.advance(20);
+
+        let info = if len - 20 > 0 {
+            let (acc, advanced_buf) = Account::from_compact(buf, len - 20);
+            buf = advanced_buf;
+            Some(acc)
+        } else {
+            None
+        };
+
+        (Self { address, info }, buf)
+    }
+}

crates/storage/db-models/src/blocks.rs

@@ -0,0 +1,85 @@
+use std::ops::Range;
+
+use reth_codecs::{add_arbitrary_tests, Compact};
+use reth_primitives::TxNumber;
+use serde::{Deserialize, Serialize};
+
+/// Total number of transactions.
+pub type NumTransactions = u64;
+
+/// The storage of the block body indices.
+///
+/// It has the pointer to the transaction Number of the first
+/// transaction in the block and the total number of transactions.
+#[derive(Debug, Default, Eq, PartialEq, Clone, Serialize, Deserialize, Compact)]
+#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
+#[add_arbitrary_tests(compact)]
+pub struct StoredBlockBodyIndices {
+    /// The number of the first transaction in this block
+    ///
+    /// Note: If the block is empty, this is the number of the first transaction
+    /// in the next non-empty block.
+    pub first_tx_num: TxNumber,
+    /// The total number of transactions in the block
+    ///
+    /// NOTE: Number of transitions is equal to number of transactions with
+    /// additional transition for block change if block has block reward or withdrawal.
+    pub tx_count: NumTransactions,
+}
+
+impl StoredBlockBodyIndices {
+    /// Return the range of transaction ids for this block.
+    pub const fn tx_num_range(&self) -> Range<TxNumber> {
+        self.first_tx_num..self.first_tx_num + self.tx_count
+    }
+
+    /// Return the index of last transaction in this block unless the block
+    /// is empty in which case it refers to the last transaction in a previous
+    /// non-empty block
+    pub const fn last_tx_num(&self) -> TxNumber {
+        self.first_tx_num.saturating_add(self.tx_count).saturating_sub(1)
+    }
+
+    /// First transaction index.
+    ///
+    /// Caution: If the block is empty, this is the number of the first transaction
+    /// in the next non-empty block.
+    pub const fn first_tx_num(&self) -> TxNumber {
+        self.first_tx_num
+    }
+
+    /// Return the index of the next transaction after this block.
+    pub const fn next_tx_num(&self) -> TxNumber {
+        self.first_tx_num + self.tx_count
+    }
+
+    /// Return a flag whether the block is empty
+    pub const fn is_empty(&self) -> bool {
+        self.tx_count == 0
+    }
+
+    /// Return number of transaction inside block
+    ///
+    /// NOTE: This is not the same as the number of transitions.
+    pub const fn tx_count(&self) -> NumTransactions {
+        self.tx_count
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::StoredBlockBodyIndices;
+
+    #[test]
+    fn block_indices() {
+        let first_tx_num = 10;
+        let tx_count = 6;
+        let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count };
+
+        assert_eq!(block_indices.first_tx_num(), first_tx_num);
+        assert_eq!(block_indices.last_tx_num(), first_tx_num + tx_count - 1);
+        assert_eq!(block_indices.next_tx_num(), first_tx_num + tx_count);
+        assert_eq!(block_indices.tx_count(), tx_count);
+        assert_eq!(block_indices.tx_num_range(), first_tx_num..first_tx_num + tx_count);
+    }
+}

crates/storage/db-models/src/lib.rs

@@ -0,0 +1,9 @@
+//! Models used in storage module
+
+/// Accounts
+pub mod accounts;
+pub use accounts::AccountBeforeTx;
+
+/// Blocks
+pub mod blocks;
+pub use blocks::StoredBlockBodyIndices;

crates/storage/db/src/tables/mod.rs

@@ -21,11 +21,11 @@ pub(crate) mod utils;
 
 use reth_db_api::{
     models::{
-        accounts::{AccountBeforeTx, BlockNumberAddress},
+        accounts::BlockNumberAddress,
         blocks::{HeaderHash, StoredBlockOmmers},
         client_version::ClientVersion,
         storage_sharded_key::StorageShardedKey,
-        CompactU256, ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals,
+        AccountBeforeTx, CompactU256, ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals,
     },
     table::{Decode, DupSort, Encode, Table},
 };

crates/storage/storage-api/Cargo.toml

@@ -14,8 +14,8 @@ workspace = true
 [dependencies]
 # reth
 reth-chainspec.workspace = true
+reth-db-models.workspace = true
 reth-execution-types.workspace = true
-reth-db-api.workspace = true
 reth-primitives.workspace = true
 reth-prune-types.workspace = true
 reth-stages-types.workspace = true

crates/storage/storage-api/src/account.rs

@@ -1,5 +1,5 @@
 use auto_impl::auto_impl;
-use reth_db_api::models::AccountBeforeTx;
+use reth_db_models::AccountBeforeTx;
 use reth_primitives::{Account, Address, BlockNumber};
 use reth_storage_errors::provider::ProviderResult;
 use std::{

crates/storage/storage-api/src/block.rs

@@ -2,7 +2,7 @@ use crate::{
     BlockIdReader, BlockNumReader, HeaderProvider, ReceiptProvider, ReceiptProviderIdExt,
     RequestsProvider, TransactionVariant, TransactionsProvider, WithdrawalsProvider,
 };
-use reth_db_api::models::StoredBlockBodyIndices;
+use reth_db_models::StoredBlockBodyIndices;
 use reth_primitives::{
     Block, BlockHashOrNumber, BlockId, BlockNumber, BlockNumberOrTag, BlockWithSenders, Header,
     Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, B256,