feat(db): set-upChangeSet tables (#135)

* add AccountChangeSet

* add StorageChangeSet

* add tests to TxNumberAddress

* minor fixes

Cargo.lock

@@ -2578,6 +2578,8 @@ dependencies = [
  "parity-scale-codec",
  "postcard",
  "rand",
+ "reth-codecs",
+ "reth-db",
  "reth-primitives",
  "reth-rpc-types",
  "serde",

crates/db/src/kv/mod.rs

@@ -253,16 +253,30 @@ mod tests {
         env.update(|tx| tx.put::<PlainStorageState>(key, value11.clone()).expect(ERROR_PUT))
             .unwrap();
 
-        // GET DUPSORT
+        // Iterate with cursor
         {
             let tx = env.tx().expect(ERROR_INIT_TX);
             let mut cursor = tx.cursor_dup::<PlainStorageState>().unwrap();
 
             // Notice that value11 and value22 have been ordered in the DB.
             assert!(Some(value00) == cursor.next_dup_val().unwrap());
-            assert!(Some(value11) == cursor.next_dup_val().unwrap());
+            assert!(Some(value11.clone()) == cursor.next_dup_val().unwrap());
             assert!(Some(value22) == cursor.next_dup_val().unwrap());
         }
+
+        // Seek value with subkey
+        {
+            let tx = env.tx().expect(ERROR_INIT_TX);
+            let mut cursor = tx.cursor_dup::<PlainStorageState>().unwrap();
+            let mut walker = cursor.walk_dup(key.into(), H256::from_low_u64_be(1)).unwrap();
+            assert_eq!(
+                value11,
+                walker
+                    .next()
+                    .expect("element should exist.")
+                    .expect("should be able to retrieve it.")
+            );
+        }
     }
 }
 

crates/interfaces/Cargo.toml

@@ -7,6 +7,7 @@ repository = "https://github.com/foundry-rs/reth"
 readme = "README.md"
 
 [dependencies]
+reth-codecs = { path = "../codecs" }
 reth-primitives = { path = "../primitives" }
 reth-rpc-types = { path = "../net/rpc-types" }
 async-trait = "0.1.57"
@@ -27,6 +28,7 @@ rand = "0.8.5"
 arbitrary = { version = "1.1.7", features = ["derive"], optional = true}
 
 [dev-dependencies]
+reth-db = { path = "../db", features = ["test-utils"] }
 test-fuzz = "3.0.4"
 tokio = { version = "1.21.2", features = ["full"] }
 tokio-stream = { version = "0.1.11", features = ["sync"] }

crates/interfaces/src/db/codecs/fuzz/mod.rs

@@ -64,6 +64,6 @@ macro_rules! impl_fuzzer {
     };
 }
 
-impl_fuzzer!(Header, Account, BlockNumHash);
+impl_fuzzer!(Header, Account, BlockNumHash, TxNumberAddress);
 
 impl_fuzzer_with_input!((IntegerList, IntegerListInput));

crates/interfaces/src/db/codecs/scale.rs

@@ -1,4 +1,4 @@
-use crate::db::{Decode, Encode, Error};
+use crate::db::{models::accounts::AccountBeforeTx, Decode, Encode, Error};
 use parity_scale_codec::decode_from_bytes;
 use reth_primitives::*;
 
@@ -37,7 +37,11 @@ macro_rules! impl_scale {
     };
 }
 
-impl_scale!(u16, H256, U256, H160, u8, u64, Header, Account, Log, Receipt, TxType, StorageEntry);
-
 impl ScaleOnly for Vec<u8> {}
 impl sealed::Sealed for Vec<u8> {}
+
+impl_scale!(u8, u32, u16, u64, U256, H256, H160);
+
+impl_scale!(Header, Account, Log, Receipt, TxType, StorageEntry);
+
+impl_scale!(AccountBeforeTx);

crates/interfaces/src/db/mod.rs

@@ -260,3 +260,22 @@ impl<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> std::iter::Iterato
         self.cursor.next_dup_val().transpose()
     }
 }
+
+#[macro_export]
+/// Implements the [`arbitrary::Arbitrary`] trait for types with fixed array types.
+macro_rules! impl_fixed_arbitrary {
+    ($name:tt, $size:tt) => {
+        #[cfg(any(test, feature = "arbitrary"))]
+        use arbitrary::{Arbitrary, Unstructured};
+
+        #[cfg(any(test, feature = "arbitrary"))]
+        impl<'a> Arbitrary<'a> for $name {
+            fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
+                let mut buffer = vec![0; $size];
+                u.fill_buffer(buffer.as_mut_slice())?;
+
+                Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat)
+            }
+        }
+    };
+}

crates/interfaces/src/db/models/accounts.rs

@@ -0,0 +1,107 @@
+//! Account related models and types.
+
+use crate::{
+    db::{
+        table::{Decode, Encode},
+        Error,
+    },
+    impl_fixed_arbitrary,
+};
+use bytes::Bytes;
+use eyre::eyre;
+use reth_codecs::main_codec;
+use reth_primitives::{Account, Address, TxNumber};
+use serde::{Deserialize, Serialize};
+
+/// Account as it is saved inside [`AccountChangeSet`]. [`Address`] is the subkey.
+#[main_codec]
+#[derive(Debug, Default, Clone)]
+pub struct AccountBeforeTx {
+    /// Address for the account. Acts as `DupSort::SubKey`.
+    address: Address,
+    /// Account state before the transaction.
+    info: Account,
+}
+
+/// [`TxNumber`] concatenated with [`Address`]. Used as a key for [`StorageChangeSet`]
+///
+/// Since it's used as a key, it isn't compressed when encoding it.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TxNumberAddress(pub (TxNumber, Address));
+
+impl TxNumberAddress {
+    /// Consumes `Self` and returns [`TxNumber`], [`Address`]
+    pub fn take(self) -> (TxNumber, Address) {
+        (self.0 .0, self.0 .1)
+    }
+}
+
+impl From<(u64, Address)> for TxNumberAddress {
+    fn from(tpl: (u64, Address)) -> Self {
+        TxNumberAddress(tpl)
+    }
+}
+
+impl Encode for TxNumberAddress {
+    type Encoded = [u8; 28];
+
+    fn encode(self) -> Self::Encoded {
+        let tx = self.0 .0;
+        let address = self.0 .1;
+
+        let mut buf = [0u8; 28];
+
+        buf[..8].copy_from_slice(&tx.to_be_bytes());
+        buf[8..].copy_from_slice(address.as_bytes());
+        buf
+    }
+}
+
+impl Decode for TxNumberAddress {
+    fn decode<B: Into<Bytes>>(value: B) -> Result<Self, Error> {
+        let value: bytes::Bytes = value.into();
+
+        let num = u64::from_be_bytes(
+            value.as_ref()[..8]
+                .try_into()
+                .map_err(|_| Error::Decode(eyre!("Into bytes error.")))?,
+        );
+        let hash = Address::decode(value.slice(8..))?;
+
+        Ok(TxNumberAddress((num, hash)))
+    }
+}
+
+impl_fixed_arbitrary!(TxNumberAddress, 28);
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use rand::{thread_rng, Rng};
+    use std::str::FromStr;
+
+    #[test]
+    fn test_tx_number_address() {
+        let num = 1u64;
+        let hash = Address::from_str("ba5e000000000000000000000000000000000000").unwrap();
+        let key = TxNumberAddress((num, hash));
+
+        let mut bytes = [0u8; 28];
+        bytes[..8].copy_from_slice(&num.to_be_bytes());
+        bytes[8..].copy_from_slice(&hash.0);
+
+        let encoded = Encode::encode(key.clone());
+        assert_eq!(encoded, bytes);
+
+        let decoded: TxNumberAddress = Decode::decode(encoded.to_vec()).unwrap();
+        assert_eq!(decoded, key);
+    }
+
+    #[test]
+    fn test_tx_number_address_rand() {
+        let mut bytes = [0u8; 28];
+        thread_rng().fill(bytes.as_mut_slice());
+        let key = TxNumberAddress::arbitrary(&mut Unstructured::new(&bytes)).unwrap();
+        assert_eq!(bytes, Encode::encode(key));
+    }
+}

crates/interfaces/src/db/models/blocks.rs

@@ -1,8 +1,11 @@
 //! Block related models and types.
 
-use crate::db::{
+use crate::{
-    table::{Decode, Encode},
+    db::{
-    Error,
+        table::{Decode, Encode},
+        Error,
+    },
+    impl_fixed_arbitrary,
 };
 use bytes::Bytes;
 use eyre::eyre;
@@ -68,18 +71,7 @@ impl Decode for BlockNumHash {
     }
 }
 
-#[cfg(any(test, feature = "arbitrary"))]
+impl_fixed_arbitrary!(BlockNumHash, 40);
-use arbitrary::{Arbitrary, Unstructured};
-
-#[cfg(any(test, feature = "arbitrary"))]
-impl<'a> Arbitrary<'a> for BlockNumHash {
-    fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
-        let mut buffer = vec![0; 40];
-        u.fill_buffer(buffer.as_mut_slice())?;
-
-        Decode::decode(buffer).map_err(|_| arbitrary::Error::IncorrectFormat)
-    }
-}
 
 #[cfg(test)]
 mod test {
@@ -97,10 +89,10 @@ mod test {
         bytes[8..].copy_from_slice(&hash.0);
 
         let encoded = Encode::encode(key.clone());
-        assert!(encoded == bytes);
+        assert_eq!(encoded, bytes);
 
         let decoded: BlockNumHash = Decode::decode(encoded.to_vec()).unwrap();
-        assert!(decoded == key);
+        assert_eq!(decoded, key);
     }
 
     #[test]
@@ -108,6 +100,6 @@ mod test {
         let mut bytes = [0u8; 40];
         thread_rng().fill(bytes.as_mut_slice());
         let key = BlockNumHash::arbitrary(&mut Unstructured::new(&bytes)).unwrap();
-        assert!(bytes == Encode::encode(key));
+        assert_eq!(bytes, Encode::encode(key));
     }
 }

crates/interfaces/src/db/models/mod.rs

@@ -1,6 +1,8 @@
 //! Implements data structures specific to the database
 
+pub mod accounts;
 pub mod blocks;
 pub mod integer_list;
 
+pub use accounts::*;
 pub use blocks::*;

crates/interfaces/src/db/tables.rs

@@ -1,11 +1,14 @@
 //! Declaration of all Database tables.
 
 use crate::db::{
-    models::blocks::{BlockNumHash, HeaderHash, NumTransactions, NumTxesInBlock},
+    models::{
+        accounts::{AccountBeforeTx, TxNumberAddress},
+        blocks::{BlockNumHash, HeaderHash, NumTransactions, NumTxesInBlock},
+    },
     DupSort,
 };
 use reth_primitives::{
-    Account, Address, BlockNumber, Header, IntegerList, Receipt, StorageEntry, H256,
+    Account, Address, BlockNumber, Header, IntegerList, Receipt, StorageEntry, TxNumber, H256,
 };
 
 /// Enum for the type of table present in libmdbx.
@@ -105,8 +108,8 @@ dupsort!(PlainStorageState => Address => [H256] StorageEntry);
 table!(AccountHistory => Address => TxNumberList);
 table!(StorageHistory => Address_StorageKey => TxNumberList);
 
-table!(AccountChangeSet => TxNumber => AccountBeforeTx);
+dupsort!(AccountChangeSet => TxNumber => [Address] AccountBeforeTx);
-table!(StorageChangeSet => TxNumber => StorageKeyBeforeTx);
+dupsort!(StorageChangeSet => TxNumberAddress => [H256] StorageEntry);
 
 table!(TxSenders => TxNumber => Address); // Is it necessary?
 table!(Config => ConfigKey => ConfigValue);
@@ -117,7 +120,6 @@ table!(SyncStage => StageId => BlockNumber);
 /// Alias Types
 
 type TxNumberList = IntegerList;
-type TxNumber = u64;
 
 //
 // TODO: Temporary types, until they're properly defined alongside with the Encode and Decode Trait
@@ -131,6 +133,4 @@ type RlpTotalDifficulty = Vec<u8>;
 type RlpTxBody = Vec<u8>;
 #[allow(non_camel_case_types)]
 type Address_StorageKey = Vec<u8>;
-type AccountBeforeTx = Vec<u8>;
-type StorageKeyBeforeTx = Vec<u8>;
 type StageId = Vec<u8>;

crates/primitives/src/lib.rs

@@ -42,6 +42,8 @@ pub type Address = H160;
 pub type BlockID = H256;
 /// TxHash is Kecack hash of rlp encoded signed transaction
 pub type TxHash = H256;
+/// TxNumber is sequence number of all existing transactions
+pub type TxNumber = u64;
 
 /// Storage Key
 pub type StorageKey = H256;