From f37ee4b83bfd70ef3afc43d6d35d2983ef49b135 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 16 May 2023 21:40:31 +0200 Subject: [PATCH] feat: add CachedReads and CachedReadsDbRef (#2596) --- Cargo.lock | 176 +++++++++---------------- crates/payload/builder/Cargo.toml | 1 + crates/payload/builder/src/database.rs | 158 ++++++++++++++++++++++ crates/payload/builder/src/lib.rs | 1 + 4 files changed, 222 insertions(+), 114 deletions(-) create mode 100644 crates/payload/builder/src/database.rs diff --git a/Cargo.lock b/Cargo.lock index fc33d1c5a..1550df04b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,8 +188,6 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "zstd 0.11.2+zstd.1.5.2", - "zstd-safe 5.0.2+zstd.1.5.2", ] [[package]] @@ -318,12 +316,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" - [[package]] name = "base64" version = "0.21.0" @@ -2678,9 +2670,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", @@ -2689,6 +2681,7 @@ dependencies = [ "rustls-native-certs", "tokio", "tokio-rustls", + "webpki-roots", ] [[package]] @@ -2908,12 +2901,11 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iri-string" -version = "0.7.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21859b667d66a4c1dacd9df0863b3efb65785474255face87f5bca39dd8407c0" +checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78" dependencies = [ - "memchr", - "serde", + "nom", ] [[package]] @@ -2983,9 +2975,9 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1822d18e4384a5e79d94dc9e4d1239cfa9fad24e55b44d2efeff5b394c9fece4" +checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -3000,15 +2992,18 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11aa5766d5c430b89cb26a99b88f3245eb91534be8126102cea9e45ee3891b22" +checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" dependencies = [ + "anyhow", "futures-channel", + "futures-timer", "futures-util", "gloo-net", "http", "jsonrpsee-core", + "jsonrpsee-types", "pin-project", "rustls-native-certs", "soketto", @@ -3022,14 +3017,16 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c6832a55f662b5a6ecc844db24b8b9c387453f923de863062c60ce33d62b81" +checksum = "a4e70b4439a751a5de7dd5ed55eacff78ebf4ffe0fc009cb1ebb11417f5b536b" dependencies = [ "anyhow", + "arrayvec", "async-lock", "async-trait", "beef", + "futures-channel", "futures-timer", "futures-util", "globset", @@ -3043,35 +3040,34 @@ dependencies = [ "soketto", "thiserror", "tokio", - "tokio-stream", "tracing", "wasm-bindgen-futures", ] [[package]] name = "jsonrpsee-http-client" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1705c65069729e3dccff6fd91ee431d5d31cabcf00ce68a62a2c6435ac713af9" +checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad" dependencies = [ "async-trait", "hyper", "hyper-rustls", "jsonrpsee-core", "jsonrpsee-types", + "rustc-hash", "serde", "serde_json", "thiserror", "tokio", - "tower", "tracing", ] [[package]] name = "jsonrpsee-proc-macros" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6027ac0b197ce9543097d02a290f550ce1d9432bf301524b013053c0b75cc94" +checksum = "baa6da1e4199c10d7b1d0a6e5e8bd8e55f351163b6f4b3cbb044672a69bd4c1c" dependencies = [ "heck", "proc-macro-crate", @@ -3082,11 +3078,13 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f06661d1a6b6e5b85469dc9c29acfbb9b3bb613797a6fd10a3ebb8a70754057" +checksum = "1fb69dad85df79527c019659a992498d03f8495390496da2f07e6c24c2b356fc" dependencies = [ + "futures-channel", "futures-util", + "http", "hyper", "jsonrpsee-core", "jsonrpsee-types", @@ -3102,9 +3100,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5bf6c75ce2a4217421154adfc65a24d2b46e77286e59bba5d9fa6544ccc8f4" +checksum = "5bd522fe1ce3702fd94812965d7bb7a3364b1c9aba743944c5a00529aae80f8c" dependencies = [ "anyhow", "beef", @@ -3116,9 +3114,9 @@ dependencies = [ [[package]] name = "jsonrpsee-wasm-client" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e6ea7c6d862e60f8baebd946c037b70c6808a4e4e31e792a4029184e3ce13a" +checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -3127,9 +3125,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.18.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64b2589680ba1ad7863f279cd2d5083c1dc0a7c0ea959d22924553050f8ab9f" +checksum = "0b83daeecfc6517cfe210df24e570fb06213533dfb990318fae781f4c7119dd9" dependencies = [ "http", "jsonrpsee-client-transport", @@ -4969,7 +4967,6 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tokio-stream", "tokio-util 0.7.7", "tower", "tracing", @@ -5121,6 +5118,7 @@ name = "reth-payload-builder" version = "0.1.0" dependencies = [ "futures-util", + "hashbrown 0.13.2", "metrics", "reth-interfaces", "reth-metrics-derive", @@ -5178,7 +5176,6 @@ dependencies = [ "tracing", "triehash", "url", - "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -5206,7 +5203,6 @@ dependencies = [ name = "reth-revm" version = "0.1.0" dependencies = [ - "once_cell", "reth-consensus-common", "reth-interfaces", "reth-primitives", @@ -5549,9 +5545,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "2.0.3" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41320af3bd6a65153d38eb1d3638ba89104cc9513c7feedb2d8510e8307dab29" +checksum = "10a3eabf08ea9e4063f5531b8735e29344d9d6eaebaa314c58253f6c17fcdf2d" dependencies = [ "k256 0.13.1", "num", @@ -5743,14 +5739,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.1" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", - "rustls-webpki", "sct", + "webpki", ] [[package]] @@ -5774,16 +5770,6 @@ dependencies = [ "base64 0.21.0", ] -[[package]] -name = "rustls-webpki" -version = "0.100.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustversion" version = "1.0.11" @@ -6737,12 +6723,13 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.0" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", "tokio", + "webpki", ] [[package]] @@ -6879,12 +6866,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "async-compression", - "base64 0.20.0", + "base64 0.13.1", "bitflags", "bytes", "futures-core", @@ -6956,9 +6943,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -7000,9 +6987,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ "matchers", "nu-ansi-term", @@ -7482,12 +7469,22 @@ dependencies = [ ] [[package]] -name = "webpki-roots" -version = "0.23.0" +name = "webpki" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "rustls-webpki", + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", ] [[package]] @@ -7766,52 +7763,3 @@ dependencies = [ "quote 1.0.26", "syn 2.0.15", ] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - -[[package]] -name = "zstd" -version = "0.12.3+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" -dependencies = [ - "zstd-safe 6.0.5+zstd.1.5.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/crates/payload/builder/Cargo.toml b/crates/payload/builder/Cargo.toml index f5028283b..ed7f526a4 100644 --- a/crates/payload/builder/Cargo.toml +++ b/crates/payload/builder/Cargo.toml @@ -31,6 +31,7 @@ futures-util = "0.3" thiserror = "1.0" sha2 = { version = "0.10", default-features = false } tracing = { workspace = true } +hashbrown = "0.13" [features] diff --git a/crates/payload/builder/src/database.rs b/crates/payload/builder/src/database.rs new file mode 100644 index 000000000..9a7263ec5 --- /dev/null +++ b/crates/payload/builder/src/database.rs @@ -0,0 +1,158 @@ +//! Database adapters for payload building. + +use hashbrown::{hash_map::Entry, HashMap}; +use reth_primitives::U256; +use revm_primitives::{ + db::{Database, DatabaseRef}, + AccountInfo, Address, Bytecode, B256, +}; +use std::cell::RefCell; + +/// A container type that caches all [DatabaseRef] reads from an underlying [DatabaseRef]. +/// +/// This is intended to be used in conjunction with [CacheDB](reth_revm_primitives::db::CacheDB) +/// during payload building which repeatedly accesses the same data. +/// +/// # Example +/// +/// ``` +/// use revm_primitives::db::DatabaseRef; +/// use reth_payload_builder::database::CachedReads; +/// use reth_revm_primitives::db::CacheDB; +/// +/// fn build_payload(db: DB) { +/// let mut cached_reads = CachedReads::default(); +/// let db_ref = cached_reads.as_db(db); +/// +/// // this is `Database` and can be used to build a payload, it never writes to `CachedReads` or the underlying database, but all reads from the underlying database are cached in `CachedReads`. +/// // Subsequent payload build attempts can use cached reads and avoid hitting the underlying database. +/// let db = CacheDB::new(db_ref); +/// +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct CachedReads { + accounts: HashMap, + contracts: HashMap, + block_hashes: HashMap, +} + +// === impl CachedReads === + +impl CachedReads { + /// Gets a [DatabaseRef] that will cache reads from the given database. + pub fn as_db(&mut self, db: DB) -> CachedReadsDBRef<'_, DB> { + CachedReadsDBRef { inner: RefCell::new(self.as_db_mut(db)) } + } + + fn as_db_mut(&mut self, db: DB) -> CachedReadsDbMut<'_, DB> { + CachedReadsDbMut { cached: self, db } + } +} + +#[derive(Debug)] +struct CachedReadsDbMut<'a, DB> { + cached: &'a mut CachedReads, + db: DB, +} + +impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> { + type Error = ::Error; + + fn basic(&mut self, address: Address) -> Result, Self::Error> { + let basic = match self.cached.accounts.entry(address) { + Entry::Occupied(entry) => entry.get().info.clone(), + Entry::Vacant(entry) => { + entry.insert(CachedAccount::new(self.db.basic(address)?)).info.clone() + } + }; + Ok(basic) + } + + fn code_by_hash(&mut self, code_hash: B256) -> Result { + let code = match self.cached.contracts.entry(code_hash) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => entry.insert(self.db.code_by_hash(code_hash)?).clone(), + }; + Ok(code) + } + + fn storage(&mut self, address: Address, index: U256) -> Result { + match self.cached.accounts.entry(address) { + Entry::Occupied(mut acc_entry) => { + let acc_entry = acc_entry.get_mut(); + match acc_entry.storage.entry(index) { + Entry::Occupied(entry) => Ok(*entry.get()), + Entry::Vacant(entry) => { + let slot = self.db.storage(address, index)?; + entry.insert(slot); + Ok(slot) + } + } + } + Entry::Vacant(acc_entry) => { + // acc needs to be loaded for us to access slots. + let info = self.db.basic(address)?; + let (account, value) = if info.is_some() { + let value = self.db.storage(address, index)?; + let mut account = CachedAccount::new(info); + account.storage.insert(index, value); + (account, value) + } else { + (CachedAccount::new(info), U256::ZERO) + }; + acc_entry.insert(account); + Ok(value) + } + } + } + + fn block_hash(&mut self, number: U256) -> Result { + let code = match self.cached.block_hashes.entry(number) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => *entry.insert(self.db.block_hash(number)?), + }; + Ok(code) + } +} + +/// A [DatabaseRef] that caches reads inside [CachedReads]. +/// +/// This is intended to be used as the [DatabaseRef] for +/// [CacheDB](reth_revm_primitives::db::CacheDB) for repeated payload build jobs. +#[derive(Debug)] +pub struct CachedReadsDBRef<'a, DB> { + inner: RefCell>, +} + +impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> { + type Error = ::Error; + + fn basic(&self, address: Address) -> Result, Self::Error> { + self.inner.borrow_mut().basic(address) + } + + fn code_by_hash(&self, code_hash: B256) -> Result { + self.inner.borrow_mut().code_by_hash(code_hash) + } + + fn storage(&self, address: Address, index: U256) -> Result { + self.inner.borrow_mut().storage(address, index) + } + + fn block_hash(&self, number: U256) -> Result { + self.inner.borrow_mut().block_hash(number) + } +} + +#[derive(Debug, Clone)] +struct CachedAccount { + info: Option, + storage: HashMap, +} + +impl CachedAccount { + fn new(info: Option) -> Self { + Self { info, storage: HashMap::new() } + } +} diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index 4247f872e..2c91a2abf 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -17,6 +17,7 @@ //! on [PayloadAttributes](reth_rpc_types::engine::PayloadAttributes). //! - [PayloadJob]: a type that can yields (better) payloads over time. +pub mod database; pub mod error; mod metrics; mod payload;