feat: add CachedReads and CachedReadsDbRef (#2596)

This commit is contained in:
Matthias Seitz
2023-05-16 21:40:31 +02:00
committed by GitHub
parent c44117075e
commit f37ee4b83b
4 changed files with 222 additions and 114 deletions

176
Cargo.lock generated
View File

@ -188,8 +188,6 @@ dependencies = [
"memchr",
"pin-project-lite",
"tokio",
"zstd 0.11.2+zstd.1.5.2",
"zstd-safe 5.0.2+zstd.1.5.2",
]
[[package]]
@ -318,12 +316,6 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
[[package]]
name = "base64"
version = "0.21.0"
@ -2678,9 +2670,9 @@ dependencies = [
[[package]]
name = "hyper-rustls"
version = "0.24.0"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7"
checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
dependencies = [
"http",
"hyper",
@ -2689,6 +2681,7 @@ dependencies = [
"rustls-native-certs",
"tokio",
"tokio-rustls",
"webpki-roots",
]
[[package]]
@ -2908,12 +2901,11 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
[[package]]
name = "iri-string"
version = "0.7.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21859b667d66a4c1dacd9df0863b3efb65785474255face87f5bca39dd8407c0"
checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78"
dependencies = [
"memchr",
"serde",
"nom",
]
[[package]]
@ -2983,9 +2975,9 @@ dependencies = [
[[package]]
name = "jsonrpsee"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1822d18e4384a5e79d94dc9e4d1239cfa9fad24e55b44d2efeff5b394c9fece4"
checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e"
dependencies = [
"jsonrpsee-client-transport",
"jsonrpsee-core",
@ -3000,15 +2992,18 @@ dependencies = [
[[package]]
name = "jsonrpsee-client-transport"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11aa5766d5c430b89cb26a99b88f3245eb91534be8126102cea9e45ee3891b22"
checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb"
dependencies = [
"anyhow",
"futures-channel",
"futures-timer",
"futures-util",
"gloo-net",
"http",
"jsonrpsee-core",
"jsonrpsee-types",
"pin-project",
"rustls-native-certs",
"soketto",
@ -3022,14 +3017,16 @@ dependencies = [
[[package]]
name = "jsonrpsee-core"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c6832a55f662b5a6ecc844db24b8b9c387453f923de863062c60ce33d62b81"
checksum = "a4e70b4439a751a5de7dd5ed55eacff78ebf4ffe0fc009cb1ebb11417f5b536b"
dependencies = [
"anyhow",
"arrayvec",
"async-lock",
"async-trait",
"beef",
"futures-channel",
"futures-timer",
"futures-util",
"globset",
@ -3043,35 +3040,34 @@ dependencies = [
"soketto",
"thiserror",
"tokio",
"tokio-stream",
"tracing",
"wasm-bindgen-futures",
]
[[package]]
name = "jsonrpsee-http-client"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1705c65069729e3dccff6fd91ee431d5d31cabcf00ce68a62a2c6435ac713af9"
checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad"
dependencies = [
"async-trait",
"hyper",
"hyper-rustls",
"jsonrpsee-core",
"jsonrpsee-types",
"rustc-hash",
"serde",
"serde_json",
"thiserror",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "jsonrpsee-proc-macros"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6027ac0b197ce9543097d02a290f550ce1d9432bf301524b013053c0b75cc94"
checksum = "baa6da1e4199c10d7b1d0a6e5e8bd8e55f351163b6f4b3cbb044672a69bd4c1c"
dependencies = [
"heck",
"proc-macro-crate",
@ -3082,11 +3078,13 @@ dependencies = [
[[package]]
name = "jsonrpsee-server"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f06661d1a6b6e5b85469dc9c29acfbb9b3bb613797a6fd10a3ebb8a70754057"
checksum = "1fb69dad85df79527c019659a992498d03f8495390496da2f07e6c24c2b356fc"
dependencies = [
"futures-channel",
"futures-util",
"http",
"hyper",
"jsonrpsee-core",
"jsonrpsee-types",
@ -3102,9 +3100,9 @@ dependencies = [
[[package]]
name = "jsonrpsee-types"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e5bf6c75ce2a4217421154adfc65a24d2b46e77286e59bba5d9fa6544ccc8f4"
checksum = "5bd522fe1ce3702fd94812965d7bb7a3364b1c9aba743944c5a00529aae80f8c"
dependencies = [
"anyhow",
"beef",
@ -3116,9 +3114,9 @@ dependencies = [
[[package]]
name = "jsonrpsee-wasm-client"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34e6ea7c6d862e60f8baebd946c037b70c6808a4e4e31e792a4029184e3ce13a"
checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d"
dependencies = [
"jsonrpsee-client-transport",
"jsonrpsee-core",
@ -3127,9 +3125,9 @@ dependencies = [
[[package]]
name = "jsonrpsee-ws-client"
version = "0.18.2"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a64b2589680ba1ad7863f279cd2d5083c1dc0a7c0ea959d22924553050f8ab9f"
checksum = "0b83daeecfc6517cfe210df24e570fb06213533dfb990318fae781f4c7119dd9"
dependencies = [
"http",
"jsonrpsee-client-transport",
@ -4969,7 +4967,6 @@ dependencies = [
"serde_json",
"thiserror",
"tokio",
"tokio-stream",
"tokio-util 0.7.7",
"tower",
"tracing",
@ -5121,6 +5118,7 @@ name = "reth-payload-builder"
version = "0.1.0"
dependencies = [
"futures-util",
"hashbrown 0.13.2",
"metrics",
"reth-interfaces",
"reth-metrics-derive",
@ -5178,7 +5176,6 @@ dependencies = [
"tracing",
"triehash",
"url",
"zstd 0.12.3+zstd.1.5.2",
]
[[package]]
@ -5206,7 +5203,6 @@ dependencies = [
name = "reth-revm"
version = "0.1.0"
dependencies = [
"once_cell",
"reth-consensus-common",
"reth-interfaces",
"reth-primitives",
@ -5549,9 +5545,9 @@ dependencies = [
[[package]]
name = "revm-precompile"
version = "2.0.3"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41320af3bd6a65153d38eb1d3638ba89104cc9513c7feedb2d8510e8307dab29"
checksum = "10a3eabf08ea9e4063f5531b8735e29344d9d6eaebaa314c58253f6c17fcdf2d"
dependencies = [
"k256 0.13.1",
"num",
@ -5743,14 +5739,14 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.21.1"
version = "0.20.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e"
checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
"webpki",
]
[[package]]
@ -5774,16 +5770,6 @@ dependencies = [
"base64 0.21.0",
]
[[package]]
name = "rustls-webpki"
version = "0.100.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.11"
@ -6737,12 +6723,13 @@ dependencies = [
[[package]]
name = "tokio-rustls"
version = "0.24.0"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5"
checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
dependencies = [
"rustls",
"tokio",
"webpki",
]
[[package]]
@ -6879,12 +6866,12 @@ dependencies = [
[[package]]
name = "tower-http"
version = "0.4.0"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
dependencies = [
"async-compression",
"base64 0.20.0",
"base64 0.13.1",
"bitflags",
"bytes",
"futures-core",
@ -6956,9 +6943,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.31"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
dependencies = [
"once_cell",
"valuable",
@ -7000,9 +6987,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.17"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
dependencies = [
"matchers",
"nu-ansi-term",
@ -7482,12 +7469,22 @@ dependencies = [
]
[[package]]
name = "webpki-roots"
version = "0.23.0"
name = "webpki"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125"
checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
dependencies = [
"rustls-webpki",
"ring",
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
dependencies = [
"webpki",
]
[[package]]
@ -7766,52 +7763,3 @@ dependencies = [
"quote 1.0.26",
"syn 2.0.15",
]
[[package]]
name = "zstd"
version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
"zstd-safe 5.0.2+zstd.1.5.2",
]
[[package]]
name = "zstd"
version = "0.12.3+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
dependencies = [
"zstd-safe 6.0.5+zstd.1.5.4",
]
[[package]]
name = "zstd-safe"
version = "5.0.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "6.0.5+zstd.1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.8+zstd.1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
dependencies = [
"cc",
"libc",
"pkg-config",
]

View File

@ -31,6 +31,7 @@ futures-util = "0.3"
thiserror = "1.0"
sha2 = { version = "0.10", default-features = false }
tracing = { workspace = true }
hashbrown = "0.13"
[features]

View File

@ -0,0 +1,158 @@
//! Database adapters for payload building.
use hashbrown::{hash_map::Entry, HashMap};
use reth_primitives::U256;
use revm_primitives::{
db::{Database, DatabaseRef},
AccountInfo, Address, Bytecode, B256,
};
use std::cell::RefCell;
/// A container type that caches all [DatabaseRef] reads from an underlying [DatabaseRef].
///
/// This is intended to be used in conjunction with [CacheDB](reth_revm_primitives::db::CacheDB)
/// during payload building which repeatedly accesses the same data.
///
/// # Example
///
/// ```
/// use revm_primitives::db::DatabaseRef;
/// use reth_payload_builder::database::CachedReads;
/// use reth_revm_primitives::db::CacheDB;
///
/// fn build_payload<DB: DatabaseRef>(db: DB) {
/// let mut cached_reads = CachedReads::default();
/// let db_ref = cached_reads.as_db(db);
///
/// // this is `Database` and can be used to build a payload, it never writes to `CachedReads` or the underlying database, but all reads from the underlying database are cached in `CachedReads`.
/// // Subsequent payload build attempts can use cached reads and avoid hitting the underlying database.
/// let db = CacheDB::new(db_ref);
///
/// }
/// ```
#[derive(Debug, Clone, Default)]
pub struct CachedReads {
accounts: HashMap<Address, CachedAccount>,
contracts: HashMap<B256, Bytecode>,
block_hashes: HashMap<U256, B256>,
}
// === impl CachedReads ===
impl CachedReads {
/// Gets a [DatabaseRef] that will cache reads from the given database.
pub fn as_db<DB>(&mut self, db: DB) -> CachedReadsDBRef<'_, DB> {
CachedReadsDBRef { inner: RefCell::new(self.as_db_mut(db)) }
}
fn as_db_mut<DB>(&mut self, db: DB) -> CachedReadsDbMut<'_, DB> {
CachedReadsDbMut { cached: self, db }
}
}
#[derive(Debug)]
struct CachedReadsDbMut<'a, DB> {
cached: &'a mut CachedReads,
db: DB,
}
impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> {
type Error = <DB as DatabaseRef>::Error;
fn basic(&mut self, address: Address) -> Result<Option<AccountInfo>, Self::Error> {
let basic = match self.cached.accounts.entry(address) {
Entry::Occupied(entry) => entry.get().info.clone(),
Entry::Vacant(entry) => {
entry.insert(CachedAccount::new(self.db.basic(address)?)).info.clone()
}
};
Ok(basic)
}
fn code_by_hash(&mut self, code_hash: B256) -> Result<Bytecode, Self::Error> {
let code = match self.cached.contracts.entry(code_hash) {
Entry::Occupied(entry) => entry.get().clone(),
Entry::Vacant(entry) => entry.insert(self.db.code_by_hash(code_hash)?).clone(),
};
Ok(code)
}
fn storage(&mut self, address: Address, index: U256) -> Result<U256, Self::Error> {
match self.cached.accounts.entry(address) {
Entry::Occupied(mut acc_entry) => {
let acc_entry = acc_entry.get_mut();
match acc_entry.storage.entry(index) {
Entry::Occupied(entry) => Ok(*entry.get()),
Entry::Vacant(entry) => {
let slot = self.db.storage(address, index)?;
entry.insert(slot);
Ok(slot)
}
}
}
Entry::Vacant(acc_entry) => {
// acc needs to be loaded for us to access slots.
let info = self.db.basic(address)?;
let (account, value) = if info.is_some() {
let value = self.db.storage(address, index)?;
let mut account = CachedAccount::new(info);
account.storage.insert(index, value);
(account, value)
} else {
(CachedAccount::new(info), U256::ZERO)
};
acc_entry.insert(account);
Ok(value)
}
}
}
fn block_hash(&mut self, number: U256) -> Result<B256, Self::Error> {
let code = match self.cached.block_hashes.entry(number) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => *entry.insert(self.db.block_hash(number)?),
};
Ok(code)
}
}
/// A [DatabaseRef] that caches reads inside [CachedReads].
///
/// This is intended to be used as the [DatabaseRef] for
/// [CacheDB](reth_revm_primitives::db::CacheDB) for repeated payload build jobs.
#[derive(Debug)]
pub struct CachedReadsDBRef<'a, DB> {
inner: RefCell<CachedReadsDbMut<'a, DB>>,
}
impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> {
type Error = <DB as DatabaseRef>::Error;
fn basic(&self, address: Address) -> Result<Option<AccountInfo>, Self::Error> {
self.inner.borrow_mut().basic(address)
}
fn code_by_hash(&self, code_hash: B256) -> Result<Bytecode, Self::Error> {
self.inner.borrow_mut().code_by_hash(code_hash)
}
fn storage(&self, address: Address, index: U256) -> Result<U256, Self::Error> {
self.inner.borrow_mut().storage(address, index)
}
fn block_hash(&self, number: U256) -> Result<B256, Self::Error> {
self.inner.borrow_mut().block_hash(number)
}
}
#[derive(Debug, Clone)]
struct CachedAccount {
info: Option<AccountInfo>,
storage: HashMap<U256, U256>,
}
impl CachedAccount {
fn new(info: Option<AccountInfo>) -> Self {
Self { info, storage: HashMap::new() }
}
}

View File

@ -17,6 +17,7 @@
//! on [PayloadAttributes](reth_rpc_types::engine::PayloadAttributes).
//! - [PayloadJob]: a type that can yields (better) payloads over time.
pub mod database;
pub mod error;
mod metrics;
mod payload;