diff --git a/Cargo.lock b/Cargo.lock index 3110f8ff0..0a831ae80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -514,7 +514,7 @@ version = "0.1.0" source = "git+https://github.com/alloy-rs/alloy?rev=ca54552#ca54552075da02339f678e5b591877ff6c2939db" dependencies = [ "alloy-json-rpc", - "base64 0.22.0", + "base64 0.22.1", "futures-util", "futures-utils-wasm", "serde", @@ -550,7 +550,7 @@ dependencies = [ "arbitrary", "derive_arbitrary", "derive_more", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "nybbles", "proptest", "proptest-derive", @@ -1006,9 +1006,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" @@ -1220,7 +1220,7 @@ dependencies = [ "cfg-if", "dashmap", "fast-float", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "icu_normalizer", "indexmap 2.2.6", "intrusive-collections", @@ -1255,7 +1255,7 @@ checksum = "c055ef3cd87ea7db014779195bc90c6adfc35de4902e3b2fe587adecbd384578" dependencies = [ "boa_macros", "boa_profiler", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "thin-vec", ] @@ -1267,7 +1267,7 @@ checksum = "0cacc9caf022d92195c827a3e5bf83f96089d4bfaff834b359ac7b6be46e9187" dependencies = [ "boa_gc", "boa_macros", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indexmap 2.2.6", "once_cell", "phf", @@ -1479,9 +1479,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" dependencies = [ "jobserver", "libc", @@ -2248,7 +2248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core 0.9.10", @@ -2256,15 +2256,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "data-encoding-macro" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c01c06f5f429efdf2bae21eb67c28b3df3cf85b7dd2d8ef09c0838dac5d33e" +checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -2272,9 +2272,9 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0047d07f2c89b17dd631c80450d69841a6b5d7fb17278cbc43d7e4cfcf2576f3" +checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" dependencies = [ "data-encoding", "syn 1.0.109", @@ -3015,9 +3015,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.29" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4556222738635b7a3417ae6130d8f52201e45a0c4d1a907f0826383adb5f85e7" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -3364,9 +3364,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -3379,7 +3379,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3388,7 +3388,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3626,7 +3626,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.5.7", "tokio", "tower-service", "tracing", @@ -3712,7 +3712,7 @@ dependencies = [ "http-body 1.0.0", "hyper 1.3.1", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.5.7", "tokio", "tower", "tower-service", @@ -4032,7 +4032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] @@ -4127,7 +4127,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.6", + "socket2 0.5.7", "widestring", "windows-sys 0.48.0", "winreg 0.50.0", @@ -4204,9 +4204,9 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4b0e68d9af1f066c06d6e2397583795b912d78537d7d907c561e82c13d69fa1" +checksum = "cfdb12a2381ea5b2e68c3469ec604a007b367778cdb14d09612c8069ebd616ad" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4222,9 +4222,9 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92f254f56af1ae84815b9b1325094743dcf05b92abb5e94da2e81a35cff0cada" +checksum = "4978087a58c3ab02efc5b07c5e5e2803024536106fd5506f558db172c889b3aa" dependencies = [ "futures-channel", "futures-util", @@ -4246,9 +4246,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "274d68152c24aa78977243bb56f28d7946e6aa309945b37d33174a3f92d89a3a" +checksum = "b4b257e1ec385e07b0255dde0b933f948b5c8b8c28d42afda9587c3a967b896d" dependencies = [ "anyhow", "async-trait", @@ -4272,9 +4272,9 @@ dependencies = [ [[package]] name = "jsonrpsee-http-client" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac13bc1e44cd00448a5ff485824a128629c945f02077804cb659c07a0ba41395" +checksum = "1ccf93fc4a0bfe05d851d37d7c32b7f370fe94336b52a2f0efc5f1981895c2e5" dependencies = [ "async-trait", "hyper 0.14.28", @@ -4292,9 +4292,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c326f9e95aeff7d707b2ffde72c22a52acc975ba1c48587776c02b90c4747a6" +checksum = "7d0bb047e79a143b32ea03974a6bf59b62c2a4c5f5d42a381c907a8bbb3f75c0" dependencies = [ "heck 0.4.1", "proc-macro-crate 3.1.0", @@ -4305,9 +4305,9 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b5bfbda5f8fb63f997102fd18f73e35e34c84c6dcdbdbbe72c6e48f6d2c959b" +checksum = "12d8b6a9674422a8572e0b0abb12feeb3f2aeda86528c80d0350c2bd0923ab41" dependencies = [ "futures-util", "http 0.2.12", @@ -4329,9 +4329,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc828e537868d6b12bbb07ec20324909a22ced6efca0057c825c3e1126b2c6d" +checksum = "150d6168405890a7a3231a3c74843f58b8959471f6df76078db2619ddee1d07d" dependencies = [ "anyhow", "beef", @@ -4342,9 +4342,9 @@ dependencies = [ [[package]] name = "jsonrpsee-wasm-client" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cf8dcee48f383e24957e238240f997ec317ba358b4e6d2e8be3f745bcdabdb5" +checksum = "f448d8eacd945cc17b6c0b42c361531ca36a962ee186342a97cdb8fca679cd77" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4353,9 +4353,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f00abe918bf34b785f87459b9205790e5361a3f7437adb50e928dc243f27eb" +checksum = "58b9db2dfd5bb1194b0ce921504df9ceae210a345bc2f6c5a61432089bbab070" dependencies = [ "http 0.2.12", "jsonrpsee-client-transport", @@ -4443,9 +4443,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libffi" @@ -4727,7 +4727,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -5476,7 +5476,7 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "serde", ] @@ -6178,7 +6178,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eae2a1ebfecc58aff952ef8ccd364329abe627762f5bf09ff42eb9d98522479" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", "memchr", ] @@ -6226,7 +6226,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6481,6 +6481,7 @@ name = "reth-codecs" version = "0.2.0-beta.6" dependencies = [ "alloy-eips", + "alloy-genesis", "alloy-primitives", "arbitrary", "bytes", @@ -7220,12 +7221,14 @@ dependencies = [ "proptest", "rand 0.8.5", "reth-beacon-consensus", + "reth-codecs", "reth-config", "reth-consensus-common", "reth-db", "reth-discv4", "reth-discv5", "reth-engine-primitives", + "reth-etl", "reth-evm", "reth-interfaces", "reth-metrics", @@ -8003,7 +8006,7 @@ dependencies = [ "derive_more", "dyn-clone", "enumn", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", "once_cell", "serde", @@ -8088,9 +8091,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1c77081a55300e016cb86f2864415b7518741879db925b8d488a0ee0d2da6bf" +checksum = "b26f4c25a604fcb3a1bcd96dd6ba37c93840de95de8198d94c0d571a74a804d1" dependencies = [ "bytemuck", "byteorder", @@ -8285,7 +8288,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] @@ -8582,11 +8585,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c85f8e96d1d6857f13768fcbd895fcb06225510022a2774ed8b5150581847b0" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -8600,9 +8603,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b3a576c4eb2924262d5951a3b737ccaf16c931e39a2810c36f9a7e25575557" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling 0.20.8", "proc-macro2", @@ -8848,9 +8851,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -9359,7 +9362,7 @@ dependencies = [ "parking_lot 0.12.2", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.6", + "socket2 0.5.7", "tokio-macros", "windows-sys 0.48.0", ] diff --git a/bin/reth/src/commands/init_state.rs b/bin/reth/src/commands/init_state.rs index fa70264e5..e0558be32 100644 --- a/bin/reth/src/commands/init_state.rs +++ b/bin/reth/src/commands/init_state.rs @@ -8,6 +8,7 @@ use crate::{ dirs::{DataDirPath, MaybePlatformPath}, }; use clap::Parser; +use reth_config::config::EtlConfig; use reth_db::{database::Database, init_db}; use reth_node_core::init::{init_from_state_dump, init_genesis}; use reth_primitives::{ChainSpec, B256}; @@ -78,11 +79,15 @@ impl InitStateCommand { info!(target: "reth::cli", "Database opened"); let provider_factory = ProviderFactory::new(db, self.chain, data_dir.static_files())?; + let etl_config = EtlConfig::new( + Some(EtlConfig::from_datadir(data_dir.data_dir())), + EtlConfig::default_file_size(), + ); info!(target: "reth::cli", "Writing genesis block"); let hash = match self.state { - Some(path) => init_at_state(path, provider_factory)?, + Some(path) => init_at_state(path, provider_factory, etl_config)?, None => init_genesis(provider_factory)?, }; @@ -95,6 +100,7 @@ impl InitStateCommand { pub fn init_at_state( state_dump_path: PathBuf, factory: ProviderFactory, + etl_config: EtlConfig, ) -> eyre::Result { info!(target: "reth::cli", path=?state_dump_path, @@ -103,5 +109,5 @@ pub fn init_at_state( let file = File::open(state_dump_path)?; let reader = BufReader::new(file); - init_from_state_dump(reader, factory) + init_from_state_dump(reader, factory, etl_config) } diff --git a/crates/node-core/Cargo.toml b/crates/node-core/Cargo.toml index 157b44970..3caf5d9d1 100644 --- a/crates/node-core/Cargo.toml +++ b/crates/node-core/Cargo.toml @@ -36,6 +36,8 @@ reth-tasks.workspace = true reth-trie.workspace = true reth-consensus-common.workspace = true reth-beacon-consensus.workspace = true +reth-etl.workspace = true +reth-codecs.workspace = true # ethereum discv5.workspace = true diff --git a/crates/node-core/src/init.rs b/crates/node-core/src/init.rs index 92b9f5696..8a7751e4e 100644 --- a/crates/node-core/src/init.rs +++ b/crates/node-core/src/init.rs @@ -1,10 +1,13 @@ //! Reth genesis initialization utility functions. +use reth_codecs::Compact; +use reth_config::config::EtlConfig; use reth_db::{ database::Database, tables, transaction::{DbTx, DbTxMut}, }; +use reth_etl::Collector; use reth_interfaces::{db::DatabaseError, provider::ProviderResult}; use reth_primitives::{ stage::StageId, Account, Address, Bytecode, ChainSpec, GenesisAccount, Receipts, @@ -293,10 +296,16 @@ pub fn insert_genesis_header( Ok(()) } -/// Initialize chain with state at specific block, from reader of state dump. +/// Reads account state from a [`BufRead`] reader and initializes it at the highest block that can +/// be found on database. +/// +/// It's similar to [`init_genesis`] but supports importing state too big to fit in memory, and can +/// be set to the highest block present. One practical usecase is to import OP mainnet state at +/// bedrock transition block. pub fn init_from_state_dump( mut reader: impl BufRead, factory: ProviderFactory, + etl_config: EtlConfig, ) -> eyre::Result { let block = factory.last_block_number()?; let hash = factory.block_hash(block)?.unwrap(); @@ -307,72 +316,15 @@ pub fn init_from_state_dump( "Initializing state at block" ); - let mut total_inserted_accounts = 0; - let mut accounts = Vec::with_capacity(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP); - let mut chunk_total_byte_len = 0; - let mut line = String::new(); - // first line can be state root, then it can be used for verifying against computed state root - reader.read_line(&mut line)?; - let expected_state_root = serde_json::from_str::(&line)?.root; - - trace!(target: "reth::cli", - root=%expected_state_root, - "Read state root from file" - ); - - line.clear(); + let expected_state_root = parse_state_root(&mut reader)?; // remaining lines are accounts + let collector = parse_accounts(&mut reader, etl_config)?; + + // write state to db let mut provider_rw = factory.provider_rw()?; - while let Ok(n) = reader.read_line(&mut line) { - chunk_total_byte_len += n; - if DEFAULT_SOFT_LIMIT_BYTE_LEN_ACCOUNTS_CHUNK <= chunk_total_byte_len || n == 0 { - // acc - total_inserted_accounts += accounts.len(); - - info!(target: "reth::cli", - chunk_total_byte_len, - parsed_new_accounts=accounts.len(), - total_inserted_accounts, - "Writing accounts to db" - ); - - // reset - chunk_total_byte_len = 0; - - // use transaction to insert genesis header - insert_genesis_hashes( - &provider_rw, - accounts.iter().map(|(address, account)| (address, account)), - )?; - insert_history( - &provider_rw, - accounts.iter().map(|(address, account)| (address, account)), - block, - )?; - - // block is already written to static files - let tx = provider_rw.deref_mut().tx_mut(); - insert_state::( - tx, - accounts.len(), - accounts.iter().map(|(address, account)| (address, account)), - block, - )?; - - accounts.clear(); - } - - if n == 0 { - break; - } - - let GenesisAccountWithAddress { genesis_account, address } = serde_json::from_str(&line)?; - accounts.push((address, genesis_account)); - - line.clear(); - } + dump_state(collector, &mut provider_rw, block)?; // compute and compare state root. this advances the stage checkpoints. let computed_state_root = compute_state_root(&provider_rw)?; @@ -396,6 +348,102 @@ pub fn init_from_state_dump( Ok(hash) } +/// Parses and returns expected state root. +fn parse_state_root(reader: &mut impl BufRead) -> eyre::Result { + let mut line = String::new(); + reader.read_line(&mut line)?; + + let expected_state_root = serde_json::from_str::(&line)?.root; + trace!(target: "reth::cli", + root=%expected_state_root, + "Read state root from file" + ); + Ok(expected_state_root) +} + +/// Parses accounts and pushes them to a [`Collector`]. +fn parse_accounts( + mut reader: impl BufRead, + etl_config: EtlConfig, +) -> Result, eyre::Error> { + let mut line = String::new(); + let mut collector = Collector::new(etl_config.file_size, etl_config.dir); + + while let Ok(n) = reader.read_line(&mut line) { + if n == 0 { + break; + } + + let GenesisAccountWithAddress { genesis_account, address } = serde_json::from_str(&line)?; + collector.insert(address, genesis_account)?; + + if !collector.is_empty() && collector.len() % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0 + { + info!(target: "reth::cli", + parsed_new_accounts=collector.len(), + ); + } + + line.clear(); + } + + Ok(collector) +} + +/// Takes a [`Collector`] and processes all accounts. +fn dump_state( + mut collector: Collector, + provider_rw: &mut DatabaseProviderRW, + block: u64, +) -> Result<(), eyre::Error> { + let accounts_len = collector.len(); + let mut accounts = Vec::with_capacity(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP); + let mut total_inserted_accounts = 0; + + for (index, entry) in collector.iter()?.enumerate() { + let (address, account) = entry?; + let (address, _) = Address::from_compact(address.as_slice(), address.len()); + let (account, _) = GenesisAccount::from_compact(account.as_slice(), account.len()); + + accounts.push((address, account)); + + if (index > 0 && index % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0) || + index == accounts_len - 1 + { + total_inserted_accounts += accounts.len(); + + info!(target: "reth::cli", + total_inserted_accounts, + "Writing accounts to db" + ); + + // use transaction to insert genesis header + insert_genesis_hashes( + provider_rw, + accounts.iter().map(|(address, account)| (address, account)), + )?; + + insert_history( + provider_rw, + accounts.iter().map(|(address, account)| (address, account)), + block, + )?; + + // block is already written to static files + let tx = provider_rw.deref_mut().tx_mut(); + insert_state::( + tx, + accounts.len(), + accounts.iter().map(|(address, account)| (address, account)), + block, + )?; + + accounts.clear(); + } + } + Ok(()) +} + /// Computes the state root (from scratch) based on the accounts and storages present in the /// database. fn compute_state_root(provider: &DatabaseProviderRW) -> eyre::Result { diff --git a/crates/storage/codecs/Cargo.toml b/crates/storage/codecs/Cargo.toml index ab8f1a323..958ccf917 100644 --- a/crates/storage/codecs/Cargo.toml +++ b/crates/storage/codecs/Cargo.toml @@ -16,6 +16,7 @@ reth-codecs-derive = { path = "./derive", default-features = false } # eth alloy-eips = { workspace = true, optional = true } +alloy-genesis = { workspace = true, optional = true } alloy-primitives.workspace = true # misc @@ -36,5 +37,5 @@ proptest-derive.workspace = true [features] default = ["std", "alloy"] std = ["alloy-primitives/std", "bytes/std"] -alloy = ["dep:alloy-eips", "dep:modular-bitfield"] +alloy = ["dep:alloy-eips", "dep:alloy-genesis", "dep:modular-bitfield"] optimism = ["reth-codecs-derive/optimism"] diff --git a/crates/storage/codecs/src/alloy/genesis_account.rs b/crates/storage/codecs/src/alloy/genesis_account.rs new file mode 100644 index 000000000..619d9db51 --- /dev/null +++ b/crates/storage/codecs/src/alloy/genesis_account.rs @@ -0,0 +1,67 @@ +use crate::Compact; +use alloy_genesis::GenesisAccount as AlloyGenesisAccount; +use alloy_primitives::{Bytes, B256, U256}; +use reth_codecs_derive::main_codec; + +/// GenesisAccount acts as bridge which simplifies Compact implementation for AlloyGenesisAccount. +/// +/// Notice: Make sure this struct is 1:1 with `alloy_genesis::GenesisAccount` +#[main_codec] +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct GenesisAccount { + /// The nonce of the account at genesis. + nonce: Option, + /// The balance of the account at genesis. + balance: U256, + /// The account's bytecode at genesis. + code: Option, + /// The account's storage at genesis. + storage: Option, + /// The account's private key. Should only be used for testing. + private_key: Option, +} + +#[main_codec] +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct StorageEntries { + entries: Vec, +} + +#[main_codec] +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct StorageEntry { + key: B256, + value: B256, +} + +impl Compact for AlloyGenesisAccount { + fn to_compact(self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + let account = GenesisAccount { + nonce: self.nonce, + balance: self.balance, + code: self.code, + storage: self.storage.map(|s| StorageEntries { + entries: s.into_iter().map(|(key, value)| StorageEntry { key, value }).collect(), + }), + private_key: self.private_key, + }; + account.to_compact(buf) + } + + fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { + let (account, _) = GenesisAccount::from_compact(buf, len); + let alloy_account = AlloyGenesisAccount { + nonce: account.nonce, + balance: account.balance, + code: account.code, + storage: account + .storage + .map(|s| s.entries.into_iter().map(|entry| (entry.key, entry.value)).collect()), + private_key: account.private_key, + }; + (alloy_account, buf) + } +} diff --git a/crates/storage/codecs/src/alloy/mod.rs b/crates/storage/codecs/src/alloy/mod.rs index aff164642..664ab2607 100644 --- a/crates/storage/codecs/src/alloy/mod.rs +++ b/crates/storage/codecs/src/alloy/mod.rs @@ -1,4 +1,5 @@ mod access_list; +mod genesis_account; mod log; mod txkind; mod withdrawal; diff --git a/crates/storage/db/src/tables/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs index 452f5c632..aed8d97ef 100644 --- a/crates/storage/db/src/tables/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -50,7 +50,9 @@ impl_compression_for_compact!( CompactU256, StageCheckpoint, PruneCheckpoint, - ClientVersion + ClientVersion, + // Non-DB + GenesisAccount ); macro_rules! impl_compression_fixed_compact { diff --git a/crates/storage/provider/src/bundle_state/state_reverts.rs b/crates/storage/provider/src/bundle_state/state_reverts.rs index 006f87b40..cc16a50cc 100644 --- a/crates/storage/provider/src/bundle_state/state_reverts.rs +++ b/crates/storage/provider/src/bundle_state/state_reverts.rs @@ -1,6 +1,6 @@ use rayon::slice::ParallelSliceMut; use reth_db::{ - cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + cursor::{DbCursorRO, DbDupCursorRO, DbDupCursorRW}, models::{AccountBeforeTx, BlockNumberAddress}, tables, transaction::{DbTx, DbTxMut}, @@ -75,30 +75,16 @@ impl StateReverts { tracing::trace!(target: "provider::reverts", "Writing account changes"); let mut account_changeset_cursor = tx.cursor_dup_write::()?; - // append entries if key is new - let should_append_accounts = - account_changeset_cursor.last()?.map_or(true, |(block_number, _)| { - block_number < first_block || block_number == first_block && block_number == 0 - }); for (block_index, mut account_block_reverts) in self.0.accounts.into_iter().enumerate() { let block_number = first_block + block_index as BlockNumber; // Sort accounts by address. account_block_reverts.par_sort_by_key(|a| a.0); for (address, info) in account_block_reverts { - if should_append_accounts { - account_changeset_cursor.append_dup( - block_number, - AccountBeforeTx { address, info: info.map(into_reth_acc) }, - )?; - } else { - // upsert on dupsort tables will append to subkey. see implementation of - // DbCursorRW::upsert for reth_db::implementation::mdbx::cursor::Cursor - account_changeset_cursor.upsert( - block_number, - AccountBeforeTx { address, info: info.map(into_reth_acc) }, - )?; - } + account_changeset_cursor.append_dup( + block_number, + AccountBeforeTx { address, info: info.map(into_reth_acc) }, + )?; } }