jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 02:49:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Breaking changes (#5191)

Browse Source 
Co-authored-by: Bjerg <onbjerg@users.noreply.github.com>
Co-authored-by: Roman Krasiuk <rokrassyuk@gmail.com>
Co-authored-by: joshieDo <ranriver@protonmail.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Oliver Nordbjerg <hi@notbjerg.me>
Co-authored-by: Thomas Coratger <thomas.coratger@gmail.com>
This commit is contained in:
Alexey Shekhirin
2024-02-29 12:37:28 +00:00
committed by GitHub
parent 025fa5f038
commit 6b5b6f7a40
252 changed files with 10154 additions and 6327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.config/nextest.toml
View File

@ -4,4 +4,4 @@ slow-timeout = { period = "30s", terminate-after = 4 }
[[profile.default.overrides]]
filter = "test(general_state_tests)"
slow-timeout = { period = "1m", terminate-after = 4 }
slow-timeout = { period = "1m", terminate-after = 10 }

2
CODEOWNERS
View File

@ -19,5 +19,5 @@ crates/metrics @onbjerg
crates/tracing @onbjerg
crates/tasks @mattsse
crates/prune @shekhirin @joshieDo
crates/snapshot @joshieDo
crates/static-file @joshieDo @shekhirin
.github/ @onbjerg @gakonst @DaniPopes

100
Cargo.lock generated
View File

@ -2301,6 +2301,7 @@ name = "ef-tests"
version = "0.1.0-alpha.21"
dependencies = [
"alloy-rlp",
"rayon",
"reth-db",
"reth-interfaces",
"reth-node-ethereum",
@ -2727,6 +2728,7 @@ dependencies = [
"async-trait",
"eyre",
"futures",
"jemallocator",
"reth-beacon-consensus",
"reth-blockchain-tree",
"reth-db",
@ -5690,6 +5692,12 @@ dependencies = [
"quick-error",
]
[[package]]
name = "retain_mut"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086"
[[package]]
name = "reth"
version = "0.1.0-alpha.21"
@ -5755,8 +5763,8 @@ dependencies = [
"reth-rpc-engine-api",
"reth-rpc-types",
"reth-rpc-types-compat",
"reth-snapshot",
"reth-stages",
"reth-static-file",
"reth-tasks",
"reth-tracing",
"reth-transaction-pool",
@ -5844,12 +5852,13 @@ dependencies = [
"reth-revm",
"reth-rpc-types",
"reth-rpc-types-compat",
"reth-snapshot",
"reth-stages",
"reth-static-file",
"reth-tasks",
"reth-tokio-util",
"reth-tracing",
"schnellru",
"tempfile",
"thiserror",
"tokio",
"tokio-stream",
@ -6143,6 +6152,16 @@ dependencies = [
"tracing",
]
[[package]]
name = "reth-etl"
version = "0.1.0-alpha.21"
dependencies = [
"rayon",
"reth-db",
"reth-primitives",
"tempfile",
]
[[package]]
name = "reth-interfaces"
version = "0.1.0-alpha.21"
@ -6344,8 +6363,9 @@ dependencies = [
"memmap2 0.7.1",
"ph",
"rand 0.8.5",
"reth-primitives",
"serde",
"sucds 0.8.1",
"sucds",
"tempfile",
"thiserror",
"tracing",
@ -6388,8 +6408,8 @@ dependencies = [
"reth-revm",
"reth-rpc",
"reth-rpc-engine-api",
"reth-snapshot",
"reth-stages",
"reth-static-file",
"reth-tasks",
"reth-tracing",
"reth-transaction-pool",
@ -6405,6 +6425,7 @@ dependencies = [
"assert_matches",
"clap",
"const-str",
"derive_more",
"dirs-next",
"eyre",
"futures",
@ -6449,8 +6470,8 @@ dependencies = [
"reth-rpc-engine-api",
"reth-rpc-types",
"reth-rpc-types-compat",
"reth-snapshot",
"reth-stages",
"reth-static-file",
"reth-tasks",
"reth-tracing",
"reth-transaction-pool",
@ -6571,6 +6592,7 @@ dependencies = [
"alloy-primitives",
"alloy-rlp",
"alloy-trie",
"anyhow",
"arbitrary",
"assert_matches",
"byteorder",
@ -6598,12 +6620,13 @@ dependencies = [
"reth-rpc-types",
"revm",
"revm-primitives",
"roaring",
"secp256k1 0.27.0",
"serde",
"serde_json",
"sha2",
"strum 0.26.1",
"sucds 0.6.0",
"sucds",
"tempfile",
"test-fuzz",
"thiserror",
@ -6627,6 +6650,7 @@ dependencies = [
"pin-project",
"rand 0.8.5",
"rayon",
"reth-codecs",
"reth-db",
"reth-interfaces",
"reth-metrics",
@ -6656,8 +6680,8 @@ dependencies = [
"reth-metrics",
"reth-primitives",
"reth-provider",
"reth-snapshot",
"reth-stages",
"reth-static-file",
"reth-tokio-util",
"thiserror",
"tokio",
@ -6867,24 +6891,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "reth-snapshot"
version = "0.1.0-alpha.21"
dependencies = [
"assert_matches",
"clap",
"reth-db",
"reth-interfaces",
"reth-nippy-jar",
"reth-primitives",
"reth-provider",
"reth-stages",
"tempfile",
"thiserror",
"tokio",
"tracing",
]
[[package]]
name = "reth-stages"
version = "0.1.0-alpha.21"
@ -6908,6 +6914,7 @@ dependencies = [
"reth-db",
"reth-downloaders",
"reth-eth-wire",
"reth-etl",
"reth-interfaces",
"reth-metrics",
"reth-node-ethereum",
@ -6915,11 +6922,34 @@ dependencies = [
"reth-primitives",
"reth-provider",
"reth-revm",
"reth-static-file",
"reth-tokio-util",
"reth-trie",
"revm",
"serde",
"serde_json",
"tempfile",
"thiserror",
"tokio",
"tokio-stream",
"tracing",
]
[[package]]
name = "reth-static-file"
version = "0.1.0-alpha.21"
dependencies = [
"assert_matches",
"clap",
"rayon",
"reth-db",
"reth-interfaces",
"reth-nippy-jar",
"reth-primitives",
"reth-provider",
"reth-stages",
"reth-tokio-util",
"tempfile",
"thiserror",
"tokio",
"tokio-stream",
@ -7194,6 +7224,17 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "roaring"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6106b5cf8587f5834158895e9715a3c6c9716c8aefab57f1f7680917191c7873"
dependencies = [
"bytemuck",
"byteorder",
"retain_mut",
]
[[package]]
name = "rolling-file"
version = "0.2.0"
@ -8068,15 +8109,6 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
[[package]]
name = "sucds"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64accd20141dfbef67ad83c51d588146cff7810616e1bda35a975be369059533"
dependencies = [
"anyhow",
]
[[package]]
name = "sucds"
version = "0.8.1"

6
Cargo.toml
View File

@ -8,6 +8,7 @@ members = [
"crates/consensus/beacon-core/",
"crates/consensus/common/",
"crates/ethereum-forks/",
"crates/etl",
"crates/interfaces/",
"crates/metrics/",
"crates/metrics/metrics-derive/",
@ -41,8 +42,8 @@ members = [
"crates/node-optimism/",
"crates/node-core/",
"crates/node-api/",
"crates/snapshot/",
"crates/stages/",
"crates/static-file/",
"crates/storage/codecs/",
"crates/storage/codecs/derive/",
"crates/storage/db/",
@ -137,6 +138,7 @@ reth-ecies = { path = "crates/net/ecies" }
reth-eth-wire = { path = "crates/net/eth-wire" }
reth-ethereum-forks = { path = "crates/ethereum-forks" }
reth-ethereum-payload-builder = { path = "crates/payload/ethereum" }
reth-etl = { path = "crates/etl" }
reth-optimism-payload-builder = { path = "crates/payload/optimism" }
reth-interfaces = { path = "crates/interfaces" }
reth-ipc = { path = "crates/rpc/ipc" }
@ -162,8 +164,8 @@ reth-rpc-builder = { path = "crates/rpc/rpc-builder" }
reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" }
reth-rpc-types = { path = "crates/rpc/rpc-types" }
reth-rpc-types-compat = { path = "crates/rpc/rpc-types-compat" }
reth-snapshot = { path = "crates/snapshot" }
reth-stages = { path = "crates/stages" }
reth-static-file = { path = "crates/static-file" }
reth-tasks = { path = "crates/tasks" }
reth-tokio-util = { path = "crates/tokio-util" }
reth-tracing = { path = "crates/tracing" }

23
README.md
View File

@ -1,4 +1,4 @@
# reth
# reth
[![CI status](https://github.com/paradigmxyz/reth/workflows/ci/badge.svg)][gh-ci]
[![cargo-deny status](https://github.com/paradigmxyz/reth/workflows/deny/badge.svg)][gh-deny]
@ -14,7 +14,7 @@
| [Developer Docs](./docs)
| [Crate Docs](https://paradigmxyz.github.io/reth/docs)
*The project is still work in progress, see the [disclaimer below](#status).*
_The project is still work in progress, see the [disclaimer below](#status)._
[codecov]: https://app.codecov.io/gh/paradigmxyz/reth
[gh-ci]: https://github.com/paradigmxyz/reth/actions/workflows/ci.yml
@ -50,6 +50,14 @@ We will be updating the documentation with the completion status of each compone
We appreciate your patience until we get there. Until then, we are happy to answer all questions in the Telegram link above.
### Database compatibility
Reth [v0.2.0-beta.1](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) includes
a [set of breaking database changes](https://github.com/paradigmxyz/reth/pull/5191) that makes it impossible to use database files produced by earlier versions.
If you had a database produced by alpha versions of Reth, you need to drop it with `reth db drop`
(using the same arguments such as `--config` or `--datadir` that you passed to `reth node`), and resync using the same `reth node` command you've used before.
## For Users
See the [Reth Book](https://paradigmxyz.github.io/reth) for instructions on how to install and run Reth.
@ -105,7 +113,7 @@ cargo test --workspace --features geth-tests
# With Ethereum Foundation tests
#
# Note: Requires cloning https://github.com/ethereum/tests
#
#
# cd testing/ef-tests && git clone https://github.com/ethereum/tests ethereum-tests
cargo test -p ef-tests --features ef-tests
```
@ -113,7 +121,7 @@ cargo test -p ef-tests --features ef-tests
We recommend using [`cargo nextest`](https://nexte.st/) to speed up testing. With nextest installed, simply substitute `cargo test` with `cargo nextest run`.
> **Note**
>
>
> Some tests use random number generators to generate test data. If you want to use a deterministic seed, you can set the `SEED` environment variable.
## Getting Help
@ -135,9 +143,10 @@ See [`SECURITY.md`](./SECURITY.md).
Reth is a new implementation of the Ethereum protocol. In the process of developing the node we investigated the design decisions other nodes have made to understand what is done well, what is not, and where we can improve the status quo.
None of this would have been possible without them, so big shoutout to the teams below:
* [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project.
* [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes.
* [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80) . Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages.
- [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project.
- [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes.
- [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80) . Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages.
[book]: https://paradigmxyz.github.io/reth/
[tg-url]: https://t.me/paradigm_reth

2
bin/reth/Cargo.toml
View File

@ -50,7 +50,7 @@ reth-payload-validator.workspace = true
reth-basic-payload-builder.workspace = true
reth-discv4.workspace = true
reth-prune.workspace = true
reth-snapshot = { workspace = true, features = ["clap"] }
reth-static-file = { workspace = true, features = ["clap"] }
reth-trie.workspace = true
reth-nippy-jar.workspace = true
reth-node-api.workspace = true

63
bin/reth/src/builder.rs
View File

@ -6,7 +6,7 @@ use fdlimit::raise_fd_limit;
use futures::{future::Either, stream, stream_select, StreamExt};
use reth_auto_seal_consensus::AutoSealBuilder;
use reth_beacon_consensus::{
hooks::{EngineHooks, PruneHook},
hooks::{EngineHooks, PruneHook, StaticFileHook},
BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN,
};
use reth_blockchain_tree::{config::BlockchainTreeConfig, ShareableBlockchainTree};
@ -40,6 +40,7 @@ use reth_primitives::format_ether;
use reth_provider::{providers::BlockchainProvider, ProviderFactory};
use reth_prune::PrunerBuilder;
use reth_rpc_engine_api::EngineApi;
use reth_static_file::StaticFileProducer;
use reth_tasks::{TaskExecutor, TaskManager};
use reth_transaction_pool::TransactionPool;
use std::{path::PathBuf, sync::Arc};
@ -127,26 +128,18 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
let prometheus_handle = self.config.install_prometheus_recorder()?;
let mut provider_factory =
ProviderFactory::new(Arc::clone(&self.db), Arc::clone(&self.config.chain));
// configure snapshotter
let snapshotter = reth_snapshot::Snapshotter::new(
provider_factory.clone(),
self.data_dir.snapshots_path(),
self.config.chain.snapshot_block_interval,
)?;
provider_factory = provider_factory.with_snapshots(
self.data_dir.snapshots_path(),
snapshotter.highest_snapshot_receiver(),
)?;
let provider_factory = ProviderFactory::new(
Arc::clone(&self.db),
Arc::clone(&self.config.chain),
self.data_dir.static_files_path(),
)?
.with_static_files_metrics();
self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.db)).await?;
debug!(target: "reth::cli", chain=%self.config.chain.chain, genesis=?self.config.chain.genesis_hash(), "Initializing genesis");
let genesis_hash = init_genesis(Arc::clone(&self.db), self.config.chain.clone())?;
let genesis_hash = init_genesis(provider_factory.clone())?;
info!(target: "reth::cli", "{}", self.config.chain.display_hardforks());
@ -270,6 +263,17 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
};
let max_block = self.config.max_block(&network_client, provider_factory.clone()).await?;
let mut hooks = EngineHooks::new();
let mut static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
prune_config.clone().unwrap_or_default().segments,
);
let static_file_producer_events = static_file_producer.events();
hooks.add(StaticFileHook::new(static_file_producer.clone(), Box::new(executor.clone())));
info!(target: "reth::cli", "StaticFileProducer initialized");
// Configure the pipeline
let (mut pipeline, client) = if self.config.dev.dev {
info!(target: "reth::cli", "Starting Reth in dev mode");
@ -301,6 +305,7 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
sync_metrics_tx,
prune_config.clone(),
max_block,
static_file_producer,
evm_config,
)
.await?;
@ -323,6 +328,7 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
sync_metrics_tx,
prune_config.clone(),
max_block,
static_file_producer,
evm_config,
)
.await?;
@ -333,22 +339,16 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
let pipeline_events = pipeline.events();
let initial_target = self.config.initial_pipeline_target(genesis_hash);
let mut hooks = EngineHooks::new();
let pruner_events = if let Some(prune_config) = prune_config {
let mut pruner = PrunerBuilder::new(prune_config.clone())
.max_reorg_depth(tree_config.max_reorg_depth() as usize)
.prune_delete_limit(self.config.chain.prune_delete_limit)
.build(provider_factory, snapshotter.highest_snapshot_receiver());
let prune_config = prune_config.unwrap_or_default();
let mut pruner = PrunerBuilder::new(prune_config.clone())
.max_reorg_depth(tree_config.max_reorg_depth() as usize)
.prune_delete_limit(self.config.chain.prune_delete_limit)
.build(provider_factory.clone());
let events = pruner.events();
hooks.add(PruneHook::new(pruner, Box::new(executor.clone())));
info!(target: "reth::cli", ?prune_config, "Pruner initialized");
Either::Left(events)
} else {
Either::Right(stream::empty())
};
let pruner_events = pruner.events();
hooks.add(PruneHook::new(pruner, Box::new(executor.clone())));
info!(target: "reth::cli", ?prune_config, "Pruner initialized");
// Configure the consensus engine
let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel(
@ -380,7 +380,8 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
} else {
Either::Right(stream::empty())
},
pruner_events.map(Into::into)
pruner_events.map(Into::into),
static_file_producer_events.map(Into::into),
);
executor.spawn_critical(
"events task",

36
bin/reth/src/commands/db/clear.rs
View File

@ -1,27 +1,53 @@
use clap::Parser;
use clap::{Parser, Subcommand};
use reth_db::{
database::Database,
static_file::iter_static_files,
table::Table,
transaction::{DbTx, DbTxMut},
TableViewer, Tables,
};
use reth_primitives::{static_file::find_fixed_range, StaticFileSegment};
use reth_provider::ProviderFactory;
/// The arguments for the `reth db clear` command
#[derive(Parser, Debug)]
pub struct Command {
/// Table name
pub table: Tables,
#[clap(subcommand)]
subcommand: Subcommands,
}
impl Command {
/// Execute `db clear` command
pub fn execute<DB: Database>(self, db: &DB) -> eyre::Result<()> {
self.table.view(&ClearViewer { db })?;
pub fn execute<DB: Database>(self, provider_factory: ProviderFactory<DB>) -> eyre::Result<()> {
match self.subcommand {
Subcommands::Mdbx { table } => {
table.view(&ClearViewer { db: provider_factory.db_ref() })?
}
Subcommands::StaticFile { segment } => {
let static_file_provider = provider_factory.static_file_provider();
let static_files = iter_static_files(static_file_provider.directory())?;
if let Some(segment_static_files) = static_files.get(&segment) {
for (block_range, _) in segment_static_files {
static_file_provider
.delete_jar(segment, find_fixed_range(block_range.start()))?;
}
}
}
}
Ok(())
}
}
#[derive(Subcommand, Debug)]
enum Subcommands {
/// Deletes all database table entries
Mdbx { table: Tables },
/// Deletes all static file segment entries
StaticFile { segment: StaticFileSegment },
}
struct ClearViewer<'a, DB: Database> {
db: &'a DB,
}

61
bin/reth/src/commands/db/diff.rs
View File

@ -6,11 +6,12 @@ use crate::{
use clap::Parser;
use reth_db::{
cursor::DbCursorRO, database::Database, mdbx::DatabaseArguments, open_db_read_only,
table::Table, transaction::DbTx, AccountChangeSet, AccountHistory, AccountsTrie,
table::Table, transaction::DbTx, AccountChangeSets, AccountsHistory, AccountsTrie,
BlockBodyIndices, BlockOmmers, BlockWithdrawals, Bytecodes, CanonicalHeaders, DatabaseEnv,
HashedAccount, HashedStorage, HeaderNumbers, HeaderTD, Headers, PlainAccountState,
PlainStorageState, PruneCheckpoints, Receipts, StorageChangeSet, StorageHistory, StoragesTrie,
SyncStage, SyncStageProgress, Tables, TransactionBlock, Transactions, TxHashNumber, TxSenders,
HashedAccounts, HashedStorages, HeaderNumbers, HeaderTerminalDifficulties, Headers,
PlainAccountState, PlainStorageState, PruneCheckpoints, Receipts, StageCheckpointProgresses,
StageCheckpoints, StorageChangeSets, StoragesHistory, StoragesTrie, Tables, TransactionBlocks,
TransactionHashNumbers, TransactionSenders, Transactions,
};
use std::{
collections::HashMap,
@ -56,7 +57,7 @@ impl Command {
///
/// The discrepancies and extra elements, along with a brief summary of the diff results are
/// then written to a file in the output directory.
pub fn execute(self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result<()> {
pub fn execute(self, tool: &DbTool<DatabaseEnv>) -> eyre::Result<()> {
// open second db
let second_db_path: PathBuf = self.secondary_datadir.join("db").into();
let second_db = open_db_read_only(
@ -70,7 +71,7 @@ impl Command {
};
for table in tables {
let primary_tx = tool.db.tx()?;
let primary_tx = tool.provider_factory.db_ref().tx()?;
let secondary_tx = second_db.tx()?;
let output_dir = self.output.clone();
@ -78,7 +79,9 @@ impl Command {
Tables::CanonicalHeaders => {
find_diffs::<CanonicalHeaders>(primary_tx, secondary_tx, output_dir)?
}
Tables::HeaderTD => find_diffs::<HeaderTD>(primary_tx, secondary_tx, output_dir)?,
Tables::HeaderTerminalDifficulties => {
find_diffs::<HeaderTerminalDifficulties>(primary_tx, secondary_tx, output_dir)?
}
Tables::HeaderNumbers => {
find_diffs::<HeaderNumbers>(primary_tx, secondary_tx, output_dir)?
}
@ -92,14 +95,14 @@ impl Command {
Tables::BlockWithdrawals => {
find_diffs::<BlockWithdrawals>(primary_tx, secondary_tx, output_dir)?
}
Tables::TransactionBlock => {
find_diffs::<TransactionBlock>(primary_tx, secondary_tx, output_dir)?
Tables::TransactionBlocks => {
find_diffs::<TransactionBlocks>(primary_tx, secondary_tx, output_dir)?
}
Tables::Transactions => {
find_diffs::<Transactions>(primary_tx, secondary_tx, output_dir)?
}
Tables::TxHashNumber => {
find_diffs::<TxHashNumber>(primary_tx, secondary_tx, output_dir)?
Tables::TransactionHashNumbers => {
find_diffs::<TransactionHashNumbers>(primary_tx, secondary_tx, output_dir)?
}
Tables::Receipts => find_diffs::<Receipts>(primary_tx, secondary_tx, output_dir)?,
Tables::PlainAccountState => {
@ -109,23 +112,23 @@ impl Command {
find_diffs::<PlainStorageState>(primary_tx, secondary_tx, output_dir)?
}
Tables::Bytecodes => find_diffs::<Bytecodes>(primary_tx, secondary_tx, output_dir)?,
Tables::AccountHistory => {
find_diffs::<AccountHistory>(primary_tx, secondary_tx, output_dir)?
Tables::AccountsHistory => {
find_diffs::<AccountsHistory>(primary_tx, secondary_tx, output_dir)?
}
Tables::StorageHistory => {
find_diffs::<StorageHistory>(primary_tx, secondary_tx, output_dir)?
Tables::StoragesHistory => {
find_diffs::<StoragesHistory>(primary_tx, secondary_tx, output_dir)?
}
Tables::AccountChangeSet => {
find_diffs::<AccountChangeSet>(primary_tx, secondary_tx, output_dir)?
Tables::AccountChangeSets => {
find_diffs::<AccountChangeSets>(primary_tx, secondary_tx, output_dir)?
}
Tables::StorageChangeSet => {
find_diffs::<StorageChangeSet>(primary_tx, secondary_tx, output_dir)?
Tables::StorageChangeSets => {
find_diffs::<StorageChangeSets>(primary_tx, secondary_tx, output_dir)?
}
Tables::HashedAccount => {
find_diffs::<HashedAccount>(primary_tx, secondary_tx, output_dir)?
Tables::HashedAccounts => {
find_diffs::<HashedAccounts>(primary_tx, secondary_tx, output_dir)?
}
Tables::HashedStorage => {
find_diffs::<HashedStorage>(primary_tx, secondary_tx, output_dir)?
Tables::HashedStorages => {
find_diffs::<HashedStorages>(primary_tx, secondary_tx, output_dir)?
}
Tables::AccountsTrie => {
find_diffs::<AccountsTrie>(primary_tx, secondary_tx, output_dir)?
@ -133,10 +136,14 @@ impl Command {
Tables::StoragesTrie => {
find_diffs::<StoragesTrie>(primary_tx, secondary_tx, output_dir)?
}
Tables::TxSenders => find_diffs::<TxSenders>(primary_tx, secondary_tx, output_dir)?,
Tables::SyncStage => find_diffs::<SyncStage>(primary_tx, secondary_tx, output_dir)?,
Tables::SyncStageProgress => {
find_diffs::<SyncStageProgress>(primary_tx, secondary_tx, output_dir)?
Tables::TransactionSenders => {
find_diffs::<TransactionSenders>(primary_tx, secondary_tx, output_dir)?
}
Tables::StageCheckpoints => {
find_diffs::<StageCheckpoints>(primary_tx, secondary_tx, output_dir)?
}
Tables::StageCheckpointProgresses => {
find_diffs::<StageCheckpointProgresses>(primary_tx, secondary_tx, output_dir)?
}
Tables::PruneCheckpoints => {
find_diffs::<PruneCheckpoints>(primary_tx, secondary_tx, output_dir)?

186
bin/reth/src/commands/db/get.rs
View File

@ -2,64 +2,152 @@ use crate::utils::DbTool;
use clap::Parser;
use reth_db::{
database::Database,
table::{DupSort, Table},
RawKey, RawTable, TableViewer, Tables,
static_file::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask},
table::{Decompress, DupSort, Table},
tables, RawKey, RawTable, Receipts, TableViewer, Transactions,
};
use reth_primitives::{BlockHash, Header, StaticFileSegment};
use tracing::error;
/// The arguments for the `reth db get` command
#[derive(Parser, Debug)]
pub struct Command {
/// The table name
///
/// NOTE: The dupsort tables are not supported now.
pub table: Tables,
#[clap(subcommand)]
subcommand: Subcommand,
}
/// The key to get content for
#[arg(value_parser = maybe_json_value_parser)]
pub key: String,
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Gets the content of a database table for the given key
Mdbx {
table: tables::Tables,
/// The subkey to get content for
#[arg(value_parser = maybe_json_value_parser)]
pub subkey: Option<String>,
/// The key to get content for
#[arg(value_parser = maybe_json_value_parser)]
key: String,
/// Output bytes instead of human-readable decoded value
#[clap(long)]
pub raw: bool,
/// The subkey to get content for
#[arg(value_parser = maybe_json_value_parser)]
subkey: Option<String>,
/// Output bytes instead of human-readable decoded value
#[clap(long)]
raw: bool,
},
/// Gets the content of a static file segment for the given key
StaticFile {
segment: StaticFileSegment,
/// The key to get content for
#[arg(value_parser = maybe_json_value_parser)]
key: String,
/// Output bytes instead of human-readable decoded value
#[clap(long)]
raw: bool,
},
}
impl Command {
/// Execute `db get` command
pub fn execute<DB: Database>(self, tool: &DbTool<'_, DB>) -> eyre::Result<()> {
self.table.view(&GetValueViewer { tool, args: &self })
}
pub fn execute<DB: Database>(self, tool: &DbTool<DB>) -> eyre::Result<()> {
match self.subcommand {
Subcommand::Mdbx { table, key, subkey, raw } => {
table.view(&GetValueViewer { tool, key, subkey, raw })?
}
Subcommand::StaticFile { segment, key, raw } => {
let (key, mask): (u64, _) = match segment {
StaticFileSegment::Headers => {
(table_key::<tables::Headers>(&key)?, <HeaderMask<Header, BlockHash>>::MASK)
}
StaticFileSegment::Transactions => (
table_key::<tables::Transactions>(&key)?,
<TransactionMask<<Transactions as Table>::Value>>::MASK,
),
StaticFileSegment::Receipts => (
table_key::<tables::Receipts>(&key)?,
<ReceiptMask<<Receipts as Table>::Value>>::MASK,
),
};
/// Get an instance of key for given table
pub fn table_key<T: Table>(&self) -> Result<T::Key, eyre::Error> {
assert_eq!(T::TABLE, self.table);
serde_json::from_str::<T::Key>(&self.key).map_err(Into::into)
}
let content = tool.provider_factory.static_file_provider().find_static_file(
segment,
|provider| {
let mut cursor = provider.cursor()?;
cursor.get(key.into(), mask).map(|result| {
result.map(|vec| {
vec.iter().map(|slice| slice.to_vec()).collect::<Vec<_>>()
})
})
},
)?;
/// Get an instance of subkey for given dupsort table
fn table_subkey<T: DupSort>(&self) -> Result<T::SubKey, eyre::Error> {
assert_eq!(T::TABLE, self.table);
serde_json::from_str::<T::SubKey>(&self.subkey.clone().unwrap_or_default())
.map_err(Into::into)
match content {
Some(content) => {
if raw {
println!("{:?}", content);
} else {
match segment {
StaticFileSegment::Headers => {
let header = Header::decompress(content[0].as_slice())?;
let block_hash = BlockHash::decompress(content[1].as_slice())?;
println!(
"{}\n{}",
serde_json::to_string_pretty(&header)?,
serde_json::to_string_pretty(&block_hash)?
);
}
StaticFileSegment::Transactions => {
let transaction = <<Transactions as Table>::Value>::decompress(
content[0].as_slice(),
)?;
println!("{}", serde_json::to_string_pretty(&transaction)?);
}
StaticFileSegment::Receipts => {
let receipt = <<Receipts as Table>::Value>::decompress(
content[0].as_slice(),
)?;
println!("{}", serde_json::to_string_pretty(&receipt)?);
}
}
}
}
None => {
error!(target: "reth::cli", "No content for the given table key.");
}
};
}
}
Ok(())
}
}
/// Get an instance of key for given table
fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
serde_json::from_str::<T::Key>(key).map_err(|e| eyre::eyre!(e))
}
/// Get an instance of subkey for given dupsort table
fn table_subkey<T: DupSort>(subkey: &Option<String>) -> Result<T::SubKey, eyre::Error> {
serde_json::from_str::<T::SubKey>(&subkey.clone().unwrap_or_default())
.map_err(|e| eyre::eyre!(e))
}
struct GetValueViewer<'a, DB: Database> {
tool: &'a DbTool<'a, DB>,
args: &'a Command,
tool: &'a DbTool<DB>,
key: String,
subkey: Option<String>,
raw: bool,
}
impl<DB: Database> TableViewer<()> for GetValueViewer<'_, DB> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(), Self::Error> {
let key = self.args.table_key::<T>()?;
let key = table_key::<T>(&self.key)?;
let content = if self.args.raw {
let content = if self.raw {
self.tool
.get::<RawTable<T>>(RawKey::from(key))?
.map(|content| format!("{:?}", content.raw_value()))
@ -81,10 +169,10 @@ impl<DB: Database> TableViewer<()> for GetValueViewer<'_, DB> {
fn view_dupsort<T: DupSort>(&self) -> Result<(), Self::Error> {
// get a key for given table
let key = self.args.table_key::<T>()?;
let key = table_key::<T>(&self.key)?;
// process dupsort table
let subkey = self.args.table_subkey::<T>()?;
let subkey = table_subkey::<T>(&self.subkey)?;
match self.tool.get_dup::<T>(key, subkey)? {
Some(content) => {
@ -113,7 +201,7 @@ mod tests {
use clap::{Args, Parser};
use reth_db::{
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
AccountHistory, HashedAccount, Headers, StorageHistory, SyncStage,
AccountsHistory, HashedAccounts, Headers, StageCheckpoints, StoragesHistory,
};
use reth_primitives::{Address, B256};
use std::str::FromStr;
@ -127,17 +215,12 @@ mod tests {
#[test]
fn parse_numeric_key_args() {
let args = CommandParser::<Command>::parse_from(["reth", "Headers", "123"]).args;
assert_eq!(args.table_key::<Headers>().unwrap(), 123);
let args = CommandParser::<Command>::parse_from([
"reth",
"HashedAccount",
"0x0ac361fe774b78f8fc4e86c1916930d150865c3fc2e21dca2e58833557608bac",
])
.args;
assert_eq!(table_key::<Headers>("123").unwrap(), 123);
assert_eq!(
args.table_key::<HashedAccount>().unwrap(),
table_key::<HashedAccounts>(
"\"0x0ac361fe774b78f8fc4e86c1916930d150865c3fc2e21dca2e58833557608bac\""
)
.unwrap(),
B256::from_str("0x0ac361fe774b78f8fc4e86c1916930d150865c3fc2e21dca2e58833557608bac")
.unwrap()
);
@ -145,16 +228,16 @@ mod tests {
#[test]
fn parse_string_key_args() {
let args =
CommandParser::<Command>::parse_from(["reth", "SyncStage", "MerkleExecution"]).args;
assert_eq!(args.table_key::<SyncStage>().unwrap(), "MerkleExecution");
assert_eq!(
table_key::<StageCheckpoints>("\"MerkleExecution\"").unwrap(),
"MerkleExecution"
);
}
#[test]
fn parse_json_key_args() {
let args = CommandParser::<Command>::parse_from(["reth", "StorageHistory", r#"{ "address": "0x01957911244e546ce519fbac6f798958fafadb41", "sharded_key": { "key": "0x0000000000000000000000000000000000000000000000000000000000000003", "highest_block_number": 18446744073709551615 } }"#]).args;
assert_eq!(
args.table_key::<StorageHistory>().unwrap(),
table_key::<StoragesHistory>(r#"{ "address": "0x01957911244e546ce519fbac6f798958fafadb41", "sharded_key": { "key": "0x0000000000000000000000000000000000000000000000000000000000000003", "highest_block_number": 18446744073709551615 } }"#).unwrap(),
StorageShardedKey::new(
Address::from_str("0x01957911244e546ce519fbac6f798958fafadb41").unwrap(),
B256::from_str(
@ -168,9 +251,8 @@ mod tests {
#[test]
fn parse_json_key_for_account_history() {
let args = CommandParser::<Command>::parse_from(["reth", "AccountHistory", r#"{ "key": "0x4448e1273fd5a8bfdb9ed111e96889c960eee145", "highest_block_number": 18446744073709551615 }"#]).args;
assert_eq!(
args.table_key::<AccountHistory>().unwrap(),
table_key::<AccountsHistory>(r#"{ "key": "0x4448e1273fd5a8bfdb9ed111e96889c960eee145", "highest_block_number": 18446744073709551615 }"#).unwrap(),
ShardedKey::new(
Address::from_str("0x4448e1273fd5a8bfdb9ed111e96889c960eee145").unwrap(),
18446744073709551615

6
bin/reth/src/commands/db/list.rs
View File

@ -50,7 +50,7 @@ pub struct Command {
impl Command {
/// Execute `db list` command
pub fn execute(self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result<()> {
pub fn execute(self, tool: &DbTool<DatabaseEnv>) -> eyre::Result<()> {
self.table.view(&ListTableViewer { tool, args: &self })
}
@ -81,7 +81,7 @@ impl Command {
}
struct ListTableViewer<'a> {
tool: &'a DbTool<'a, DatabaseEnv>,
tool: &'a DbTool<DatabaseEnv>,
args: &'a Command,
}
@ -89,7 +89,7 @@ impl TableViewer<()> for ListTableViewer<'_> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(), Self::Error> {
self.tool.db.view(|tx| {
self.tool.provider_factory.db_ref().view(|tx| {
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?;
let total_entries = stats.entries();

136
bin/reth/src/commands/db/mod.rs
View File

@ -9,18 +9,13 @@ use crate::{
utils::DbTool,
};
use clap::{Parser, Subcommand};
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::WrapErr;
use human_bytes::human_bytes;
use reth_db::{
database::Database,
mdbx,
mdbx::DatabaseArguments,
open_db, open_db_read_only,
version::{get_db_version, DatabaseVersionError, DB_VERSION},
Tables,
};
use reth_primitives::ChainSpec;
use reth_provider::ProviderFactory;
use std::{
io::{self, Write},
sync::Arc,
@ -30,7 +25,8 @@ mod clear;
mod diff;
mod get;
mod list;
mod snapshots;
mod static_files;
mod stats;
/// DB List TUI
mod tui;
@ -71,7 +67,7 @@ pub struct Command {
/// `reth db` subcommands
pub enum Subcommands {
/// Lists all the tables, their entry count and their size
Stats,
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Create a diff between two database tables or two entire databases.
@ -86,8 +82,8 @@ pub enum Subcommands {
},
/// Deletes all table entries
Clear(clear::Command),
/// Snapshots tables from database
Snapshot(snapshots::Command),
/// Creates static files from database tables
CreateStaticFiles(static_files::Command),
/// Lists current and local database versions
Version,
/// Returns the full database path
@ -100,102 +96,30 @@ impl Command {
// add network name to data dir
let data_dir = self.datadir.unwrap_or_chain_default(self.chain.chain);
let db_path = data_dir.db_path();
let static_files_path = data_dir.static_files_path();
match self.command {
// TODO: We'll need to add this on the DB trait.
Subcommands::Stats { .. } => {
Subcommands::Stats(command) => {
let db = open_db_read_only(
&db_path,
DatabaseArguments::default().log_level(self.db.log_level),
)?;
let tool = DbTool::new(&db, self.chain.clone())?;
let mut stats_table = ComfyTable::new();
stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
stats_table.set_header([
"Table Name",
"# Entries",
"Branch Pages",
"Leaf Pages",
"Overflow Pages",
"Total Size",
]);
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path)?;
tool.db.view(|tx| {
let mut tables =
Tables::ALL.iter().map(|table| table.name()).collect::<Vec<_>>();
tables.sort();
let mut total_size = 0;
for table in tables {
let table_db =
tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?;
let stats = tx
.inner
.db_stat(&table_db)
.wrap_err(format!("Could not find table: {table}"))?;
// Defaults to 16KB right now but we should
// re-evaluate depending on the DB we end up using
// (e.g. REDB does not have these options as configurable intentionally)
let page_size = stats.page_size() as usize;
let leaf_pages = stats.leaf_pages();
let branch_pages = stats.branch_pages();
let overflow_pages = stats.overflow_pages();
let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages;
total_size += table_size;
let mut row = Row::new();
row.add_cell(Cell::new(table))
.add_cell(Cell::new(stats.entries()))
.add_cell(Cell::new(branch_pages))
.add_cell(Cell::new(leaf_pages))
.add_cell(Cell::new(overflow_pages))
.add_cell(Cell::new(human_bytes(table_size as f64)));
stats_table.add_row(row);
}
let max_widths = stats_table.column_max_content_widths();
let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
stats_table.add_row(seperator);
let mut row = Row::new();
row.add_cell(Cell::new("Total DB size"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_size as f64)));
stats_table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let freelist_size = freelist *
tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
let mut row = Row::new();
row.add_cell(Cell::new("Freelist size"))
.add_cell(Cell::new(freelist))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(freelist_size as f64)));
stats_table.add_row(row);
Ok::<(), eyre::Report>(())
})??;
println!("{stats_table}");
let tool = DbTool::new(provider_factory, self.chain.clone())?;
command.execute(data_dir, &tool)?;
}
Subcommands::List(command) => {
let db = open_db_read_only(
&db_path,
DatabaseArguments::default().log_level(self.db.log_level),
)?;
let tool = DbTool::new(&db, self.chain.clone())?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path)?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
command.execute(&tool)?;
}
Subcommands::Diff(command) => {
@ -203,7 +127,10 @@ impl Command {
&db_path,
DatabaseArguments::default().log_level(self.db.log_level),
)?;
let tool = DbTool::new(&db, self.chain.clone())?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path)?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
command.execute(&tool)?;
}
Subcommands::Get(command) => {
@ -211,13 +138,16 @@ impl Command {
&db_path,
DatabaseArguments::default().log_level(self.db.log_level),
)?;
let tool = DbTool::new(&db, self.chain.clone())?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path)?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
command.execute(&tool)?;
}
Subcommands::Drop { force } => {
if !force {
// Ask for confirmation
print!("Are you sure you want to drop the database at {db_path:?}? This cannot be undone. (y/N): ");
print!("Are you sure you want to drop the database at {data_dir}? This cannot be undone. (y/N): ");
// Flush the buffer to ensure the message is printed immediately
io::stdout().flush().unwrap();
@ -232,16 +162,22 @@ impl Command {
let db =
open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?;
let mut tool = DbTool::new(&db, self.chain.clone())?;
tool.drop(db_path)?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path.clone())?;
let mut tool = DbTool::new(provider_factory, self.chain.clone())?;
tool.drop(db_path, static_files_path)?;
}
Subcommands::Clear(command) => {
let db =
open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?;
command.execute(&db)?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), static_files_path)?;
command.execute(provider_factory)?;
}
Subcommands::Snapshot(command) => {
command.execute(&db_path, self.db.log_level, self.chain.clone())?;
Subcommands::CreateStaticFiles(command) => {
command.execute(data_dir, self.db.log_level, self.chain.clone())?;
}
Subcommands::Version => {
let local_db_version = match get_db_version(&db_path) {

23
bin/reth/src/commands/db/snapshots/bench.rs → bin/reth/src/commands/db/static_files/bench.rs
View File

@ -1,7 +1,7 @@
use reth_db::DatabaseEnv;
use reth_primitives::{
snapshot::{Compression, Filters},
ChainSpec, SnapshotSegment,
static_file::{Compression, Filters},
StaticFileSegment,
};
use reth_provider::{DatabaseProviderRO, ProviderFactory};
use std::{fmt::Debug, sync::Arc, time::Instant};
@ -16,11 +16,11 @@ pub(crate) enum BenchKind {
pub(crate) fn bench<F1, F2, R>(
bench_kind: BenchKind,
db: (DatabaseEnv, Arc<ChainSpec>),
segment: SnapshotSegment,
provider_factory: Arc<ProviderFactory<DatabaseEnv>>,
segment: StaticFileSegment,
filters: Filters,
compression: Compression,
mut snapshot_method: F1,
mut static_file_method: F1,
database_method: F2,
) -> eyre::Result<()>
where
@ -28,22 +28,19 @@ where
F2: Fn(DatabaseProviderRO<DatabaseEnv>) -> eyre::Result<R>,
R: Debug + PartialEq,
{
let (db, chain) = db;
println!();
println!("############");
println!("## [{segment:?}] [{compression:?}] [{filters:?}] [{bench_kind:?}]");
let snap_result = {
let static_file_result = {
let start = Instant::now();
let result = snapshot_method()?;
let result = static_file_method()?;
let end = start.elapsed().as_micros();
println!("# snapshot {bench_kind:?} | {end} μs");
println!("# static file {bench_kind:?} | {end} μs");
result
};
let db_result = {
let factory = ProviderFactory::new(db, chain);
let provider = factory.provider()?;
let provider = provider_factory.provider()?;
let start = Instant::now();
let result = database_method(provider)?;
let end = start.elapsed().as_micros();
@ -51,7 +48,7 @@ where
result
};
assert_eq!(snap_result, db_result);
assert_eq!(static_file_result, db_result);
Ok(())
}

65
bin/reth/src/commands/db/snapshots/headers.rs → bin/reth/src/commands/db/static_files/headers.rs
View File

@ -3,38 +3,27 @@ use super::{
Command,
};
use rand::{seq::SliceRandom, Rng};
use reth_db::{mdbx::DatabaseArguments, open_db_read_only, snapshot::HeaderMask};
use reth_interfaces::db::LogLevel;
use reth_db::{static_file::HeaderMask, DatabaseEnv};
use reth_primitives::{
snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction},
BlockHash, ChainSpec, Header, SnapshotSegment,
static_file::{Compression, Filters, InclusionFilter, PerfectHashingFunction},
BlockHash, Header, StaticFileSegment,
};
use reth_provider::{
providers::SnapshotProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory,
TransactionsProviderExt,
};
use std::{
path::{Path, PathBuf},
sync::Arc,
providers::StaticFileProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory,
};
use std::{ops::RangeInclusive, path::PathBuf, sync::Arc};
impl Command {
pub(crate) fn bench_headers_snapshot(
pub(crate) fn bench_headers_static_file(
&self,
db_path: &Path,
log_level: Option<LogLevel>,
chain: Arc<ChainSpec>,
provider_factory: Arc<ProviderFactory<DatabaseEnv>>,
compression: Compression,
inclusion_filter: InclusionFilter,
phf: Option<PerfectHashingFunction>,
) -> eyre::Result<()> {
let db_args = DatabaseArguments::default().log_level(log_level);
let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
let provider = factory.provider()?;
let provider = provider_factory.provider()?;
let tip = provider.last_block_number()?;
let block_range =
self.block_ranges(tip).first().expect("has been generated before").clone();
let block_range = *self.block_ranges(tip).first().expect("has been generated before");
let filters = if let Some(phf) = self.with_filters.then_some(phf).flatten() {
Filters::WithFilters(inclusion_filter, phf)
@ -42,19 +31,16 @@ impl Command {
Filters::WithoutFilters
};
let mut row_indexes = block_range.clone().collect::<Vec<_>>();
let range: RangeInclusive<u64> = (&block_range).into();
let mut row_indexes = range.collect::<Vec<_>>();
let mut rng = rand::thread_rng();
let tx_range = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
.provider()?
.transaction_range_by_block_range(block_range.clone())?;
let path: PathBuf = SnapshotSegment::Headers
.filename_with_configuration(filters, compression, &block_range, &tx_range)
let path: PathBuf = StaticFileSegment::Headers
.filename_with_configuration(filters, compression, &block_range)
.into();
let provider = SnapshotProvider::new(PathBuf::default())?;
let provider = StaticFileProvider::new(PathBuf::default())?;
let jar_provider = provider.get_segment_provider_from_block(
SnapshotSegment::Headers,
StaticFileSegment::Headers,
self.from,
Some(&path),
)?;
@ -63,8 +49,8 @@ impl Command {
for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
bench(
bench_kind,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Headers,
provider_factory.clone(),
StaticFileSegment::Headers,
filters,
compression,
|| {
@ -94,8 +80,8 @@ impl Command {
let num = row_indexes[rng.gen_range(0..row_indexes.len())];
bench(
BenchKind::RandomOne,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Headers,
provider_factory.clone(),
StaticFileSegment::Headers,
filters,
compression,
|| {
@ -114,16 +100,15 @@ impl Command {
// BENCHMARK QUERYING A RANDOM HEADER BY HASH
{
let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
let header_hash =
ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
.header_by_number(num)?
.ok_or(ProviderError::HeaderNotFound(num.into()))?
.hash_slow();
let header_hash = provider_factory
.header_by_number(num)?
.ok_or(ProviderError::HeaderNotFound(num.into()))?
.hash_slow();
bench(
BenchKind::RandomHash,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Headers,
provider_factory.clone(),
StaticFileSegment::Headers,
filters,
compression,
|| {

161
bin/reth/src/commands/db/snapshots/mod.rs → bin/reth/src/commands/db/static_files/mod.rs
View File

@ -9,14 +9,17 @@ use reth_db::{
};
use reth_interfaces::db::LogLevel;
use reth_nippy_jar::{NippyJar, NippyJarCursor};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::{
snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentHeader},
BlockNumber, ChainSpec, SnapshotSegment,
static_file::{
Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentConfig,
SegmentHeader, SegmentRangeInclusive,
},
BlockNumber, ChainSpec, StaticFileSegment,
};
use reth_provider::{BlockNumReader, ProviderFactory, TransactionsProviderExt};
use reth_snapshot::{segments as snap_segments, segments::Segment};
use reth_provider::{BlockNumReader, ProviderFactory};
use reth_static_file::{segments as static_file_segments, segments::Segment};
use std::{
ops::RangeInclusive,
path::{Path, PathBuf},
sync::Arc,
time::{Duration, Instant},
@ -28,20 +31,20 @@ mod receipts;
mod transactions;
#[derive(Parser, Debug)]
/// Arguments for the `reth db snapshot` command.
/// Arguments for the `reth db create-static-files` command.
pub struct Command {
/// Snapshot segments to generate.
segments: Vec<SnapshotSegment>,
/// Static File segments to generate.
segments: Vec<StaticFileSegment>,
/// Starting block for the snapshot.
/// Starting block for the static file.
#[arg(long, short, default_value = "0")]
from: BlockNumber,
/// Number of blocks in the snapshot.
/// Number of blocks in the static file.
#[arg(long, short, default_value = "500000")]
block_interval: u64,
/// Sets the number of snapshots built in parallel. Note: Each parallel build is
/// Sets the number of static files built in parallel. Note: Each parallel build is
/// memory-intensive.
#[arg(
long, short,
@ -50,15 +53,15 @@ pub struct Command {
)]
parallel: u64,
/// Flag to skip snapshot creation and print snapshot files stats.
/// Flag to skip static file creation and print static files stats.
#[arg(long, default_value = "false")]
only_stats: bool,
/// Flag to enable database-to-snapshot benchmarking.
/// Flag to enable database-to-static file benchmarking.
#[arg(long, default_value = "false")]
bench: bool,
/// Flag to skip snapshot creation and only run benchmarks on existing snapshots.
/// Flag to skip static file creation and only run benchmarks on existing static files.
#[arg(long, default_value = "false")]
only_bench: bool,
@ -76,30 +79,33 @@ pub struct Command {
}
impl Command {
/// Execute `db snapshot` command
/// Execute `db create-static-files` command
pub fn execute(
self,
db_path: &Path,
data_dir: ChainPath<DataDirPath>,
log_level: Option<LogLevel>,
chain: Arc<ChainSpec>,
) -> eyre::Result<()> {
let all_combinations =
self.segments.iter().cartesian_product(self.compression.iter()).cartesian_product(
if self.phf.is_empty() {
vec![None]
} else {
self.phf.iter().copied().map(Some).collect::<Vec<_>>()
},
);
let all_combinations = self
.segments
.iter()
.cartesian_product(self.compression.iter().copied())
.cartesian_product(if self.phf.is_empty() {
vec![None]
} else {
self.phf.iter().copied().map(Some).collect::<Vec<_>>()
});
let db = open_db_read_only(
data_dir.db_path().as_path(),
DatabaseArguments::default()
.log_level(log_level)
.max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)),
)?;
let provider_factory =
Arc::new(ProviderFactory::new(db, chain.clone(), data_dir.static_files_path())?);
{
let db = open_db_read_only(
db_path,
DatabaseArguments::default()
.max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)),
)?;
let factory = Arc::new(ProviderFactory::new(db, chain.clone()));
if !self.only_bench {
for ((mode, compression), phf) in all_combinations.clone() {
let filters = if let Some(phf) = self.with_filters.then_some(phf).flatten() {
@ -109,17 +115,21 @@ impl Command {
};
match mode {
SnapshotSegment::Headers => self.generate_snapshot::<DatabaseEnv>(
factory.clone(),
snap_segments::Headers::new(*compression, filters),
StaticFileSegment::Headers => self.generate_static_file::<DatabaseEnv>(
provider_factory.clone(),
static_file_segments::Headers,
SegmentConfig { filters, compression },
)?,
SnapshotSegment::Transactions => self.generate_snapshot::<DatabaseEnv>(
factory.clone(),
snap_segments::Transactions::new(*compression, filters),
)?,
SnapshotSegment::Receipts => self.generate_snapshot::<DatabaseEnv>(
factory.clone(),
snap_segments::Receipts::new(*compression, filters),
StaticFileSegment::Transactions => self
.generate_static_file::<DatabaseEnv>(
provider_factory.clone(),
static_file_segments::Transactions,
SegmentConfig { filters, compression },
)?,
StaticFileSegment::Receipts => self.generate_static_file::<DatabaseEnv>(
provider_factory.clone(),
static_file_segments::Receipts,
SegmentConfig { filters, compression },
)?,
}
}
@ -127,29 +137,23 @@ impl Command {
}
if self.only_bench || self.bench {
for ((mode, compression), phf) in all_combinations.clone() {
for ((mode, compression), phf) in all_combinations {
match mode {
SnapshotSegment::Headers => self.bench_headers_snapshot(
db_path,
log_level,
chain.clone(),
*compression,
StaticFileSegment::Headers => self.bench_headers_static_file(
provider_factory.clone(),
compression,
InclusionFilter::Cuckoo,
phf,
)?,
SnapshotSegment::Transactions => self.bench_transactions_snapshot(
db_path,
log_level,
chain.clone(),
*compression,
StaticFileSegment::Transactions => self.bench_transactions_static_file(
provider_factory.clone(),
compression,
InclusionFilter::Cuckoo,
phf,
)?,
SnapshotSegment::Receipts => self.bench_receipts_snapshot(
db_path,
log_level,
chain.clone(),
*compression,
StaticFileSegment::Receipts => self.bench_receipts_static_file(
provider_factory.clone(),
compression,
InclusionFilter::Cuckoo,
phf,
)?,
@ -161,30 +165,31 @@ impl Command {
}
/// Generates successive inclusive block ranges up to the tip starting at `self.from`.
fn block_ranges(&self, tip: BlockNumber) -> Vec<RangeInclusive<BlockNumber>> {
fn block_ranges(&self, tip: BlockNumber) -> Vec<SegmentRangeInclusive> {
let mut from = self.from;
let mut ranges = Vec::new();
while from <= tip {
let end_range = std::cmp::min(from + self.block_interval - 1, tip);
ranges.push(from..=end_range);
ranges.push(SegmentRangeInclusive::new(from, end_range));
from = end_range + 1;
}
ranges
}
/// Generates snapshots from `self.from` with a `self.block_interval`. Generates them in
/// Generates static files from `self.from` with a `self.block_interval`. Generates them in
/// parallel if specified.
fn generate_snapshot<DB: Database>(
fn generate_static_file<DB: Database>(
&self,
factory: Arc<ProviderFactory<DB>>,
segment: impl Segment + Send + Sync,
segment: impl Segment<DB>,
config: SegmentConfig,
) -> eyre::Result<()> {
let dir = PathBuf::default();
let ranges = self.block_ranges(factory.best_block_number()?);
let mut created_snapshots = vec![];
let mut created_static_files = vec![];
// Filter/PHF is memory intensive, so we have to limit the parallelism.
for block_ranges in ranges.chunks(self.parallel as usize) {
@ -194,34 +199,36 @@ impl Command {
let provider = factory.provider()?;
if !self.only_stats {
segment.snapshot::<DB>(&provider, &dir, block_range.clone())?;
segment.create_static_file_file(
&provider,
dir.as_path(),
config,
block_range.into(),
)?;
}
let tx_range =
provider.transaction_range_by_block_range(block_range.clone())?;
Ok(segment.segment().filename(block_range, &tx_range))
Ok(segment.segment().filename(block_range))
})
.collect::<Result<Vec<_>, eyre::Report>>()?;
created_snapshots.extend(created_files);
created_static_files.extend(created_files);
}
self.stats(created_snapshots)
self.stats(created_static_files)
}
/// Prints detailed statistics for each snapshot, including loading time.
/// Prints detailed statistics for each static file, including loading time.
///
/// This function loads each snapshot from the provided paths and prints
/// statistics about various aspects of each snapshot, such as filters size,
/// This function loads each static file from the provided paths and prints
/// statistics about various aspects of each static file, such as filters size,
/// offset index size, offset list size, and loading time.
fn stats(&self, snapshots: Vec<impl AsRef<Path>>) -> eyre::Result<()> {
fn stats(&self, static_files: Vec<impl AsRef<Path>>) -> eyre::Result<()> {
let mut total_filters_size = 0;
let mut total_index_size = 0;
let mut total_duration = Duration::new(0, 0);
let mut total_file_size = 0;
for snap in &snapshots {
for snap in &static_files {
let start_time = Instant::now();
let jar = NippyJar::<SegmentHeader>::load(snap.as_ref())?;
let _cursor = NippyJarCursor::new(&jar)?;
@ -233,7 +240,7 @@ impl Command {
total_duration += duration;
total_file_size += file_size;
println!("Snapshot: {:?}", snap.as_ref().file_name());
println!("StaticFile: {:?}", snap.as_ref().file_name());
println!(" File Size: {:>7}", human_bytes(file_size as f64));
println!(" Filters Size: {:>7}", human_bytes(jar.filter_size() as f64));
println!(" Offset Index Size: {:>7}", human_bytes(jar.offsets_index_size() as f64));
@ -244,7 +251,7 @@ impl Command {
);
}
let avg_duration = total_duration / snapshots.len() as u32;
let avg_duration = total_duration / static_files.len() as u32;
println!("Total Filters Size: {:>7}", human_bytes(total_filters_size as f64));
println!("Total Offset Index Size: {:>7}", human_bytes(total_index_size as f64));

55
bin/reth/src/commands/db/snapshots/receipts.rs → bin/reth/src/commands/db/static_files/receipts.rs
View File

@ -3,38 +3,28 @@ use super::{
Command, Compression, PerfectHashingFunction,
};
use rand::{seq::SliceRandom, Rng};
use reth_db::{mdbx::DatabaseArguments, open_db_read_only, snapshot::ReceiptMask};
use reth_interfaces::db::LogLevel;
use reth_db::{static_file::ReceiptMask, DatabaseEnv};
use reth_primitives::{
snapshot::{Filters, InclusionFilter},
ChainSpec, Receipt, SnapshotSegment,
static_file::{Filters, InclusionFilter},
Receipt, StaticFileSegment,
};
use reth_provider::{
providers::SnapshotProvider, BlockNumReader, ProviderError, ProviderFactory, ReceiptProvider,
providers::StaticFileProvider, BlockNumReader, ProviderError, ProviderFactory, ReceiptProvider,
TransactionsProvider, TransactionsProviderExt,
};
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use std::{path::PathBuf, sync::Arc};
impl Command {
pub(crate) fn bench_receipts_snapshot(
pub(crate) fn bench_receipts_static_file(
&self,
db_path: &Path,
log_level: Option<LogLevel>,
chain: Arc<ChainSpec>,
provider_factory: Arc<ProviderFactory<DatabaseEnv>>,
compression: Compression,
inclusion_filter: InclusionFilter,
phf: Option<PerfectHashingFunction>,
) -> eyre::Result<()> {
let db_args = DatabaseArguments::default().log_level(log_level);
let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
let provider = factory.provider()?;
let provider = provider_factory.provider()?;
let tip = provider.last_block_number()?;
let block_range =
self.block_ranges(tip).first().expect("has been generated before").clone();
let block_range = *self.block_ranges(tip).first().expect("has been generated before");
let filters = if let Some(phf) = self.with_filters.then_some(phf).flatten() {
Filters::WithFilters(inclusion_filter, phf)
@ -44,19 +34,18 @@ impl Command {
let mut rng = rand::thread_rng();
let tx_range = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
.provider()?
.transaction_range_by_block_range(block_range.clone())?;
let tx_range =
provider_factory.provider()?.transaction_range_by_block_range(block_range.into())?;
let mut row_indexes = tx_range.clone().collect::<Vec<_>>();
let path: PathBuf = SnapshotSegment::Receipts
.filename_with_configuration(filters, compression, &block_range, &tx_range)
let path: PathBuf = StaticFileSegment::Receipts
.filename_with_configuration(filters, compression, &block_range)
.into();
let provider = SnapshotProvider::new(PathBuf::default())?;
let provider = StaticFileProvider::new(PathBuf::default())?;
let jar_provider = provider.get_segment_provider_from_block(
SnapshotSegment::Receipts,
StaticFileSegment::Receipts,
self.from,
Some(&path),
)?;
@ -65,8 +54,8 @@ impl Command {
for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
bench(
bench_kind,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Receipts,
provider_factory.clone(),
StaticFileSegment::Receipts,
filters,
compression,
|| {
@ -96,8 +85,8 @@ impl Command {
let num = row_indexes[rng.gen_range(0..row_indexes.len())];
bench(
BenchKind::RandomOne,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Receipts,
provider_factory.clone(),
StaticFileSegment::Receipts,
filters,
compression,
|| {
@ -116,15 +105,15 @@ impl Command {
// BENCHMARK QUERYING A RANDOM RECEIPT BY HASH
{
let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
let tx_hash = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
let tx_hash = provider_factory
.transaction_by_id(num)?
.ok_or(ProviderError::ReceiptNotFound(num.into()))?
.hash();
bench(
BenchKind::RandomHash,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Receipts,
provider_factory,
StaticFileSegment::Receipts,
filters,
compression,
|| {

60
bin/reth/src/commands/db/snapshots/transactions.rs → bin/reth/src/commands/db/static_files/transactions.rs
View File

@ -3,38 +3,29 @@ use super::{
Command, Compression, PerfectHashingFunction,
};
use rand::{seq::SliceRandom, Rng};
use reth_db::{mdbx::DatabaseArguments, open_db_read_only, snapshot::TransactionMask};
use reth_interfaces::db::LogLevel;
use reth_db::{static_file::TransactionMask, DatabaseEnv};
use reth_primitives::{
snapshot::{Filters, InclusionFilter},
ChainSpec, SnapshotSegment, TransactionSignedNoHash,
static_file::{Filters, InclusionFilter},
StaticFileSegment, TransactionSignedNoHash,
};
use reth_provider::{
providers::SnapshotProvider, BlockNumReader, ProviderError, ProviderFactory,
providers::StaticFileProvider, BlockNumReader, ProviderError, ProviderFactory,
TransactionsProvider, TransactionsProviderExt,
};
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use std::{path::PathBuf, sync::Arc};
impl Command {
pub(crate) fn bench_transactions_snapshot(
pub(crate) fn bench_transactions_static_file(
&self,
db_path: &Path,
log_level: Option<LogLevel>,
chain: Arc<ChainSpec>,
provider_factory: Arc<ProviderFactory<DatabaseEnv>>,
compression: Compression,
inclusion_filter: InclusionFilter,
phf: Option<PerfectHashingFunction>,
) -> eyre::Result<()> {
let db_args = DatabaseArguments::default().log_level(log_level);
let factory = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone());
let provider = factory.provider()?;
let provider = provider_factory.provider()?;
let tip = provider.last_block_number()?;
let block_range =
self.block_ranges(tip).first().expect("has been generated before").clone();
let block_range = *self.block_ranges(tip).first().expect("has been generated before");
let filters = if let Some(phf) = self.with_filters.then_some(phf).flatten() {
Filters::WithFilters(inclusion_filter, phf)
@ -44,16 +35,16 @@ impl Command {
let mut rng = rand::thread_rng();
let tx_range = provider.transaction_range_by_block_range(block_range.clone())?;
let tx_range = provider.transaction_range_by_block_range(block_range.into())?;
let mut row_indexes = tx_range.clone().collect::<Vec<_>>();
let path: PathBuf = SnapshotSegment::Transactions
.filename_with_configuration(filters, compression, &block_range, &tx_range)
let path: PathBuf = StaticFileSegment::Transactions
.filename_with_configuration(filters, compression, &block_range)
.into();
let provider = SnapshotProvider::new(PathBuf::default())?;
let provider = StaticFileProvider::new(PathBuf::default())?;
let jar_provider = provider.get_segment_provider_from_block(
SnapshotSegment::Transactions,
StaticFileSegment::Transactions,
self.from,
Some(&path),
)?;
@ -62,8 +53,8 @@ impl Command {
for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] {
bench(
bench_kind,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Transactions,
provider_factory.clone(),
StaticFileSegment::Transactions,
filters,
compression,
|| {
@ -94,8 +85,8 @@ impl Command {
let num = row_indexes[rng.gen_range(0..row_indexes.len())];
bench(
BenchKind::RandomOne,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Transactions,
provider_factory.clone(),
StaticFileSegment::Transactions,
filters,
compression,
|| {
@ -115,16 +106,15 @@ impl Command {
// BENCHMARK QUERYING A RANDOM TRANSACTION BY HASH
{
let num = row_indexes[rng.gen_range(0..row_indexes.len())] as u64;
let transaction_hash =
ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
.transaction_by_id(num)?
.ok_or(ProviderError::TransactionNotFound(num.into()))?
.hash();
let transaction_hash = provider_factory
.transaction_by_id(num)?
.ok_or(ProviderError::TransactionNotFound(num.into()))?
.hash();
bench(
BenchKind::RandomHash,
(open_db_read_only(db_path, db_args)?, chain.clone()),
SnapshotSegment::Transactions,
provider_factory,
StaticFileSegment::Transactions,
filters,
compression,
|| {

290
bin/reth/src/commands/db/stats.rs Normal file
View File

@ -0,0 +1,290 @@
use crate::utils::DbTool;
use clap::Parser;
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::WrapErr;
use human_bytes::human_bytes;
use itertools::Itertools;
use reth_db::{database::Database, mdbx, static_file::iter_static_files, DatabaseEnv, Tables};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::static_file::{find_fixed_range, SegmentRangeInclusive};
use reth_provider::providers::StaticFileProvider;
use std::fs::File;
#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
pub struct Command {
/// Show only the total size for static files.
#[arg(long, default_value_t = false)]
only_total_size: bool,
/// Show only the summary per static file segment.
#[arg(long, default_value_t = false)]
summary: bool,
}
impl Command {
/// Execute `db stats` command
pub fn execute(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<DatabaseEnv>,
) -> eyre::Result<()> {
let static_files_stats_table = self.static_files_stats_table(data_dir)?;
println!("{static_files_stats_table}");
println!("\n");
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
Ok(())
}
fn db_stats_table(&self, tool: &DbTool<DatabaseEnv>) -> eyre::Result<ComfyTable> {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"Table Name",
"# Entries",
"Branch Pages",
"Leaf Pages",
"Overflow Pages",
"Total Size",
]);
tool.provider_factory.db_ref().view(|tx| {
let mut db_tables = Tables::ALL.iter().map(|table| table.name()).collect::<Vec<_>>();
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
let stats = tx
.inner
.db_stat(&table_db)
.wrap_err(format!("Could not find table: {db_table}"))?;
// Defaults to 16KB right now but we should
// re-evaluate depending on the DB we end up using
// (e.g. REDB does not have these options as configurable intentionally)
let page_size = stats.page_size() as usize;
let leaf_pages = stats.leaf_pages();
let branch_pages = stats.branch_pages();
let overflow_pages = stats.overflow_pages();
let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages;
total_size += table_size;
let mut row = Row::new();
row.add_cell(Cell::new(db_table))
.add_cell(Cell::new(stats.entries()))
.add_cell(Cell::new(branch_pages))
.add_cell(Cell::new(leaf_pages))
.add_cell(Cell::new(overflow_pages))
.add_cell(Cell::new(human_bytes(table_size as f64)));
table.add_row(row);
}
let max_widths = table.column_max_content_widths();
let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(seperator);
let mut row = Row::new();
row.add_cell(Cell::new("Tables"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let freelist_size =
freelist * tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
let mut row = Row::new();
row.add_cell(Cell::new("Freelist"))
.add_cell(Cell::new(freelist))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(freelist_size as f64)));
table.add_row(row);
Ok::<(), eyre::Report>(())
})??;
Ok(table)
}
fn static_files_stats_table(
&self,
data_dir: ChainPath<DataDirPath>,
) -> eyre::Result<ComfyTable> {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
if !self.only_total_size {
table.set_header([
"Segment",
"Block Range",
"Transaction Range",
"Shape (columns x rows)",
"Data Size",
"Index Size",
"Offsets Size",
"Config Size",
"Total Size",
]);
} else {
table.set_header([
"Segment",
"Block Range",
"Transaction Range",
"Shape (columns x rows)",
"Size",
]);
}
let static_files = iter_static_files(data_dir.static_files_path())?;
let static_file_provider = StaticFileProvider::new(data_dir.static_files_path())?;
let mut total_data_size = 0;
let mut total_index_size = 0;
let mut total_offsets_size = 0;
let mut total_config_size = 0;
for (segment, ranges) in static_files.into_iter().sorted_by_key(|(segment, _)| *segment) {
let (
mut segment_columns,
mut segment_rows,
mut segment_data_size,
mut segment_index_size,
mut segment_offsets_size,
mut segment_config_size,
) = (0, 0, 0, 0, 0, 0);
for (block_range, tx_range) in &ranges {
let fixed_block_range = find_fixed_range(block_range.start());
let jar_provider = static_file_provider
.get_segment_provider(segment, || Some(fixed_block_range), None)?
.expect("something went wrong");
let columns = jar_provider.columns();
let rows = jar_provider.rows();
let data_size = File::open(jar_provider.data_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let index_size = File::open(jar_provider.index_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let offsets_size = File::open(jar_provider.offsets_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let config_size = File::open(jar_provider.config_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
if self.summary {
if segment_columns > 0 {
assert_eq!(segment_columns, columns);
} else {
segment_columns = columns;
}
segment_rows += rows;
segment_data_size += data_size;
segment_index_size += index_size;
segment_offsets_size += offsets_size;
segment_config_size += config_size;
} else {
let mut row = Row::new();
row.add_cell(Cell::new(segment))
.add_cell(Cell::new(format!("{block_range}")))
.add_cell(Cell::new(
tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range}")),
))
.add_cell(Cell::new(format!("{columns} x {rows}")));
if !self.only_total_size {
row.add_cell(Cell::new(human_bytes(data_size as f64)))
.add_cell(Cell::new(human_bytes(index_size as f64)))
.add_cell(Cell::new(human_bytes(offsets_size as f64)))
.add_cell(Cell::new(human_bytes(config_size as f64)));
}
row.add_cell(Cell::new(human_bytes(
(data_size + index_size + offsets_size + config_size) as f64,
)));
table.add_row(row);
}
total_data_size += data_size;
total_index_size += index_size;
total_offsets_size += offsets_size;
total_config_size += config_size;
}
if self.summary {
let first_ranges = ranges.first().expect("not empty list of ranges");
let last_ranges = ranges.last().expect("not empty list of ranges");
let block_range =
SegmentRangeInclusive::new(first_ranges.0.start(), last_ranges.0.end());
let tx_range = first_ranges
.1
.zip(last_ranges.1)
.map(|(first, last)| SegmentRangeInclusive::new(first.start(), last.end()));
let mut row = Row::new();
row.add_cell(Cell::new(segment))
.add_cell(Cell::new(format!("{block_range}")))
.add_cell(Cell::new(
tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range}")),
))
.add_cell(Cell::new(format!("{segment_columns} x {segment_rows}")));
if !self.only_total_size {
row.add_cell(Cell::new(human_bytes(segment_data_size as f64)))
.add_cell(Cell::new(human_bytes(segment_index_size as f64)))
.add_cell(Cell::new(human_bytes(segment_offsets_size as f64)))
.add_cell(Cell::new(human_bytes(segment_config_size as f64)));
}
row.add_cell(Cell::new(human_bytes(
(segment_data_size +
segment_index_size +
segment_offsets_size +
segment_config_size) as f64,
)));
table.add_row(row);
}
}
let max_widths = table.column_max_content_widths();
let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(seperator);
let mut row = Row::new();
row.add_cell(Cell::new("Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""));
if !self.only_total_size {
row.add_cell(Cell::new(human_bytes(total_data_size as f64)))
.add_cell(Cell::new(human_bytes(total_index_size as f64)))
.add_cell(Cell::new(human_bytes(total_offsets_size as f64)))
.add_cell(Cell::new(human_bytes(total_config_size as f64)));
}
row.add_cell(Cell::new(human_bytes(
(total_data_size + total_index_size + total_offsets_size + total_config_size) as f64,
)));
table.add_row(row);
Ok(table)
}
}

12
bin/reth/src/commands/debug_cmd/build_block.rs
View File

@ -116,7 +116,11 @@ impl Command {
///
/// If the database is empty, returns the genesis block.
fn lookup_best_block(&self, db: Arc<DatabaseEnv>) -> RethResult<Arc<SealedBlock>> {
let factory = ProviderFactory::new(db, self.chain.clone());
let factory = ProviderFactory::new(
db,
self.chain.clone(),
self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(),
)?;
let provider = factory.provider()?;
let best_number =
@ -155,7 +159,11 @@ impl Command {
// initialize the database
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let provider_factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain));
let provider_factory = ProviderFactory::new(
Arc::clone(&db),
Arc::clone(&self.chain),
data_dir.static_files_path(),
)?;
let consensus: Arc<dyn Consensus> = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain)));

34
bin/reth/src/commands/debug_cmd/execution.rs
View File

@ -27,13 +27,16 @@ use reth_network::{NetworkEvents, NetworkHandle};
use reth_network_api::NetworkInfo;
use reth_node_core::init::init_genesis;
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::{fs, stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, B256};
use reth_primitives::{
fs, stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, PruneModes, B256,
};
use reth_provider::{BlockExecutionWriter, HeaderSyncMode, ProviderFactory, StageCheckpointReader};
use reth_stages::{
sets::DefaultStages,
stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage, TotalDifficultyStage},
stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage},
Pipeline, StageSet,
};
use reth_static_file::StaticFileProducer;
use reth_tasks::TaskExecutor;
use std::{
net::{SocketAddr, SocketAddrV4},
@ -92,6 +95,7 @@ impl Command {
consensus: Arc<dyn Consensus>,
provider_factory: ProviderFactory<DB>,
task_executor: &TaskExecutor,
static_file_producer: StaticFileProducer<DB>,
) -> eyre::Result<Pipeline<DB>>
where
DB: Database + Unpin + Clone + 'static,
@ -123,11 +127,7 @@ impl Command {
header_downloader,
body_downloader,
factory.clone(),
)
.set(
TotalDifficultyStage::new(consensus)
.with_commit_threshold(stage_conf.total_difficulty.commit_threshold),
)
)?
.set(SenderRecoveryStage {
commit_threshold: stage_conf.sender_recovery.commit_threshold,
})
@ -147,7 +147,7 @@ impl Command {
config.prune.clone().map(|prune| prune.segments).unwrap_or_default(),
)),
)
.build(provider_factory);
.build(provider_factory, static_file_producer);
Ok(pipeline)
}
@ -170,7 +170,11 @@ impl Command {
self.network.discovery.addr,
self.network.discovery.port,
)))
.build(ProviderFactory::new(db, self.chain.clone()))
.build(ProviderFactory::new(
db,
self.chain.clone(),
self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(),
)?)
.start_network()
.await?;
info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network");
@ -206,10 +210,11 @@ impl Command {
fs::create_dir_all(&db_path)?;
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let provider_factory = ProviderFactory::new(db.clone(), self.chain.clone());
let provider_factory =
ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?;
debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis(db.clone(), self.chain.clone())?;
init_genesis(provider_factory.clone())?;
let consensus: Arc<dyn Consensus> = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain)));
@ -226,6 +231,12 @@ impl Command {
)
.await?;
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
// Configure the pipeline
let fetch_client = network.fetch_client().await?;
let mut pipeline = self.build_pipeline(
@ -234,6 +245,7 @@ impl Command {
Arc::clone(&consensus),
provider_factory.clone(),
&ctx.task_executor,
static_file_producer,
)?;
let provider = provider_factory.provider()?;

23
bin/reth/src/commands/debug_cmd/in_memory_merkle.rs
View File

@ -20,9 +20,8 @@ use reth_network_api::NetworkInfo;
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::{fs, stage::StageId, BlockHashOrNumber, ChainSpec};
use reth_provider::{
AccountExtReader, BlockWriter, ExecutorFactory, HashingWriter, HeaderProvider,
LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
StorageReader,
AccountExtReader, ExecutorFactory, HashingWriter, HeaderProvider, LatestStateProviderRef,
OriginalValuesKnown, ProviderFactory, StageCheckpointReader, StorageReader,
};
use reth_tasks::TaskExecutor;
use reth_trie::{updates::TrieKey, StateRoot};
@ -95,7 +94,11 @@ impl Command {
self.network.discovery.addr,
self.network.discovery.port,
)))
.build(ProviderFactory::new(db, self.chain.clone()))
.build(ProviderFactory::new(
db,
self.chain.clone(),
self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(),
)?)
.start_network()
.await?;
info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network");
@ -115,7 +118,7 @@ impl Command {
// initialize the database
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let factory = ProviderFactory::new(&db, self.chain.clone());
let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?;
let provider = factory.provider()?;
// Look up merkle checkpoint
@ -165,8 +168,10 @@ impl Command {
let executor_factory =
reth_revm::EvmProcessorFactory::new(self.chain.clone(), EthEvmConfig::default());
let mut executor =
executor_factory.with_state(LatestStateProviderRef::new(provider.tx_ref()));
let mut executor = executor_factory.with_state(LatestStateProviderRef::new(
provider.tx_ref(),
factory.static_file_provider(),
));
let merkle_block_td =
provider.header_td_by_number(merkle_block_number)?.unwrap_or_default();
@ -192,14 +197,14 @@ impl Command {
let provider_rw = factory.provider_rw()?;
// Insert block, state and hashes
provider_rw.insert_block(
provider_rw.insert_historical_block(
block
.clone()
.try_seal_with_senders()
.map_err(|_| BlockValidationError::SenderRecoveryError)?,
None,
)?;
block_state.write_to_db(provider_rw.tx_ref(), OriginalValuesKnown::No)?;
block_state.write_to_storage(provider_rw.tx_ref(), None, OriginalValuesKnown::No)?;
let storage_lists = provider_rw.changed_storages_with_range(block.number..=block.number)?;
let storages = provider_rw.plain_state_storages(storage_lists)?;
provider_rw.insert_storage_for_hashing(storages)?;

8
bin/reth/src/commands/debug_cmd/merkle.rs
View File

@ -105,7 +105,11 @@ impl Command {
self.network.discovery.addr,
self.network.discovery.port,
)))
.build(ProviderFactory::new(db, self.chain.clone()))
.build(ProviderFactory::new(
db,
self.chain.clone(),
self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(),
)?)
.start_network()
.await?;
info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network");
@ -125,7 +129,7 @@ impl Command {
// initialize the database
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let factory = ProviderFactory::new(&db, self.chain.clone());
let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?;
let provider_rw = factory.provider_rw()?;
// Configure and build network

21
bin/reth/src/commands/debug_cmd/replay_engine.rs
View File

@ -25,10 +25,11 @@ use reth_node_ethereum::{EthEngineTypes, EthEvmConfig};
#[cfg(feature = "optimism")]
use reth_node_optimism::{OptimismEngineTypes, OptimismEvmConfig};
use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService};
use reth_primitives::{fs, ChainSpec};
use reth_primitives::{fs, ChainSpec, PruneModes};
use reth_provider::{providers::BlockchainProvider, CanonStateSubscriptions, ProviderFactory};
use reth_revm::EvmProcessorFactory;
use reth_stages::Pipeline;
use reth_static_file::StaticFileProducer;
use reth_tasks::TaskExecutor;
use reth_transaction_pool::noop::NoopTransactionPool;
use std::{
@ -101,7 +102,11 @@ impl Command {
self.network.discovery.addr,
self.network.discovery.port,
)))
.build(ProviderFactory::new(db, self.chain.clone()))
.build(ProviderFactory::new(
db,
self.chain.clone(),
self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(),
)?)
.start_network()
.await?;
info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network");
@ -121,7 +126,8 @@ impl Command {
// Initialize the database
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let provider_factory = ProviderFactory::new(db.clone(), self.chain.clone());
let provider_factory =
ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?;
let consensus: Arc<dyn Consensus> = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain)));
@ -191,7 +197,14 @@ impl Command {
let (consensus_engine_tx, consensus_engine_rx) = mpsc::unbounded_channel();
let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel(
network_client,
Pipeline::builder().build(provider_factory),
Pipeline::builder().build(
provider_factory.clone(),
StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
),
),
blockchain_db.clone(),
Box::new(ctx.task_executor.clone()),
Box::new(network),

33
bin/reth/src/commands/import.rs
View File

@ -21,12 +21,13 @@ use reth_downloaders::{
use reth_interfaces::consensus::Consensus;
use reth_node_core::{events::node::NodeEvent, init::init_genesis};
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::{stage::StageId, ChainSpec, B256};
use reth_primitives::{stage::StageId, ChainSpec, PruneModes, B256};
use reth_provider::{HeaderSyncMode, ProviderFactory, StageCheckpointReader};
use reth_stages::{
prelude::*,
stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage, TotalDifficultyStage},
stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage},
};
use reth_static_file::StaticFileProducer;
use std::{path::PathBuf, sync::Arc};
use tokio::sync::watch;
use tracing::{debug, info};
@ -89,11 +90,12 @@ impl ImportCommand {
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
info!(target: "reth::cli", "Database opened");
let provider_factory = ProviderFactory::new(db.clone(), self.chain.clone());
let provider_factory =
ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?;
debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis(db.clone(), self.chain.clone())?;
init_genesis(provider_factory.clone())?;
let consensus = Arc::new(BeaconConsensus::new(self.chain.clone()));
info!(target: "reth::cli", "Consensus engine initialized");
@ -106,8 +108,20 @@ impl ImportCommand {
let tip = file_client.tip().expect("file client has no tip");
info!(target: "reth::cli", "Chain file imported");
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
let (mut pipeline, events) = self
.build_import_pipeline(config, provider_factory.clone(), &consensus, file_client)
.build_import_pipeline(
config,
provider_factory.clone(),
&consensus,
file_client,
static_file_producer,
)
.await?;
// override the tip
@ -142,6 +156,7 @@ impl ImportCommand {
provider_factory: ProviderFactory<DB>,
consensus: &Arc<C>,
file_client: Arc<FileClient>,
static_file_producer: StaticFileProducer<DB>,
) -> eyre::Result<(Pipeline<DB>, impl Stream<Item = NodeEvent>)>
where
DB: Database + Clone + Unpin + 'static,
@ -176,11 +191,7 @@ impl ImportCommand {
header_downloader,
body_downloader,
factory.clone(),
)
.set(
TotalDifficultyStage::new(consensus.clone())
.with_commit_threshold(config.stages.total_difficulty.commit_threshold),
)
)?
.set(SenderRecoveryStage {
commit_threshold: config.stages.sender_recovery.commit_threshold,
})
@ -201,7 +212,7 @@ impl ImportCommand {
config.prune.map(|prune| prune.segments).unwrap_or_default(),
)),
)
.build(provider_factory);
.build(provider_factory, static_file_producer);
let events = pipeline.events().map(Into::into);

7
bin/reth/src/commands/init_cmd.rs
View File

@ -11,6 +11,7 @@ use clap::Parser;
use reth_db::{init_db, mdbx::DatabaseArguments};
use reth_node_core::init::init_genesis;
use reth_primitives::ChainSpec;
use reth_provider::ProviderFactory;
use std::sync::Arc;
use tracing::info;
@ -56,8 +57,12 @@ impl InitCommand {
Arc::new(init_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
info!(target: "reth::cli", "Database opened");
let provider_factory =
ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?;
info!(target: "reth::cli", "Writing genesis block");
let hash = init_genesis(db, self.chain)?;
let hash = init_genesis(provider_factory)?;
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
Ok(())

6
bin/reth/src/commands/p2p/mod.rs
View File

@ -131,7 +131,11 @@ impl Command {
network_config_builder = self.discovery.apply_to_builder(network_config_builder);
let network = network_config_builder
.build(Arc::new(ProviderFactory::new(noop_db, self.chain.clone())))
.build(Arc::new(ProviderFactory::new(
noop_db,
self.chain.clone(),
data_dir.static_files_path(),
)?))
.start_network()
.await?;

9
bin/reth/src/commands/recover/storage_tries.rs
View File

@ -50,10 +50,11 @@ impl Command {
fs::create_dir_all(&db_path)?;
let db = Arc::new(init_db(db_path, Default::default())?);
debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis(db.clone(), self.chain.clone())?;
let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?;
debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis(factory.clone())?;
let factory = ProviderFactory::new(&db, self.chain);
let mut provider = factory.provider_rw()?;
let best_block = provider.best_block_number()?;
let best_header = provider
@ -62,7 +63,7 @@ impl Command {
let mut deleted_tries = 0;
let tx_mut = provider.tx_mut();
let mut hashed_account_cursor = tx_mut.cursor_read::<tables::HashedAccount>()?;
let mut hashed_account_cursor = tx_mut.cursor_read::<tables::HashedAccounts>()?;
let mut storage_trie_cursor = tx_mut.cursor_dup_read::<tables::StoragesTrie>()?;
let mut entry = storage_trie_cursor.first()?;

117
bin/reth/src/commands/stage/drop.rs
View File

@ -10,12 +10,15 @@ use crate::{
};
use clap::Parser;
use reth_db::{
database::Database, mdbx::DatabaseArguments, open_db, tables, transaction::DbTxMut, DatabaseEnv,
database::Database, mdbx::DatabaseArguments, open_db, static_file::iter_static_files, tables,
transaction::DbTxMut, DatabaseEnv,
};
use reth_node_core::init::{insert_genesis_header, insert_genesis_state};
use reth_primitives::{fs, stage::StageId, ChainSpec};
use reth_primitives::{
fs, stage::StageId, static_file::find_fixed_range, ChainSpec, StaticFileSegment,
};
use reth_provider::ProviderFactory;
use std::sync::Arc;
use tracing::info;
/// `reth drop-stage` command
#[derive(Debug, Parser)]
@ -58,23 +61,59 @@ impl Command {
let db =
open_db(db_path.as_ref(), DatabaseArguments::default().log_level(self.db.log_level))?;
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?;
let static_file_provider = provider_factory.static_file_provider();
let tool = DbTool::new(&db, self.chain.clone())?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
tool.db.update(|tx| {
match &self.stage {
let static_file_segment = match self.stage {
StageEnum::Headers => Some(StaticFileSegment::Headers),
StageEnum::Bodies => Some(StaticFileSegment::Transactions),
StageEnum::Execution => Some(StaticFileSegment::Receipts),
_ => None,
};
// Delete static file segment data before inserting the genesis header below
if let Some(static_file_segment) = static_file_segment {
let static_file_provider = tool.provider_factory.static_file_provider();
let static_files = iter_static_files(static_file_provider.directory())?;
if let Some(segment_static_files) = static_files.get(&static_file_segment) {
for (block_range, _) in segment_static_files {
static_file_provider
.delete_jar(static_file_segment, find_fixed_range(block_range.start()))?;
}
}
}
tool.provider_factory.db_ref().update(|tx| {
match self.stage {
StageEnum::Headers => {
tx.clear::<tables::CanonicalHeaders>()?;
tx.clear::<tables::Headers>()?;
tx.clear::<tables::HeaderTerminalDifficulties>()?;
tx.clear::<tables::HeaderNumbers>()?;
tx.put::<tables::StageCheckpoints>(
StageId::Headers.to_string(),
Default::default(),
)?;
insert_genesis_header::<DatabaseEnv>(tx, static_file_provider, self.chain)?;
}
StageEnum::Bodies => {
tx.clear::<tables::BlockBodyIndices>()?;
tx.clear::<tables::Transactions>()?;
tx.clear::<tables::TransactionBlock>()?;
tx.clear::<tables::TransactionBlocks>()?;
tx.clear::<tables::BlockOmmers>()?;
tx.clear::<tables::BlockWithdrawals>()?;
tx.put::<tables::SyncStage>(StageId::Bodies.to_string(), Default::default())?;
insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
tx.put::<tables::StageCheckpoints>(
StageId::Bodies.to_string(),
Default::default(),
)?;
insert_genesis_header::<DatabaseEnv>(tx, static_file_provider, self.chain)?;
}
StageEnum::Senders => {
tx.clear::<tables::TxSenders>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::TransactionSenders>()?;
tx.put::<tables::StageCheckpoints>(
StageId::SenderRecovery.to_string(),
Default::default(),
)?;
@ -82,41 +121,41 @@ impl Command {
StageEnum::Execution => {
tx.clear::<tables::PlainAccountState>()?;
tx.clear::<tables::PlainStorageState>()?;
tx.clear::<tables::AccountChangeSet>()?;
tx.clear::<tables::StorageChangeSet>()?;
tx.clear::<tables::AccountChangeSets>()?;
tx.clear::<tables::StorageChangeSets>()?;
tx.clear::<tables::Bytecodes>()?;
tx.clear::<tables::Receipts>()?;
tx.put::<tables::SyncStage>(
tx.put::<tables::StageCheckpoints>(
StageId::Execution.to_string(),
Default::default(),
)?;
insert_genesis_state::<DatabaseEnv>(tx, self.chain.genesis())?;
}
StageEnum::AccountHashing => {
tx.clear::<tables::HashedAccount>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::HashedAccounts>()?;
tx.put::<tables::StageCheckpoints>(
StageId::AccountHashing.to_string(),
Default::default(),
)?;
}
StageEnum::StorageHashing => {
tx.clear::<tables::HashedStorage>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::HashedStorages>()?;
tx.put::<tables::StageCheckpoints>(
StageId::StorageHashing.to_string(),
Default::default(),
)?;
}
StageEnum::Hashing => {
// Clear hashed accounts
tx.clear::<tables::HashedAccount>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::HashedAccounts>()?;
tx.put::<tables::StageCheckpoints>(
StageId::AccountHashing.to_string(),
Default::default(),
)?;
// Clear hashed storages
tx.clear::<tables::HashedStorage>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::HashedStorages>()?;
tx.put::<tables::StageCheckpoints>(
StageId::StorageHashing.to_string(),
Default::default(),
)?;
@ -124,54 +163,42 @@ impl Command {
StageEnum::Merkle => {
tx.clear::<tables::AccountsTrie>()?;
tx.clear::<tables::StoragesTrie>()?;
tx.put::<tables::SyncStage>(
tx.put::<tables::StageCheckpoints>(
StageId::MerkleExecute.to_string(),
Default::default(),
)?;
tx.put::<tables::SyncStage>(
tx.put::<tables::StageCheckpoints>(
StageId::MerkleUnwind.to_string(),
Default::default(),
)?;
tx.delete::<tables::SyncStageProgress>(
tx.delete::<tables::StageCheckpointProgresses>(
StageId::MerkleExecute.to_string(),
None,
)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountHistory>()?;
tx.clear::<tables::StorageHistory>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
tx.put::<tables::StageCheckpoints>(
StageId::IndexAccountHistory.to_string(),
Default::default(),
)?;
tx.put::<tables::SyncStage>(
tx.put::<tables::StageCheckpoints>(
StageId::IndexStorageHistory.to_string(),
Default::default(),
)?;
}
StageEnum::TotalDifficulty => {
tx.clear::<tables::HeaderTD>()?;
tx.put::<tables::SyncStage>(
StageId::TotalDifficulty.to_string(),
Default::default(),
)?;
insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
}
StageEnum::TxLookup => {
tx.clear::<tables::TxHashNumber>()?;
tx.put::<tables::SyncStage>(
tx.clear::<tables::TransactionHashNumbers>()?;
tx.put::<tables::StageCheckpoints>(
StageId::TransactionLookup.to_string(),
Default::default(),
)?;
insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
}
_ => {
info!("Nothing to do for stage {:?}", self.stage);
return Ok(())
insert_genesis_header::<DatabaseEnv>(tx, static_file_provider, self.chain)?;
}
}
tx.put::<tables::SyncStage>(StageId::Finish.to_string(), Default::default())?;
tx.put::<tables::StageCheckpoints>(StageId::Finish.to_string(), Default::default())?;
Ok::<_, eyre::Error>(())
})??;

78
bin/reth/src/commands/stage/dump/execution.rs
View File

@ -5,29 +5,38 @@ use reth_db::{
cursor::DbCursorRO, database::Database, table::TableImporter, tables, transaction::DbTx,
DatabaseEnv,
};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::{stage::StageCheckpoint, ChainSpec};
use reth_provider::ProviderFactory;
use reth_primitives::stage::StageCheckpoint;
use reth_provider::{ChainSpecProvider, ProviderFactory};
use reth_revm::EvmProcessorFactory;
use reth_stages::{stages::ExecutionStage, Stage, UnwindInput};
use std::{path::PathBuf, sync::Arc};
use tracing::info;
pub(crate) async fn dump_execution_stage<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
to: u64,
output_db: &PathBuf,
output_datadir: ChainPath<DataDirPath>,
should_run: bool,
) -> Result<()> {
let (output_db, tip_block_number) = setup(from, to, output_db, db_tool)?;
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db_path(), db_tool)?;
import_tables_with_range(&output_db, db_tool, from, to)?;
unwind_and_copy(db_tool, from, tip_block_number, &output_db).await?;
if should_run {
dry_run(db_tool.chain.clone(), output_db, to, from).await?;
dry_run(
ProviderFactory::new(
output_db,
db_tool.chain.clone(),
output_datadir.static_files_path(),
)?,
to,
from,
)
.await?;
}
Ok(())
@ -36,30 +45,50 @@ pub(crate) async fn dump_execution_stage<DB: Database>(
/// Imports all the tables that can be copied over a range.
fn import_tables_with_range<DB: Database>(
output_db: &DatabaseEnv,
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
to: u64,
) -> eyre::Result<()> {
// We're not sharing the transaction in case the memory grows too much.
output_db.update(|tx| {
tx.import_table_with_range::<tables::CanonicalHeaders, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::CanonicalHeaders, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::HeaderTD, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::HeaderTerminalDifficulties, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::Headers, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::Headers, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::BlockBodyIndices, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::BlockBodyIndices, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::BlockOmmers, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::BlockOmmers, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
// Find range of transactions that need to be copied over
let (from_tx, to_tx) = db_tool.db.view(|read_tx| {
let (from_tx, to_tx) = db_tool.provider_factory.db_ref().view(|read_tx| {
let mut read_cursor = read_tx.cursor_read::<tables::BlockBodyIndices>()?;
let (_, from_block) =
read_cursor.seek(from)?.ok_or(eyre::eyre!("BlockBody {from} does not exist."))?;
@ -74,14 +103,18 @@ fn import_tables_with_range<DB: Database>(
output_db.update(|tx| {
tx.import_table_with_range::<tables::Transactions, _>(
&db_tool.db.tx()?,
&db_tool.provider_factory.db_ref().tx()?,
Some(from_tx),
to_tx,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::TxSenders, _>(&db_tool.db.tx()?, Some(from_tx), to_tx)
tx.import_table_with_range::<tables::TransactionSenders, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from_tx),
to_tx,
)
})??;
Ok(())
@ -91,13 +124,12 @@ fn import_tables_with_range<DB: Database>(
/// PlainAccountState safely. There might be some state dependency from an address
/// which hasn't been changed in the given range.
async fn unwind_and_copy<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
tip_block_number: u64,
output_db: &DatabaseEnv,
) -> eyre::Result<()> {
let factory = ProviderFactory::new(db_tool.db, db_tool.chain.clone());
let provider = factory.provider_rw()?;
let provider = db_tool.provider_factory.provider_rw()?;
let mut exec_stage = ExecutionStage::new_with_factory(EvmProcessorFactory::new(
db_tool.chain.clone(),
@ -125,22 +157,20 @@ async fn unwind_and_copy<DB: Database>(
/// Try to re-execute the stage without committing
async fn dry_run<DB: Database>(
chain: Arc<ChainSpec>,
output_db: DB,
output_provider_factory: ProviderFactory<DB>,
to: u64,
from: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Executing stage. [dry-run]");
let factory = ProviderFactory::new(&output_db, chain.clone());
let mut exec_stage = ExecutionStage::new_with_factory(EvmProcessorFactory::new(
chain.clone(),
output_provider_factory.chain_spec().clone(),
EthEvmConfig::default(),
));
let input =
reth_stages::ExecInput { target: Some(to), checkpoint: Some(StageCheckpoint::new(from)) };
exec_stage.execute(&factory.provider_rw()?, input)?;
exec_stage.execute(&output_provider_factory.provider_rw()?, input)?;
info!(target: "reth::cli", "Success");

38
bin/reth/src/commands/stage/dump/hashing_account.rs
View File

@ -2,30 +2,43 @@ use super::setup;
use crate::utils::DbTool;
use eyre::Result;
use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv};
use reth_primitives::{stage::StageCheckpoint, BlockNumber, ChainSpec};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::{stage::StageCheckpoint, BlockNumber};
use reth_provider::ProviderFactory;
use reth_stages::{stages::AccountHashingStage, Stage, UnwindInput};
use std::{path::PathBuf, sync::Arc};
use tracing::info;
pub(crate) async fn dump_hashing_account_stage<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: BlockNumber,
to: BlockNumber,
output_db: &PathBuf,
output_datadir: ChainPath<DataDirPath>,
should_run: bool,
) -> Result<()> {
let (output_db, tip_block_number) = setup(from, to, output_db, db_tool)?;
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db_path(), db_tool)?;
// Import relevant AccountChangeSets
output_db.update(|tx| {
tx.import_table_with_range::<tables::AccountChangeSet, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::AccountChangeSets, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
unwind_and_copy(db_tool, from, tip_block_number, &output_db)?;
if should_run {
dry_run(db_tool.chain.clone(), output_db, to, from).await?;
dry_run(
ProviderFactory::new(
output_db,
db_tool.chain.clone(),
output_datadir.static_files_path(),
)?,
to,
from,
)
.await?;
}
Ok(())
@ -33,13 +46,12 @@ pub(crate) async fn dump_hashing_account_stage<DB: Database>(
/// Dry-run an unwind to FROM block and copy the necessary table data to the new database.
fn unwind_and_copy<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
tip_block_number: u64,
output_db: &DatabaseEnv,
) -> eyre::Result<()> {
let factory = ProviderFactory::new(db_tool.db, db_tool.chain.clone());
let provider = factory.provider_rw()?;
let provider = db_tool.provider_factory.provider_rw()?;
let mut exec_stage = AccountHashingStage::default();
exec_stage.unwind(
@ -59,15 +71,13 @@ fn unwind_and_copy<DB: Database>(
/// Try to re-execute the stage straightaway
async fn dry_run<DB: Database>(
chain: Arc<ChainSpec>,
output_db: DB,
output_provider_factory: ProviderFactory<DB>,
to: u64,
from: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Executing stage.");
let factory = ProviderFactory::new(&output_db, chain);
let provider = factory.provider_rw()?;
let provider = output_provider_factory.provider_rw()?;
let mut stage = AccountHashingStage {
clean_threshold: 1, // Forces hashing from scratch
..Default::default()

35
bin/reth/src/commands/stage/dump/hashing_storage.rs
View File

@ -2,25 +2,34 @@ use super::setup;
use crate::utils::DbTool;
use eyre::Result;
use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv};
use reth_primitives::{stage::StageCheckpoint, ChainSpec};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::stage::StageCheckpoint;
use reth_provider::ProviderFactory;
use reth_stages::{stages::StorageHashingStage, Stage, UnwindInput};
use std::{path::PathBuf, sync::Arc};
use tracing::info;
pub(crate) async fn dump_hashing_storage_stage<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
to: u64,
output_db: &PathBuf,
output_datadir: ChainPath<DataDirPath>,
should_run: bool,
) -> Result<()> {
let (output_db, tip_block_number) = setup(from, to, output_db, db_tool)?;
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db_path(), db_tool)?;
unwind_and_copy(db_tool, from, tip_block_number, &output_db)?;
if should_run {
dry_run(db_tool.chain.clone(), output_db, to, from).await?;
dry_run(
ProviderFactory::new(
output_db,
db_tool.chain.clone(),
output_datadir.static_files_path(),
)?,
to,
from,
)
.await?;
}
Ok(())
@ -28,13 +37,12 @@ pub(crate) async fn dump_hashing_storage_stage<DB: Database>(
/// Dry-run an unwind to FROM block and copy the necessary table data to the new database.
fn unwind_and_copy<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: u64,
tip_block_number: u64,
output_db: &DatabaseEnv,
) -> eyre::Result<()> {
let factory = ProviderFactory::new(db_tool.db, db_tool.chain.clone());
let provider = factory.provider_rw()?;
let provider = db_tool.provider_factory.provider_rw()?;
let mut exec_stage = StorageHashingStage::default();
@ -51,22 +59,21 @@ fn unwind_and_copy<DB: Database>(
// TODO optimize we can actually just get the entries we need for both these tables
output_db
.update(|tx| tx.import_dupsort::<tables::PlainStorageState, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_dupsort::<tables::StorageChangeSet, _>(&unwind_inner_tx))??;
output_db
.update(|tx| tx.import_dupsort::<tables::StorageChangeSets, _>(&unwind_inner_tx))??;
Ok(())
}
/// Try to re-execute the stage straightaway
async fn dry_run<DB: Database>(
chain: Arc<ChainSpec>,
output_db: DB,
output_provider_factory: ProviderFactory<DB>,
to: u64,
from: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Executing stage.");
let factory = ProviderFactory::new(&output_db, chain);
let provider = factory.provider_rw()?;
let provider = output_provider_factory.provider_rw()?;
let mut stage = StorageHashingStage {
clean_threshold: 1, // Forces hashing from scratch
..Default::default()

51
bin/reth/src/commands/stage/dump/merkle.rs
View File

@ -2,8 +2,9 @@ use super::setup;
use crate::utils::DbTool;
use eyre::Result;
use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::{stage::StageCheckpoint, BlockNumber, ChainSpec, PruneModes};
use reth_primitives::{stage::StageCheckpoint, BlockNumber, PruneModes};
use reth_provider::ProviderFactory;
use reth_stages::{
stages::{
@ -12,30 +13,46 @@ use reth_stages::{
},
Stage, UnwindInput,
};
use std::{path::PathBuf, sync::Arc};
use tracing::info;
pub(crate) async fn dump_merkle_stage<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
from: BlockNumber,
to: BlockNumber,
output_db: &PathBuf,
output_datadir: ChainPath<DataDirPath>,
should_run: bool,
) -> Result<()> {
let (output_db, tip_block_number) = setup(from, to, output_db, db_tool)?;
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db_path(), db_tool)?;
output_db.update(|tx| {
tx.import_table_with_range::<tables::Headers, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::Headers, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::AccountChangeSet, _>(&db_tool.db.tx()?, Some(from), to)
tx.import_table_with_range::<tables::AccountChangeSets, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
unwind_and_copy(db_tool, (from, to), tip_block_number, &output_db).await?;
if should_run {
dry_run(db_tool.chain.clone(), output_db, to, from).await?;
dry_run(
ProviderFactory::new(
output_db,
db_tool.chain.clone(),
output_datadir.static_files_path(),
)?,
to,
from,
)
.await?;
}
Ok(())
@ -43,14 +60,13 @@ pub(crate) async fn dump_merkle_stage<DB: Database>(
/// Dry-run an unwind to FROM block and copy the necessary table data to the new database.
async fn unwind_and_copy<DB: Database>(
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
range: (u64, u64),
tip_block_number: u64,
output_db: &DatabaseEnv,
) -> eyre::Result<()> {
let (from, to) = range;
let factory = ProviderFactory::new(db_tool.db, db_tool.chain.clone());
let provider = factory.provider_rw()?;
let provider = db_tool.provider_factory.provider_rw()?;
let unwind = UnwindInput {
unwind_to: from,
@ -100,10 +116,11 @@ async fn unwind_and_copy<DB: Database>(
let unwind_inner_tx = provider.into_tx();
// TODO optimize we can actually just get the entries we need
output_db.update(|tx| tx.import_dupsort::<tables::StorageChangeSet, _>(&unwind_inner_tx))??;
output_db
.update(|tx| tx.import_dupsort::<tables::StorageChangeSets, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_table::<tables::HashedAccount, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_dupsort::<tables::HashedStorage, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_table::<tables::HashedAccounts, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_dupsort::<tables::HashedStorages, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_table::<tables::AccountsTrie, _>(&unwind_inner_tx))??;
output_db.update(|tx| tx.import_dupsort::<tables::StoragesTrie, _>(&unwind_inner_tx))??;
@ -112,14 +129,12 @@ async fn unwind_and_copy<DB: Database>(
/// Try to re-execute the stage straightaway
async fn dry_run<DB: Database>(
chain: Arc<ChainSpec>,
output_db: DB,
output_provider_factory: ProviderFactory<DB>,
to: u64,
from: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Executing stage.");
let factory = ProviderFactory::new(&output_db, chain);
let provider = factory.provider_rw()?;
let provider = output_provider_factory.provider_rw()?;
let mut stage = MerkleStage::Execution {
// Forces updating the root instead of calculating from scratch

72
bin/reth/src/commands/stage/dump/mod.rs
View File

@ -14,7 +14,9 @@ use reth_db::{
cursor::DbCursorRO, database::Database, init_db, table::TableImporter, tables,
transaction::DbTx, DatabaseEnv,
};
use reth_node_core::dirs::PlatformPath;
use reth_primitives::ChainSpec;
use reth_provider::ProviderFactory;
use std::{path::PathBuf, sync::Arc};
use tracing::info;
@ -79,9 +81,9 @@ pub enum Stages {
/// Stage command that takes a range
#[derive(Debug, Clone, Parser)]
pub struct StageCommand {
/// The path to the new database folder.
/// The path to the new datadir folder.
#[arg(long, value_name = "OUTPUT_PATH", verbatim_doc_comment)]
output_db: PathBuf,
output_datadir: PlatformPath<DataDirPath>,
/// From which block.
#[arg(long, short)]
@ -104,22 +106,53 @@ impl Command {
info!(target: "reth::cli", path = ?db_path, "Opening database");
let db =
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
let provider_factory =
ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?;
info!(target: "reth::cli", "Database opened");
let tool = DbTool::new(&db, self.chain.clone())?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
match &self.command {
Stages::Execution(StageCommand { output_db, from, to, dry_run, .. }) => {
dump_execution_stage(&tool, *from, *to, output_db, *dry_run).await?
Stages::Execution(StageCommand { output_datadir, from, to, dry_run, .. }) => {
dump_execution_stage(
&tool,
*from,
*to,
output_datadir.with_chain(self.chain.chain),
*dry_run,
)
.await?
}
Stages::StorageHashing(StageCommand { output_db, from, to, dry_run, .. }) => {
dump_hashing_storage_stage(&tool, *from, *to, output_db, *dry_run).await?
Stages::StorageHashing(StageCommand { output_datadir, from, to, dry_run, .. }) => {
dump_hashing_storage_stage(
&tool,
*from,
*to,
output_datadir.with_chain(self.chain.chain),
*dry_run,
)
.await?
}
Stages::AccountHashing(StageCommand { output_db, from, to, dry_run, .. }) => {
dump_hashing_account_stage(&tool, *from, *to, output_db, *dry_run).await?
Stages::AccountHashing(StageCommand { output_datadir, from, to, dry_run, .. }) => {
dump_hashing_account_stage(
&tool,
*from,
*to,
output_datadir.with_chain(self.chain.chain),
*dry_run,
)
.await?
}
Stages::Merkle(StageCommand { output_db, from, to, dry_run, .. }) => {
dump_merkle_stage(&tool, *from, *to, output_db, *dry_run).await?
Stages::Merkle(StageCommand { output_datadir, from, to, dry_run, .. }) => {
dump_merkle_stage(
&tool,
*from,
*to,
output_datadir.with_chain(self.chain.chain),
*dry_run,
)
.await?
}
}
@ -133,24 +166,27 @@ pub(crate) fn setup<DB: Database>(
from: u64,
to: u64,
output_db: &PathBuf,
db_tool: &DbTool<'_, DB>,
db_tool: &DbTool<DB>,
) -> eyre::Result<(DatabaseEnv, u64)> {
assert!(from < to, "FROM block should be bigger than TO block.");
info!(target: "reth::cli", ?output_db, "Creating separate db");
let output_db = init_db(output_db, Default::default())?;
let output_datadir = init_db(output_db, Default::default())?;
output_db.update(|tx| {
output_datadir.update(|tx| {
tx.import_table_with_range::<tables::BlockBodyIndices, _>(
&db_tool.db.tx()?,
&db_tool.provider_factory.db_ref().tx()?,
Some(from - 1),
to + 1,
)
})??;
let (tip_block_number, _) =
db_tool.db.view(|tx| tx.cursor_read::<tables::BlockBodyIndices>()?.last())??.expect("some");
let (tip_block_number, _) = db_tool
.provider_factory
.db_ref()
.view(|tx| tx.cursor_read::<tables::BlockBodyIndices>()?.last())??
.expect("some");
Ok((output_db, tip_block_number))
Ok((output_datadir, tip_block_number))
}

43
bin/reth/src/commands/stage/run.rs
View File

@ -19,16 +19,16 @@ use reth_db::{init_db, mdbx::DatabaseArguments};
use reth_downloaders::bodies::bodies::BodiesDownloaderBuilder;
use reth_node_ethereum::EthEvmConfig;
use reth_primitives::ChainSpec;
use reth_provider::{ProviderFactory, StageCheckpointReader};
use reth_provider::{ProviderFactory, StageCheckpointReader, StageCheckpointWriter};
use reth_stages::{
stages::{
AccountHashingStage, BodyStage, ExecutionStage, ExecutionStageThresholds,
IndexAccountHistoryStage, IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage,
StorageHashingStage, TransactionLookupStage,
},
ExecInput, Stage, StageExt, UnwindInput,
ExecInput, ExecOutput, Stage, StageExt, UnwindInput, UnwindOutput,
};
use std::{any::Any, net::SocketAddr, path::PathBuf, sync::Arc};
use std::{any::Any, net::SocketAddr, path::PathBuf, sync::Arc, time::Instant};
use tracing::*;
/// `reth stage` command
@ -103,6 +103,10 @@ pub struct Command {
// e.g. query the DB size, or any table data.
#[arg(long, short)]
commit: bool,
/// Save stage checkpoints
#[arg(long)]
checkpoints: bool,
}
impl Command {
@ -127,7 +131,11 @@ impl Command {
Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?);
info!(target: "reth::cli", "Database opened");
let factory = ProviderFactory::new(Arc::clone(&db), self.chain.clone());
let factory = ProviderFactory::new(
Arc::clone(&db),
self.chain.clone(),
data_dir.static_files_path(),
)?;
let mut provider_rw = factory.provider_rw()?;
if let Some(listen_addr) = self.metrics {
@ -165,8 +173,11 @@ impl Command {
let default_peers_path = data_dir.known_peers_path();
let provider_factory =
Arc::new(ProviderFactory::new(db.clone(), self.chain.clone()));
let provider_factory = Arc::new(ProviderFactory::new(
db.clone(),
self.chain.clone(),
data_dir.static_files_path(),
)?);
let network = self
.network
@ -250,8 +261,12 @@ impl Command {
if !self.skip_unwind {
while unwind.checkpoint.block_number > self.from {
let unwind_output = unwind_stage.unwind(&provider_rw, unwind)?;
unwind.checkpoint = unwind_output.checkpoint;
let UnwindOutput { checkpoint } = unwind_stage.unwind(&provider_rw, unwind)?;
unwind.checkpoint = checkpoint;
if self.checkpoints {
provider_rw.save_stage_checkpoint(unwind_stage.id(), checkpoint)?;
}
if self.commit {
provider_rw.commit()?;
@ -265,21 +280,27 @@ impl Command {
checkpoint: Some(checkpoint.with_block_number(self.from)),
};
let start = Instant::now();
info!(target: "reth::cli", stage = %self.stage, "Executing stage");
loop {
exec_stage.execute_ready(input).await?;
let output = exec_stage.execute(&provider_rw, input)?;
let ExecOutput { checkpoint, done } = exec_stage.execute(&provider_rw, input)?;
input.checkpoint = Some(output.checkpoint);
input.checkpoint = Some(checkpoint);
if self.checkpoints {
provider_rw.save_stage_checkpoint(exec_stage.id(), checkpoint)?;
}
if self.commit {
provider_rw.commit()?;
provider_rw = factory.provider_rw()?;
}
if output.done {
if done {
break
}
}
info!(target: "reth::cli", stage = %self.stage, time = ?start.elapsed(), "Finished stage");
Ok(())
}

2
bin/reth/src/commands/stage/unwind.rs
View File

@ -68,7 +68,7 @@ impl Command {
eyre::bail!("Cannot unwind genesis block")
}
let factory = ProviderFactory::new(&db, self.chain.clone());
let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?;
let provider = factory.provider_rw()?;
let blocks_and_execution = provider

4
bin/reth/src/commands/test_vectors/tables.rs
View File

@ -58,12 +58,12 @@ pub(crate) fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
generate!([
(CanonicalHeaders, PER_TABLE, TABLE),
(HeaderTD, PER_TABLE, TABLE),
(HeaderTerminalDifficulties, PER_TABLE, TABLE),
(HeaderNumbers, PER_TABLE, TABLE),
(Headers, PER_TABLE, TABLE),
(BlockBodyIndices, PER_TABLE, TABLE),
(BlockOmmers, 100, TABLE),
(TxHashNumber, PER_TABLE, TABLE),
(TransactionHashNumbers, PER_TABLE, TABLE),
(Transactions, 100, TABLE),
(PlainStorageState, PER_TABLE, DUPSORT),
(PlainAccountState, PER_TABLE, TABLE)

42
bin/reth/src/utils.rs
View File

@ -10,6 +10,7 @@ use reth_db::{
DatabaseError, RawTable, TableRawRow,
};
use reth_primitives::{fs, ChainSpec};
use reth_provider::ProviderFactory;
use std::{path::Path, rc::Rc, sync::Arc};
use tracing::info;
@ -24,17 +25,17 @@ pub use reth_node_core::utils::*;
/// Wrapper over DB that implements many useful DB queries.
#[derive(Debug)]
pub struct DbTool<'a, DB: Database> {
/// The database that the db tool will use.
pub db: &'a DB,
pub struct DbTool<DB: Database> {
/// The provider factory that the db tool will use.
pub provider_factory: ProviderFactory<DB>,
/// The [ChainSpec] that the db tool will use.
pub chain: Arc<ChainSpec>,
}
impl<'a, DB: Database> DbTool<'a, DB> {
impl<DB: Database> DbTool<DB> {
/// Takes a DB where the tables have already been created.
pub fn new(db: &'a DB, chain: Arc<ChainSpec>) -> eyre::Result<Self> {
Ok(Self { db, chain })
pub fn new(provider_factory: ProviderFactory<DB>, chain: Arc<ChainSpec>) -> eyre::Result<Self> {
Ok(Self { provider_factory, chain })
}
/// Grabs the contents of the table within a certain index range and places the
@ -50,7 +51,7 @@ impl<'a, DB: Database> DbTool<'a, DB> {
let mut hits = 0;
let data = self.db.view(|tx| {
let data = self.provider_factory.db_ref().view(|tx| {
let mut cursor =
tx.cursor_read::<RawTable<T>>().expect("Was not able to obtain a cursor.");
@ -118,27 +119,38 @@ impl<'a, DB: Database> DbTool<'a, DB> {
/// Grabs the content of the table for the given key
pub fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>> {
self.db.view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
self.provider_factory.db_ref().view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
}
/// Grabs the content of the DupSort table for the given key and subkey
pub fn get_dup<T: DupSort>(&self, key: T::Key, subkey: T::SubKey) -> Result<Option<T::Value>> {
self.db
self.provider_factory
.db_ref()
.view(|tx| tx.cursor_dup_read::<T>()?.seek_by_key_subkey(key, subkey))?
.map_err(|e| eyre::eyre!(e))
}
/// Drops the database at the given path.
pub fn drop(&mut self, path: impl AsRef<Path>) -> Result<()> {
let path = path.as_ref();
info!(target: "reth::cli", "Dropping database at {:?}", path);
fs::remove_dir_all(path)?;
/// Drops the database and the static files at the given path.
pub fn drop(
&mut self,
db_path: impl AsRef<Path>,
static_files_path: impl AsRef<Path>,
) -> Result<()> {
let db_path = db_path.as_ref();
info!(target: "reth::cli", "Dropping database at {:?}", db_path);
fs::remove_dir_all(db_path)?;
let static_files_path = static_files_path.as_ref();
info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path);
fs::remove_dir_all(static_files_path)?;
fs::create_dir_all(static_files_path)?;
Ok(())
}
/// Drops the provided table from the database.
pub fn drop_table<T: Table>(&mut self) -> Result<()> {
self.db.update(|tx| tx.clear::<T>())??;
self.provider_factory.db_ref().update(|tx| tx.clear::<T>())??;
Ok(())
}
}

6
book/SUMMARY.md
View File

@ -37,9 +37,13 @@
- [`reth db list`](./cli/reth/db/list.md)
- [`reth db diff`](./cli/reth/db/diff.md)
- [`reth db get`](./cli/reth/db/get.md)
- [`reth db get mdbx`](./cli/reth/db/get/mdbx.md)
- [`reth db get static-file`](./cli/reth/db/get/static-file.md)
- [`reth db drop`](./cli/reth/db/drop.md)
- [`reth db clear`](./cli/reth/db/clear.md)
- [`reth db snapshot`](./cli/reth/db/snapshot.md)
- [`reth db clear mdbx`](./cli/reth/db/clear/mdbx.md)
- [`reth db clear static-file`](./cli/reth/db/clear/static-file.md)
- [`reth db create-static-files`](./cli/reth/db/create-static-files.md)
- [`reth db version`](./cli/reth/db/version.md)
- [`reth db path`](./cli/reth/db/path.md)
- [`reth stage`](./cli/reth/stage.md)

6
book/cli/SUMMARY.md vendored
View File

@ -8,9 +8,13 @@
- [`reth db list`](./reth/db/list.md)
- [`reth db diff`](./reth/db/diff.md)
- [`reth db get`](./reth/db/get.md)
- [`reth db get mdbx`](./reth/db/get/mdbx.md)
- [`reth db get static-file`](./reth/db/get/static-file.md)
- [`reth db drop`](./reth/db/drop.md)
- [`reth db clear`](./reth/db/clear.md)
- [`reth db snapshot`](./reth/db/snapshot.md)
- [`reth db clear mdbx`](./reth/db/clear/mdbx.md)
- [`reth db clear static-file`](./reth/db/clear/static-file.md)
- [`reth db create-static-files`](./reth/db/create-static-files.md)
- [`reth db version`](./reth/db/version.md)
- [`reth db path`](./reth/db/path.md)
- [`reth stage`](./reth/stage.md)

20
book/cli/reth/db.md vendored
View File

@ -7,16 +7,16 @@ $ reth db --help
Usage: reth db [OPTIONS] <COMMAND>
Commands:
stats Lists all the tables, their entry count and their size
list Lists the contents of a table
diff Create a diff between two database tables or two entire databases
get Gets the content of a table for the given key
drop Deletes all database entries
clear Deletes all table entries
snapshot Snapshots tables from database
version Lists current and local database versions
path Returns the full database path
help Print this message or the help of the given subcommand(s)
stats Lists all the tables, their entry count and their size
list Lists the contents of a table
diff Create a diff between two database tables or two entire databases
get Gets the content of a table for the given key
drop Deletes all database entries
clear Deletes all table entries
create-static-files Creates static files from database tables
version Lists current and local database versions
path Returns the full database path
help Print this message or the help of the given subcommand(s)
Options:
--datadir <DATA_DIR>

9
book/cli/reth/db/clear.md vendored
View File

@ -4,11 +4,12 @@ Deletes all table entries
```bash
$ reth db clear --help
Usage: reth db clear [OPTIONS] <TABLE>
Usage: reth db clear [OPTIONS] <COMMAND>
Arguments:
<TABLE>
Table name
Commands:
mdbx Deletes all database table entries
static-file Deletes all static file segment entries
help Print this message or the help of the given subcommand(s)
Options:
--datadir <DATA_DIR>

124
book/cli/reth/db/clear/mdbx.md vendored Normal file
View File

@ -0,0 +1,124 @@
# reth db clear mdbx
Deletes all database table entries
```bash
$ reth db clear mdbx --help
Usage: reth db clear mdbx [OPTIONS] <TABLE>
Arguments:
<TABLE>
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

127
book/cli/reth/db/clear/static-file.md vendored Normal file
View File

@ -0,0 +1,127 @@
# reth db clear static-file
Deletes all static file segment entries
```bash
$ reth db clear static-file --help
Usage: reth db clear static-file [OPTIONS] <SEGMENT>
Arguments:
<SEGMENT>
Possible values:
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: Static File segment responsible for the `Transactions` table
- receipts: Static File segment responsible for the `Receipts` table
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

127
book/cli/reth/db/clear/static_file.md vendored Normal file
View File

@ -0,0 +1,127 @@
# reth db clear static-file
Deletes all static_file segment entries
```bash
$ reth db clear static-file --help
Usage: reth db clear static-file [OPTIONS] <SEGMENT>
Arguments:
<SEGMENT>
Possible values:
- headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: StaticFile segment responsible for the `Transactions` table
- receipts: StaticFile segment responsible for the `Receipts` table
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

30
book/cli/reth/db/snapshot.md → book/cli/reth/db/create-static-files.md vendored
View File

@ -1,19 +1,19 @@
# reth db snapshot
# reth db create-static-files
Snapshots tables from database
Creates static files from database tables
```bash
$ reth db snapshot --help
Usage: reth db snapshot [OPTIONS] [SEGMENTS]...
$ reth db create-static-files --help
Usage: reth db create-static-files [OPTIONS] [SEGMENTS]...
Arguments:
[SEGMENTS]...
Snapshot segments to generate
Static File segments to generate
Possible values:
- headers: Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables
- transactions: Snapshot segment responsible for the `Transactions` table
- receipts: Snapshot segment responsible for the `Receipts` table
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: Static File segment responsible for the `Transactions` table
- receipts: Static File segment responsible for the `Receipts` table
Options:
--datadir <DATA_DIR>
@ -28,12 +28,12 @@ Options:
[default: default]
-f, --from <FROM>
Starting block for the snapshot
Starting block for the static file
[default: 0]
-b, --block-interval <BLOCK_INTERVAL>
Number of blocks in the snapshot
Number of blocks in the static file
[default: 500000]
@ -47,18 +47,18 @@ Options:
[default: mainnet]
-p, --parallel <PARALLEL>
Sets the number of snapshots built in parallel. Note: Each parallel build is memory-intensive
Sets the number of static files built in parallel. Note: Each parallel build is memory-intensive
[default: 1]
--only-stats
Flag to skip snapshot creation and print snapshot files stats
Flag to skip static file creation and print static files stats
--bench
Flag to enable database-to-snapshot benchmarking
Flag to enable database-to-static file benchmarking
--only-bench
Flag to skip snapshot creation and only run benchmarks on existing snapshots
Flag to skip static file creation and only run benchmarks on existing static files
-c, --compression <COMPRESSION>
Compression algorithms to use
@ -69,7 +69,7 @@ Options:
- lz4: LZ4 compression algorithm
- zstd: Zstandard (Zstd) compression algorithm
- zstd-with-dictionary: Zstandard (Zstd) compression algorithm with a dictionary
- uncompressed: No compression, uncompressed snapshot
- uncompressed: No compression
--with-filters
Flag to enable inclusion list filters and PHFs

20
book/cli/reth/db/get.md vendored
View File

@ -4,19 +4,12 @@ Gets the content of a table for the given key
```bash
$ reth db get --help
Usage: reth db get [OPTIONS] <TABLE> <KEY> [SUBKEY]
Usage: reth db get [OPTIONS] <COMMAND>
Arguments:
<TABLE>
The table name
NOTE: The dupsort tables are not supported now.
<KEY>
The key to get content for
[SUBKEY]
The subkey to get content for
Commands:
mdbx Gets the content of a database table for the given key
static-file Gets the content of a static file segment for the given key
help Print this message or the help of the given subcommand(s)
Options:
--datadir <DATA_DIR>
@ -30,9 +23,6 @@ Options:
[default: default]
--raw
Output bytes instead of human-readable decoded value
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.

133
book/cli/reth/db/get/mdbx.md vendored Normal file
View File

@ -0,0 +1,133 @@
# reth db get mdbx
Gets the content of a database table for the given key
```bash
$ reth db get mdbx --help
Usage: reth db get mdbx [OPTIONS] <TABLE> <KEY> [SUBKEY]
Arguments:
<TABLE>
<KEY>
The key to get content for
[SUBKEY]
The subkey to get content for
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--raw
Output bytes instead of human-readable decoded value
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

133
book/cli/reth/db/get/static-file.md vendored Normal file
View File

@ -0,0 +1,133 @@
# reth db get static-file
Gets the content of a static file segment for the given key
```bash
$ reth db get static-file --help
Usage: reth db get static-file [OPTIONS] <SEGMENT> <KEY>
Arguments:
<SEGMENT>
Possible values:
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: Static File segment responsible for the `Transactions` table
- receipts: Static File segment responsible for the `Receipts` table
<KEY>
The key to get content for
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--raw
Output bytes instead of human-readable decoded value
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

133
book/cli/reth/db/get/static_file.md vendored Normal file
View File

@ -0,0 +1,133 @@
# reth db get static-file
Gets the content of a static_file segment for the given key
```bash
$ reth db get static-file --help
Usage: reth db get static-file [OPTIONS] <SEGMENT> <KEY>
Arguments:
<SEGMENT>
Possible values:
- headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: StaticFile segment responsible for the `Transactions` table
- receipts: StaticFile segment responsible for the `Receipts` table
<KEY>
The key to get content for
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
--raw
Output bytes instead of human-readable decoded value
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

174
book/cli/reth/db/static_file.md vendored Normal file
View File

@ -0,0 +1,174 @@
# reth db static-file
StaticFiles tables from database
```bash
$ reth db static-file --help
Usage: reth db static-file [OPTIONS] [SEGMENTS]...
Arguments:
[SEGMENTS]...
StaticFile segments to generate
Possible values:
- headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: StaticFile segment responsible for the `Transactions` table
- receipts: StaticFile segment responsible for the `Receipts` table
Options:
--datadir <DATA_DIR>
The path to the data dir for all reth files and subdirectories.
Defaults to the OS-specific data directory:
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
- Windows: `{FOLDERID_RoamingAppData}/reth/`
- macOS: `$HOME/Library/Application Support/reth/`
[default: default]
-f, --from <FROM>
Starting block for the static_file
[default: 0]
-b, --block-interval <BLOCK_INTERVAL>
Number of blocks in the static_file
[default: 500000]
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, goerli, holesky, dev
[default: mainnet]
-p, --parallel <PARALLEL>
Sets the number of static files built in parallel. Note: Each parallel build is memory-intensive
[default: 1]
--only-stats
Flag to skip static_file creation and print static_file files stats
--bench
Flag to enable database-to-static_file benchmarking
--only-bench
Flag to skip static_file creation and only run benchmarks on existing static files
-c, --compression <COMPRESSION>
Compression algorithms to use
[default: uncompressed]
Possible values:
- lz4: LZ4 compression algorithm
- zstd: Zstandard (Zstd) compression algorithm
- zstd-with-dictionary: Zstandard (Zstd) compression algorithm with a dictionary
- uncompressed: No compression, uncompressed static_file
--with-filters
Flag to enable inclusion list filters and PHFs
--phf <PHF>
Specifies the perfect hashing function to use
Possible values:
- fmph: Fingerprint-Based Minimal Perfect Hash Function
- go-fmph: Fingerprint-Based Minimal Perfect Hash Function with Group Optimization
--instance <INSTANCE>
Add a new instance of a node.
Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine.
Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other.
Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2
[default: 1]
-h, --help
Print help (see a summary with '-h')
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ]
--log.file.format <FORMAT>
The format to use for logs written to the log file
[default: terminal]
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
[default: 5]
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
[default: always]
Possible values:
- always: Colors on
- auto: Colors on
- never: Colors off
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
```

6
book/cli/reth/db/stats.md vendored
View File

@ -18,6 +18,9 @@ Options:
[default: default]
--only-total-size
Show only the total size for static files
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
@ -27,6 +30,9 @@ Options:
[default: mainnet]
--summary
Show only the summary per static file segment
--instance <INSTANCE>
Add a new instance of a node.

7
book/cli/reth/node.md vendored
View File

@ -131,14 +131,14 @@ Networking:
--pooled-tx-response-soft-limit <BYTES>
Soft limit for the byte size of a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on assembling a [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request. Spec'd at 2 MiB.
<https://github.com/ethereum/devp2p/blob/master/caps/eth.md#protocol-messages>.
[default: 2097152]
--pooled-tx-pack-soft-limit <BYTES>
Default soft limit for the byte size of a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on assembling a [`GetPooledTransactions`](reth_eth_wire::PooledTransactions) request. This defaults to less than the [`SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE`], at 2 MiB, used when assembling a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response. Default is 128 KiB
[default: 131072]
RPC:
@ -533,4 +533,3 @@ Display:
-q, --quiet
Silence all log output
```

23
book/cli/reth/stage/drop.md vendored
View File

@ -57,18 +57,17 @@ Database:
<STAGE>
Possible values:
- headers: The headers stage within the pipeline
- bodies: The bodies stage within the pipeline
- senders: The senders stage within the pipeline
- execution: The execution stage within the pipeline
- account-hashing: The account hashing stage within the pipeline
- storage-hashing: The storage hashing stage within the pipeline
- hashing: The hashing stage within the pipeline
- merkle: The Merkle stage within the pipeline
- tx-lookup: The transaction lookup stage within the pipeline
- account-history: The account history stage within the pipeline
- storage-history: The storage history stage within the pipeline
- total-difficulty: The total difficulty stage within the pipeline
- headers: The headers stage within the pipeline
- bodies: The bodies stage within the pipeline
- senders: The senders stage within the pipeline
- execution: The execution stage within the pipeline
- account-hashing: The account hashing stage within the pipeline
- storage-hashing: The storage hashing stage within the pipeline
- hashing: The hashing stage within the pipeline
- merkle: The Merkle stage within the pipeline
- tx-lookup: The transaction lookup stage within the pipeline
- account-history: The account history stage within the pipeline
- storage-history: The storage history stage within the pipeline
Logging:
--log.stdout.format <FORMAT>

6
book/cli/reth/stage/dump/account-hashing.md vendored
View File

@ -4,11 +4,11 @@ AccountHashing stage
```bash
$ reth stage dump account-hashing --help
Usage: reth stage dump account-hashing [OPTIONS] --output-db <OUTPUT_PATH> --from <FROM> --to <TO>
Usage: reth stage dump account-hashing [OPTIONS] --output-datadir <OUTPUT_PATH> --from <FROM> --to <TO>
Options:
--output-db <OUTPUT_PATH>
The path to the new database folder.
--output-datadir <OUTPUT_PATH>
The path to the new datadir folder.
-f, --from <FROM>
From which block

6
book/cli/reth/stage/dump/execution.md vendored
View File

@ -4,11 +4,11 @@ Execution stage
```bash
$ reth stage dump execution --help
Usage: reth stage dump execution [OPTIONS] --output-db <OUTPUT_PATH> --from <FROM> --to <TO>
Usage: reth stage dump execution [OPTIONS] --output-datadir <OUTPUT_PATH> --from <FROM> --to <TO>
Options:
--output-db <OUTPUT_PATH>
The path to the new database folder.
--output-datadir <OUTPUT_PATH>
The path to the new datadir folder.
-f, --from <FROM>
From which block

6
book/cli/reth/stage/dump/merkle.md vendored
View File

@ -4,11 +4,11 @@ Merkle stage
```bash
$ reth stage dump merkle --help
Usage: reth stage dump merkle [OPTIONS] --output-db <OUTPUT_PATH> --from <FROM> --to <TO>
Usage: reth stage dump merkle [OPTIONS] --output-datadir <OUTPUT_PATH> --from <FROM> --to <TO>
Options:
--output-db <OUTPUT_PATH>
The path to the new database folder.
--output-datadir <OUTPUT_PATH>
The path to the new datadir folder.
-f, --from <FROM>
From which block

6
book/cli/reth/stage/dump/storage-hashing.md vendored
View File

@ -4,11 +4,11 @@ StorageHashing stage
```bash
$ reth stage dump storage-hashing --help
Usage: reth stage dump storage-hashing [OPTIONS] --output-db <OUTPUT_PATH> --from <FROM> --to <TO>
Usage: reth stage dump storage-hashing [OPTIONS] --output-datadir <OUTPUT_PATH> --from <FROM> --to <TO>
Options:
--output-db <OUTPUT_PATH>
The path to the new database folder.
--output-datadir <OUTPUT_PATH>
The path to the new datadir folder.
-f, --from <FROM>
From which block

35
book/cli/reth/stage/run.md vendored
View File

@ -11,18 +11,17 @@ Arguments:
The name of the stage to run
Possible values:
- headers: The headers stage within the pipeline
- bodies: The bodies stage within the pipeline
- senders: The senders stage within the pipeline
- execution: The execution stage within the pipeline
- account-hashing: The account hashing stage within the pipeline
- storage-hashing: The storage hashing stage within the pipeline
- hashing: The hashing stage within the pipeline
- merkle: The Merkle stage within the pipeline
- tx-lookup: The transaction lookup stage within the pipeline
- account-history: The account history stage within the pipeline
- storage-history: The storage history stage within the pipeline
- total-difficulty: The total difficulty stage within the pipeline
- headers: The headers stage within the pipeline
- bodies: The bodies stage within the pipeline
- senders: The senders stage within the pipeline
- execution: The execution stage within the pipeline
- account-hashing: The account hashing stage within the pipeline
- storage-hashing: The storage hashing stage within the pipeline
- hashing: The hashing stage within the pipeline
- merkle: The Merkle stage within the pipeline
- tx-lookup: The transaction lookup stage within the pipeline
- account-history: The account history stage within the pipeline
- storage-history: The storage history stage within the pipeline
Options:
--config <FILE>
@ -152,6 +151,18 @@ Networking:
--max-inbound-peers <MAX_INBOUND_PEERS>
Maximum number of inbound requests. default: 30
--pooled-tx-response-soft-limit <BYTES>
Soft limit for the byte size of a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on assembling a [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request. Spec'd at 2 MiB.
<https://github.com/ethereum/devp2p/blob/master/caps/eth.md#protocol-messages>.
[default: 2097152]
--pooled-tx-pack-soft-limit <BYTES>
Default soft limit for the byte size of a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response on assembling a [`GetPooledTransactions`](reth_eth_wire::PooledTransactions) request. This defaults to less than the [`SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE`], at 2 MiB, used when assembling a [`PooledTransactions`](reth_eth_wire::PooledTransactions) response. Default is 128 KiB
[default: 131072]
Database:
--db.log-level <LOG_LEVEL>
Database logging level. Levels higher than "notice" require a debug build

17
book/run/config.md
View File

@ -12,7 +12,6 @@ The configuration file contains the following sections:
- [`[stages]`](#the-stages-section) -- Configuration of the individual sync stages
- [`headers`](#headers)
- [`total_difficulty`](#total_difficulty)
- [`bodies`](#bodies)
- [`sender_recovery`](#sender_recovery)
- [`execution`](#execution)
@ -64,20 +63,6 @@ downloader_request_limit = 1000
commit_threshold = 10000
```
### `total_difficulty`
The total difficulty stage calculates the total difficulty reached for each header in the chain.
```toml
[stages.total_difficulty]
# The amount of headers to calculate the total difficulty for
# before writing the results to disk.
#
# Lower thresholds correspond to more frequent disk I/O (writes),
# but lowers memory usage
commit_threshold = 100000
```
### `bodies`
The bodies section controls both the behavior of the bodies stage, which download historical block bodies, as well as the primary downloader that fetches block bodies over P2P.
@ -207,7 +192,7 @@ The transaction lookup stage builds an index of transaction hashes to their sequ
#
# Lower thresholds correspond to more frequent disk I/O (writes),
# but lowers memory usage
commit_threshold = 5000000
chunk_size = 5000000
```
### `index_account_history`

2
book/run/observability.md
View File

@ -12,7 +12,7 @@ Now, as the node is running, you can `curl` the endpoint you provided to the `--
curl 127.0.0.1:9001
```
The response from this is quite descriptive, but it can be a bit verbose. Plus, it's just a snapshot of the metrics at the time that you `curl`ed the endpoint.
The response from this is quite descriptive, but it can be a bit verbose. Plus, it's just a static_file of the metrics at the time that you `curl`ed the endpoint.
You can run the following command in a separate terminal to periodically poll the endpoint, and just print the values (without the header text) to the terminal:

180
book/run/pruning.md
View File

@ -2,7 +2,7 @@
> Pruning and full node are new features of Reth,
> and we will be happy to hear about your experience using them either
> on [GitHub](https://github.com/paradigmxyz/reth/issues) or in the [Telegram group](https://t.me/paradigm_reth).
> on [GitHub](https://github.com/paradigmxyz/reth/issues) or in the [Telegram group](https://t.me/paradigm_reth).
By default, Reth runs as an archive node. Such nodes have all historical blocks and the state at each of these blocks
available for querying and tracing.
@ -14,14 +14,15 @@ the steps for running Reth as a full node, what caveats to expect and how to con
- Archive node – Reth node that has all historical data from genesis.
- Pruned node – Reth node that has its historical data pruned partially or fully through
a [custom configuration](./config.md#the-prune-section).
a [custom configuration](./config.md#the-prune-section).
- Full Node – Reth node that has the latest state and historical data for only the last 10064 blocks available
for querying in the same way as an archive node.
for querying in the same way as an archive node.
The node type that was chosen when first [running a node](./run-a-node.md) **can not** be changed after
the initial sync. Turning Archive into Pruned, or Pruned into Full is not supported.
## Modes
### Archive Node
Default mode, follow the steps from the previous chapter on [how to run on mainnet or official testnets](./mainnet.md).
@ -36,6 +37,7 @@ the previous chapter on [how to run on mainnet or official testnets](./mainnet.m
To run Reth as a full node, follow the steps from the previous chapter on
[how to run on mainnet or official testnets](./mainnet.md), and add a `--full` flag. For example:
```bash
RUST_LOG=info reth node \
--full \
@ -61,7 +63,7 @@ Different segments take up different amounts of disk space.
If pruned fully, this is the total freed space you'll get, per segment:
| Segment | Size |
|--------------------|-------|
| ------------------ | ----- |
| Sender Recovery | 75GB |
| Transaction Lookup | 150GB |
| Receipts | 250GB |
@ -73,6 +75,7 @@ If pruned fully, this is the total freed space you'll get, per segment:
Full node occupies at least 950GB.
Essentially, the full node is the same as following configuration for the pruned node:
```toml
[prune]
block_interval = 5
@ -91,15 +94,18 @@ storage_history = { distance = 10_064 }
```
Meaning, it prunes:
- Account History and Storage History up to the last 10064 blocks
- All of Sender Recovery data. The caveat is that it's pruned gradually after the initial sync
is completed, so the disk space is reclaimed slowly.
is completed, so the disk space is reclaimed slowly.
- Receipts up to the last 10064 blocks, preserving all receipts with the logs from Beacon Deposit Contract
Given the aforementioned segment sizes, we get the following full node size:
```text
Archive Node - Receipts - AccountHistory - StorageHistory = Full Node
Archive Node - Receipts - AccountsHistory - StoragesHistory = Full Node
```
```text
2.14TB - 250GB - 240GB - 700GB = 950GB
```
@ -108,6 +114,7 @@ Archive Node - Receipts - AccountHistory - StorageHistory = Full Node
As it was mentioned in the [pruning configuration chapter](./config.md#the-prune-section), there are several segments which can be pruned
independently of each other:
- Sender Recovery
- Transaction Lookup
- Receipts
@ -121,11 +128,10 @@ become unavailable.
The following tables describe RPC methods available in the full node.
#### `debug` namespace
| RPC | Note |
|----------------------------|------------------------------------------------------------|
| -------------------------- | ---------------------------------------------------------- |
| `debug_getRawBlock` | |
| `debug_getRawHeader` | |
| `debug_getRawReceipts` | Only for the last 10064 blocks and Beacon Deposit Contract |
@ -137,11 +143,10 @@ The following tables describe RPC methods available in the full node.
| `debug_traceCallMany` | Only for the last 10064 blocks |
| `debug_traceTransaction` | Only for the last 10064 blocks |
#### `eth` namespace
| RPC / Segment | Note |
|-------------------------------------------|------------------------------------------------------------|
| ----------------------------------------- | ---------------------------------------------------------- |
| `eth_accounts` | |
| `eth_blockNumber` | |
| `eth_call` | Only for the last 10064 blocks |
@ -189,7 +194,7 @@ The following tables describe RPC methods available in the full node.
#### `net` namespace
| RPC / Segment |
|-----------------|
| --------------- |
| `net_listening` |
| `net_peerCount` |
| `net_version` |
@ -197,7 +202,7 @@ The following tables describe RPC methods available in the full node.
#### `trace` namespace
| RPC / Segment | Note |
|---------------------------------|--------------------------------|
| ------------------------------- | ------------------------------ |
| `trace_block` | Only for the last 10064 blocks |
| `trace_call` | Only for the last 10064 blocks |
| `trace_callMany` | Only for the last 10064 blocks |
@ -210,109 +215,108 @@ The following tables describe RPC methods available in the full node.
#### `txpool` namespace
| RPC / Segment |
|----------------------|
| -------------------- |
| `txpool_content` |
| `txpool_contentFrom` |
| `txpool_inspect` |
| `txpool_status` |
### Pruned Node
The following tables describe the requirements for prune segments, per RPC method:
- ✅ – if the segment is pruned, the RPC method still works
- ❌ - if the segment is pruned, the RPC method doesn't work anymore
#### `debug` namespace
| RPC / Segment | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
|----------------------------|-----------------|--------------------|----------|-----------------|-----------------|
| `debug_getRawBlock` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `debug_getRawHeader` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `debug_getRawReceipts` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `debug_getRawTransaction` | ✅ | ❌ | ✅ | ✅ | ✅ |
| `debug_traceBlock` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceBlockByHash` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceBlockByNumber` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceCall` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceCallMany` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceTransaction` | ✅ | ✅ | ✅ | ❌ | ❌ |
| -------------------------- | --------------- | ------------------ | -------- | --------------- | --------------- |
| `debug_getRawBlock` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `debug_getRawHeader` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `debug_getRawReceipts` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `debug_getRawTransaction` | ✅ | ❌ | ✅ | ✅ | ✅ |
| `debug_traceBlock` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceBlockByHash` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceBlockByNumber` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceCall` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceCallMany` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `debug_traceTransaction` | ✅ | ✅ | ✅ | ❌ | ❌ |
#### `eth` namespace
| RPC / Segment | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
|-------------------------------------------|-----------------|--------------------|----------|-----------------|-----------------|
| `eth_accounts` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_blockNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_call` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_chainId` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_createAccessList` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_estimateGas` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_feeHistory` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_gasPrice` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBalance` | ✅ | ✅ | ✅ | ❌ | ✅ |
| `eth_getBlockByHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockByNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockReceipts` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getBlockTransactionCountByHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockTransactionCountByNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getCode` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getFilterChanges` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getFilterLogs` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getLogs` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getStorageAt` | ✅ | ✅ | ✅ | ✅ | ❌ |
| `eth_getTransactionByBlockHashAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getTransactionByBlockNumberAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getTransactionByHash` | ✅ | ❌ | ✅ | ✅ | ✅ |
| `eth_getTransactionCount` | ✅ | ✅ | ✅ | ❌ | ✅ |
| `eth_getTransactionReceipt` | ✅ | ❌ | ❌ | ✅ | ✅ |
| `eth_getUncleByBlockHashAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleByBlockNumberAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleCountByBlockHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleCountByBlockNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_maxPriorityFeePerGas` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_mining` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newBlockFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newPendingTransactionFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_protocolVersion` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sendRawTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sendTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sign` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_signTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_signTypedData` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_subscribe` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_syncing` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_uninstallFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_unsubscribe` | ✅ | ✅ | ✅ | ✅ | ✅ |
| ----------------------------------------- | --------------- | ------------------ | -------- | --------------- | --------------- |
| `eth_accounts` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_blockNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_call` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_chainId` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_createAccessList` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_estimateGas` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `eth_feeHistory` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_gasPrice` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBalance` | ✅ | ✅ | ✅ | ❌ | ✅ |
| `eth_getBlockByHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockByNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockReceipts` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getBlockTransactionCountByHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getBlockTransactionCountByNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getCode` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getFilterChanges` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getFilterLogs` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getLogs` | ✅ | ✅ | ❌ | ✅ | ✅ |
| `eth_getStorageAt` | ✅ | ✅ | ✅ | ✅ | ❌ |
| `eth_getTransactionByBlockHashAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getTransactionByBlockNumberAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getTransactionByHash` | ✅ | ❌ | ✅ | ✅ | ✅ |
| `eth_getTransactionCount` | ✅ | ✅ | ✅ | ❌ | ✅ |
| `eth_getTransactionReceipt` | ✅ | ❌ | ❌ | ✅ | ✅ |
| `eth_getUncleByBlockHashAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleByBlockNumberAndIndex` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleCountByBlockHash` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_getUncleCountByBlockNumber` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_maxPriorityFeePerGas` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_mining` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newBlockFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_newPendingTransactionFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_protocolVersion` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sendRawTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sendTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_sign` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_signTransaction` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_signTypedData` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_subscribe` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_syncing` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_uninstallFilter` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `eth_unsubscribe` | ✅ | ✅ | ✅ | ✅ | ✅ |
#### `net` namespace
| RPC / Segment | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
|-----------------|-----------------|--------------------|----------|-----------------|-----------------|
| `net_listening` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `net_peerCount` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `net_version` | ✅ | ✅ | ✅ | ✅ | ✅ |
| --------------- | --------------- | ------------------ | -------- | --------------- | --------------- |
| `net_listening` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `net_peerCount` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `net_version` | ✅ | ✅ | ✅ | ✅ | ✅ |
#### `trace` namespace
| RPC / Segment | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
|---------------------------------|-----------------|--------------------|----------|-----------------|-----------------|
| `trace_block` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_call` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_callMany` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_get` | ✅ | ❌ | ✅ | ❌ | ❌ |
| `trace_rawTransaction` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_replayBlockTransactions` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_replayTransaction` | ✅ | ❌ | ✅ | ❌ | ❌ |
| `trace_transaction` | ✅ | ❌ | ✅ | ❌ | ❌ |
| ------------------------------- | --------------- | ------------------ | -------- | --------------- | --------------- |
| `trace_block` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_call` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_callMany` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_get` | ✅ | ❌ | ✅ | ❌ | ❌ |
| `trace_rawTransaction` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_replayBlockTransactions` | ✅ | ✅ | ✅ | ❌ | ❌ |
| `trace_replayTransaction` | ✅ | ❌ | ✅ | ❌ | ❌ |
| `trace_transaction` | ✅ | ❌ | ✅ | ❌ | ❌ |
#### `txpool` namespace
| RPC / Segment | Sender Recovery | Transaction Lookup | Receipts | Account History | Storage History |
|----------------------|-----------------|--------------------|----------|-----------------|-----------------|
| `txpool_content` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_contentFrom` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_inspect` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_status` | ✅ | ✅ | ✅ | ✅ | ✅ |
| -------------------- | --------------- | ------------------ | -------- | --------------- | --------------- |
| `txpool_content` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_contentFrom` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_inspect` | ✅ | ✅ | ✅ | ✅ | ✅ |
| `txpool_status` | ✅ | ✅ | ✅ | ✅ | ✅ |

6
crates/blockchain-tree/src/blockchain_tree.rs
View File

@ -1294,7 +1294,7 @@ mod tests {
let provider = factory.provider_rw().unwrap();
provider
.insert_block(
.insert_historical_block(
genesis.try_seal_with_senders().expect("invalid tx signature in genesis"),
None,
)
@ -1309,7 +1309,7 @@ mod tests {
}
provider
.tx_ref()
.put::<tables::SyncStage>("Finish".to_string(), StageCheckpoint::new(10))
.put::<tables::StageCheckpoints>("Finish".to_string(), StageCheckpoint::new(10))
.unwrap();
provider.commit().unwrap();
}
@ -1423,7 +1423,7 @@ mod tests {
.unwrap();
let account = Account { balance: initial_signer_balance, ..Default::default() };
provider_rw.tx_ref().put::<tables::PlainAccountState>(signer, account).unwrap();
provider_rw.tx_ref().put::<tables::HashedAccount>(keccak256(signer), account).unwrap();
provider_rw.tx_ref().put::<tables::HashedAccounts>(keccak256(signer), account).unwrap();
provider_rw.commit().unwrap();
}

38
crates/blockchain-tree/src/externals.rs
View File

@ -1,9 +1,11 @@
//! Blockchain tree externals.
use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx};
use reth_db::{
cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx,
};
use reth_interfaces::{consensus::Consensus, RethResult};
use reth_primitives::{BlockHash, BlockNumber};
use reth_provider::ProviderFactory;
use reth_primitives::{BlockHash, BlockNumber, StaticFileSegment};
use reth_provider::{ProviderFactory, StatsReader};
use std::{collections::BTreeMap, sync::Arc};
/// A container for external components.
@ -44,13 +46,39 @@ impl<DB: Database, EVM> TreeExternals<DB, EVM> {
&self,
num_hashes: usize,
) -> RethResult<BTreeMap<BlockNumber, BlockHash>> {
Ok(self
// Fetch the latest canonical hashes from the database
let mut hashes = self
.provider_factory
.provider()?
.tx_ref()
.cursor_read::<tables::CanonicalHeaders>()?
.walk_back(None)?
.take(num_hashes)
.collect::<Result<BTreeMap<BlockNumber, BlockHash>, _>>()?)
.collect::<Result<BTreeMap<BlockNumber, BlockHash>, _>>()?;
// Fetch the same number of latest canonical hashes from the static_files and merge them
// with the database hashes. It is needed due to the fact that we're writing
// directly to static_files in pipeline sync, but to the database in live sync,
// which means that the latest canonical hashes in the static file might be more recent
// than in the database, and vice versa, or even some ranges of the latest
// `num_hashes` blocks may be in database, and some ranges in static_files.
let static_file_provider = self.provider_factory.static_file_provider();
let total_headers = static_file_provider.count_entries::<tables::Headers>()? as u64;
if total_headers > 0 {
let range =
total_headers.saturating_sub(1).saturating_sub(num_hashes as u64)..total_headers;
hashes.extend(range.clone().zip(static_file_provider.fetch_range_with_predicate(
StaticFileSegment::Headers,
range,
|cursor, number| cursor.get_one::<HeaderMask<BlockHash>>(number.into()),
|_| true,
)?));
}
// We may have fetched more than `num_hashes` hashes, so we need to truncate the result to
// the requested number.
let hashes = hashes.into_iter().rev().take(num_hashes).collect();
Ok(hashes)
}
}

28
crates/config/src/config.rs
View File

@ -52,8 +52,6 @@ impl Config {
pub struct StageConfig {
/// Header stage configuration.
pub headers: HeadersConfig,
/// Total Difficulty stage configuration
pub total_difficulty: TotalDifficultyConfig,
/// Body stage configuration.
pub bodies: BodiesConfig,
/// Sender Recovery stage configuration.
@ -107,21 +105,6 @@ impl Default for HeadersConfig {
}
}
/// Total difficulty stage configuration
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub struct TotalDifficultyConfig {
/// The maximum number of total difficulty entries to sum up before committing progress to the
/// database.
pub commit_threshold: u64,
}
impl Default for TotalDifficultyConfig {
fn default() -> Self {
Self { commit_threshold: 100_000 }
}
}
/// Body stage configuration.
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
#[serde(default)]
@ -242,13 +225,13 @@ impl Default for MerkleConfig {
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub struct TransactionLookupConfig {
/// The maximum number of transactions to process before committing progress to the database.
pub commit_threshold: u64,
/// The maximum number of transactions to process before writing to disk.
pub chunk_size: u64,
}
impl Default for TransactionLookupConfig {
fn default() -> Self {
Self { commit_threshold: 5_000_000 }
Self { chunk_size: 5_000_000 }
}
}
@ -359,9 +342,6 @@ downloader_max_buffered_responses = 100
downloader_request_limit = 1000
commit_threshold = 10000
[stages.total_difficulty]
commit_threshold = 100000
[stages.bodies]
downloader_request_limit = 200
downloader_stream_batch_size = 1000
@ -388,7 +368,7 @@ commit_threshold = 100000
clean_threshold = 50000
[stages.transaction_lookup]
commit_threshold = 5000000
chunk_size = 5000000
[stages.index_account_history]
commit_threshold = 100000

3
crates/consensus/beacon/Cargo.toml
View File

@ -23,7 +23,7 @@ reth-tasks.workspace = true
reth-payload-builder.workspace = true
reth-payload-validator.workspace = true
reth-prune.workspace = true
reth-snapshot.workspace = true
reth-static-file.workspace = true
reth-tokio-util.workspace = true
reth-node-api.workspace = true
@ -59,6 +59,7 @@ reth-node-ethereum.workspace = true
reth-node-optimism.workspace = true
assert_matches.workspace = true
tempfile.workspace = true
[features]
optimism = [

3
crates/consensus/beacon/src/engine/hooks/controller.rs
View File

@ -10,7 +10,6 @@ use tracing::debug;
#[derive(Debug)]
pub(crate) struct PolledHook {
#[allow(dead_code)]
pub(crate) name: &'static str,
pub(crate) event: EngineHookEvent,
pub(crate) db_access_level: EngineHookDBAccessLevel,
@ -151,6 +150,8 @@ impl EngineHooksController {
);
return Poll::Ready(Ok(result))
} else {
debug!(target: "consensus::engine::hooks", hook = hook.name(), "Next hook is not ready");
}
Poll::Pending

4
crates/consensus/beacon/src/engine/hooks/mod.rs
View File

@ -11,8 +11,8 @@ pub(crate) use controller::{EngineHooksController, PolledHook};
mod prune;
pub use prune::PruneHook;
mod snapshot;
pub use snapshot::SnapshotHook;
mod static_file;
pub use static_file::StaticFileHook;
/// Collection of [engine hooks][`EngineHook`].
#[derive(Default)]

156
crates/consensus/beacon/src/engine/hooks/snapshot.rs
View File

@ -1,156 +0,0 @@
//! Snapshot hook for the engine implementation.
use crate::{
engine::hooks::{EngineContext, EngineHook, EngineHookError, EngineHookEvent},
hooks::EngineHookDBAccessLevel,
};
use futures::FutureExt;
use reth_db::database::Database;
use reth_interfaces::{RethError, RethResult};
use reth_primitives::BlockNumber;
use reth_snapshot::{Snapshotter, SnapshotterError, SnapshotterWithResult};
use reth_tasks::TaskSpawner;
use std::task::{ready, Context, Poll};
use tokio::sync::oneshot;
/// Manages snapshotting under the control of the engine.
///
/// This type controls the [Snapshotter].
#[derive(Debug)]
pub struct SnapshotHook<DB> {
/// The current state of the snapshotter.
state: SnapshotterState<DB>,
/// The type that can spawn the snapshotter task.
task_spawner: Box<dyn TaskSpawner>,
}
impl<DB: Database + 'static> SnapshotHook<DB> {
/// Create a new instance
pub fn new(snapshotter: Snapshotter<DB>, task_spawner: Box<dyn TaskSpawner>) -> Self {
Self { state: SnapshotterState::Idle(Some(snapshotter)), task_spawner }
}
/// Advances the snapshotter state.
///
/// This checks for the result in the channel, or returns pending if the snapshotter is idle.
fn poll_snapshotter(&mut self, cx: &mut Context<'_>) -> Poll<RethResult<EngineHookEvent>> {
let result = match self.state {
SnapshotterState::Idle(_) => return Poll::Pending,
SnapshotterState::Running(ref mut fut) => {
ready!(fut.poll_unpin(cx))
}
};
let event = match result {
Ok((snapshotter, result)) => {
self.state = SnapshotterState::Idle(Some(snapshotter));
match result {
Ok(_) => EngineHookEvent::Finished(Ok(())),
Err(err) => EngineHookEvent::Finished(Err(err.into())),
}
}
Err(_) => {
// failed to receive the snapshotter
EngineHookEvent::Finished(Err(EngineHookError::ChannelClosed))
}
};
Poll::Ready(Ok(event))
}
/// This will try to spawn the snapshotter if it is idle:
/// 1. Check if snapshotting is needed through [Snapshotter::get_snapshot_targets] and then
/// [SnapshotTargets::any](reth_snapshot::SnapshotTargets::any).
/// 2.
/// 1. If snapshotting is needed, pass snapshot request to the [Snapshotter::run] and spawn
/// it in a separate task. Set snapshotter state to [SnapshotterState::Running].
/// 2. If snapshotting is not needed, set snapshotter state back to
/// [SnapshotterState::Idle].
///
/// If snapshotter is already running, do nothing.
fn try_spawn_snapshotter(
&mut self,
finalized_block_number: BlockNumber,
) -> RethResult<Option<EngineHookEvent>> {
Ok(match &mut self.state {
SnapshotterState::Idle(snapshotter) => {
let Some(mut snapshotter) = snapshotter.take() else { return Ok(None) };
let targets = snapshotter.get_snapshot_targets(finalized_block_number)?;
// Check if the snapshotting of any data has been requested.
if targets.any() {
let (tx, rx) = oneshot::channel();
self.task_spawner.spawn_critical_blocking(
"snapshotter task",
Box::pin(async move {
let result = snapshotter.run(targets);
let _ = tx.send((snapshotter, result));
}),
);
self.state = SnapshotterState::Running(rx);
Some(EngineHookEvent::Started)
} else {
self.state = SnapshotterState::Idle(Some(snapshotter));
Some(EngineHookEvent::NotReady)
}
}
SnapshotterState::Running(_) => None,
})
}
}
impl<DB: Database + 'static> EngineHook for SnapshotHook<DB> {
fn name(&self) -> &'static str {
"Snapshot"
}
fn poll(
&mut self,
cx: &mut Context<'_>,
ctx: EngineContext,
) -> Poll<RethResult<EngineHookEvent>> {
let Some(finalized_block_number) = ctx.finalized_block_number else {
return Poll::Ready(Ok(EngineHookEvent::NotReady))
};
// Try to spawn a snapshotter
match self.try_spawn_snapshotter(finalized_block_number)? {
Some(EngineHookEvent::NotReady) => return Poll::Pending,
Some(event) => return Poll::Ready(Ok(event)),
None => (),
}
// Poll snapshotter and check its status
self.poll_snapshotter(cx)
}
fn db_access_level(&self) -> EngineHookDBAccessLevel {
EngineHookDBAccessLevel::ReadOnly
}
}
/// The possible snapshotter states within the sync controller.
///
/// [SnapshotterState::Idle] means that the snapshotter is currently idle.
/// [SnapshotterState::Running] means that the snapshotter is currently running.
#[derive(Debug)]
enum SnapshotterState<DB> {
/// Snapshotter is idle.
Idle(Option<Snapshotter<DB>>),
/// Snapshotter is running and waiting for a response
Running(oneshot::Receiver<SnapshotterWithResult<DB>>),
}
impl From<SnapshotterError> for EngineHookError {
fn from(err: SnapshotterError) -> Self {
match err {
SnapshotterError::InconsistentData(_) => EngineHookError::Internal(Box::new(err)),
SnapshotterError::Interface(err) => err.into(),
SnapshotterError::Database(err) => RethError::Database(err).into(),
SnapshotterError::Provider(err) => RethError::Provider(err).into(),
}
}
}

163
crates/consensus/beacon/src/engine/hooks/static_file.rs Normal file
View File

@ -0,0 +1,163 @@
//! StaticFile hook for the engine implementation.
use crate::{
engine::hooks::{EngineContext, EngineHook, EngineHookError, EngineHookEvent},
hooks::EngineHookDBAccessLevel,
};
use futures::FutureExt;
use reth_db::database::Database;
use reth_interfaces::RethResult;
use reth_primitives::{static_file::HighestStaticFiles, BlockNumber};
use reth_static_file::{StaticFileProducer, StaticFileProducerWithResult};
use reth_tasks::TaskSpawner;
use std::task::{ready, Context, Poll};
use tokio::sync::oneshot;
use tracing::trace;
/// Manages producing static files under the control of the engine.
///
/// This type controls the [StaticFileProducer].
#[derive(Debug)]
pub struct StaticFileHook<DB> {
/// The current state of the static_file_producer.
state: StaticFileProducerState<DB>,
/// The type that can spawn the static_file_producer task.
task_spawner: Box<dyn TaskSpawner>,
}
impl<DB: Database + 'static> StaticFileHook<DB> {
/// Create a new instance
pub fn new(
static_file_producer: StaticFileProducer<DB>,
task_spawner: Box<dyn TaskSpawner>,
) -> Self {
Self { state: StaticFileProducerState::Idle(Some(static_file_producer)), task_spawner }
}
/// Advances the static_file_producer state.
///
/// This checks for the result in the channel, or returns pending if the static_file_producer is
/// idle.
fn poll_static_file_producer(
&mut self,
cx: &mut Context<'_>,
) -> Poll<RethResult<EngineHookEvent>> {
let result = match self.state {
StaticFileProducerState::Idle(_) => return Poll::Pending,
StaticFileProducerState::Running(ref mut fut) => {
ready!(fut.poll_unpin(cx))
}
};
let event = match result {
Ok((static_file_producer, result)) => {
self.state = StaticFileProducerState::Idle(Some(static_file_producer));
match result {
Ok(_) => EngineHookEvent::Finished(Ok(())),
Err(err) => EngineHookEvent::Finished(Err(err.into())),
}
}
Err(_) => {
// failed to receive the static_file_producer
EngineHookEvent::Finished(Err(EngineHookError::ChannelClosed))
}
};
Poll::Ready(Ok(event))
}
/// This will try to spawn the static_file_producer if it is idle:
/// 1. Check if producing static files is needed through
/// [StaticFileProducer::get_static_file_targets] and then
/// [StaticFileTargets::any](reth_static_file::StaticFileTargets::any).
/// 2.
/// 1. If producing static files is needed, pass static file request to the
/// [StaticFileProducer::run] and spawn it in a separate task. Set static file producer
/// state to [StaticFileProducerState::Running].
/// 2. If producing static files is not needed, set static file producer state back to
/// [StaticFileProducerState::Idle].
///
/// If static_file_producer is already running, do nothing.
fn try_spawn_static_file_producer(
&mut self,
finalized_block_number: BlockNumber,
) -> RethResult<Option<EngineHookEvent>> {
Ok(match &mut self.state {
StaticFileProducerState::Idle(static_file_producer) => {
let Some(mut static_file_producer) = static_file_producer.take() else {
trace!(target: "consensus::engine::hooks::static_file", "StaticFileProducer is already running but the state is idle");
return Ok(None);
};
let targets = static_file_producer.get_static_file_targets(HighestStaticFiles {
headers: Some(finalized_block_number),
receipts: Some(finalized_block_number),
transactions: Some(finalized_block_number),
})?;
// Check if the moving data to static files has been requested.
if targets.any() {
let (tx, rx) = oneshot::channel();
self.task_spawner.spawn_critical_blocking(
"static_file_producer task",
Box::pin(async move {
let result = static_file_producer.run(targets);
let _ = tx.send((static_file_producer, result));
}),
);
self.state = StaticFileProducerState::Running(rx);
Some(EngineHookEvent::Started)
} else {
self.state = StaticFileProducerState::Idle(Some(static_file_producer));
Some(EngineHookEvent::NotReady)
}
}
StaticFileProducerState::Running(_) => None,
})
}
}
impl<DB: Database + 'static> EngineHook for StaticFileHook<DB> {
fn name(&self) -> &'static str {
"StaticFile"
}
fn poll(
&mut self,
cx: &mut Context<'_>,
ctx: EngineContext,
) -> Poll<RethResult<EngineHookEvent>> {
let Some(finalized_block_number) = ctx.finalized_block_number else {
trace!(target: "consensus::engine::hooks::static_file", ?ctx, "Finalized block number is not available");
return Poll::Pending;
};
// Try to spawn a static_file_producer
match self.try_spawn_static_file_producer(finalized_block_number)? {
Some(EngineHookEvent::NotReady) => return Poll::Pending,
Some(event) => return Poll::Ready(Ok(event)),
None => (),
}
// Poll static_file_producer and check its status
self.poll_static_file_producer(cx)
}
fn db_access_level(&self) -> EngineHookDBAccessLevel {
EngineHookDBAccessLevel::ReadOnly
}
}
/// The possible static_file_producer states within the sync controller.
///
/// [StaticFileProducerState::Idle] means that the static file producer is currently idle.
/// [StaticFileProducerState::Running] means that the static file producer is currently running.
#[derive(Debug)]
enum StaticFileProducerState<DB> {
/// [StaticFileProducer] is idle.
Idle(Option<StaticFileProducer<DB>>),
/// [StaticFileProducer] is running and waiting for a response
Running(oneshot::Receiver<StaticFileProducerWithResult<DB>>),
}

134
crates/consensus/beacon/src/engine/mod.rs
View File

@ -361,6 +361,9 @@ where
warn!(
target: "consensus::engine",
hook = %hook.name(),
head_block_hash = ?state.head_block_hash,
safe_block_hash = ?state.safe_block_hash,
finalized_block_hash = ?state.finalized_block_hash,
"Hook is in progress, skipping forkchoice update. \
This may affect the performance of your node as a validator."
);
@ -1502,7 +1505,9 @@ where
debug!(target: "consensus::engine", hash=?new_head.hash(), number=new_head.number, "Canonicalized new head");
// we can update the FCU blocks
let _ = self.update_canon_chain(new_head, &target);
if let Err(err) = self.update_canon_chain(new_head, &target) {
debug!(target: "consensus::engine", ?err, ?target, "Failed to update the canonical chain tracker");
}
// we're no longer syncing
self.sync_state_updater.update_sync_state(SyncState::Idle);
@ -1704,9 +1709,18 @@ where
None
}
fn on_hook_result(&self, result: PolledHook) -> Result<(), BeaconConsensusEngineError> {
if result.db_access_level.is_read_write() {
match result.event {
fn on_hook_result(&self, polled_hook: PolledHook) -> Result<(), BeaconConsensusEngineError> {
if let EngineHookEvent::Finished(Err(error)) = &polled_hook.event {
error!(
target: "consensus::engine",
name = %polled_hook.name,
?error,
"Hook finished with error"
)
}
if polled_hook.db_access_level.is_read_write() {
match polled_hook.event {
EngineHookEvent::NotReady => {}
EngineHookEvent::Started => {
// If the hook has read-write access to the database, it means that the engine
@ -1889,9 +1903,7 @@ mod tests {
};
use assert_matches::assert_matches;
use reth_interfaces::test_utils::generators::{self, Rng};
use reth_primitives::{
stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, B256, MAINNET, U256,
};
use reth_primitives::{stage::StageCheckpoint, ChainSpecBuilder, MAINNET};
use reth_provider::{BlockWriter, ProviderFactory};
use reth_rpc_types::engine::{ForkchoiceState, ForkchoiceUpdated, PayloadStatus};
use reth_rpc_types_compat::engine::payload::try_block_to_payload_v1;
@ -2064,12 +2076,10 @@ mod tests {
}
fn insert_blocks<'a, DB: Database>(
db: DB,
chain: Arc<ChainSpec>,
provider_factory: ProviderFactory<DB>,
mut blocks: impl Iterator<Item = &'a SealedBlock>,
) {
let factory = ProviderFactory::new(db, chain);
let provider = factory.provider_rw().unwrap();
let provider = provider_factory.provider_rw().unwrap();
blocks
.try_for_each(|b| {
provider
@ -2085,8 +2095,9 @@ mod tests {
mod fork_choice_updated {
use super::*;
use reth_db::{tables, transaction::DbTxMut};
use reth_db::{tables, test_utils::create_test_static_files_dir, transaction::DbTxMut};
use reth_interfaces::test_utils::generators::random_block;
use reth_primitives::U256;
use reth_rpc_types::engine::ForkchoiceUpdateError;
#[tokio::test]
@ -2139,10 +2150,18 @@ mod tests {
let genesis = random_block(&mut rng, 0, None, None, Some(0));
let block1 = random_block(&mut rng, 1, Some(genesis.hash()), None, Some(0));
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1].into_iter(),
);
env.db
.update(|tx| {
tx.put::<tables::SyncStage>(
tx.put::<tables::StageCheckpoints>(
StageId::Finish.to_string(),
StageCheckpoint::new(block1.number),
)
@ -2189,7 +2208,15 @@ mod tests {
let genesis = random_block(&mut rng, 0, None, None, Some(0));
let block1 = random_block(&mut rng, 1, Some(genesis.hash()), None, Some(0));
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1].into_iter(),
);
let mut engine_rx = spawn_consensus_engine(consensus_engine);
@ -2205,7 +2232,15 @@ mod tests {
let invalid_rx = env.send_forkchoice_updated(next_forkchoice_state).await;
// Insert next head immediately after sending forkchoice update
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&next_head].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&next_head].into_iter(),
);
let expected_result = ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing);
assert_matches!(invalid_rx, Ok(result) => assert_eq!(result, expected_result));
@ -2239,7 +2274,15 @@ mod tests {
let genesis = random_block(&mut rng, 0, None, None, Some(0));
let block1 = random_block(&mut rng, 1, Some(genesis.hash()), None, Some(0));
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1].into_iter(),
);
let engine = spawn_consensus_engine(consensus_engine);
@ -2287,8 +2330,12 @@ mod tests {
block3.header.set_difficulty(U256::from(1));
insert_blocks(
env.db.as_ref(),
chain_spec.clone(),
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1, &block2, &block3].into_iter(),
);
@ -2330,7 +2377,15 @@ mod tests {
let genesis = random_block(&mut rng, 0, None, None, Some(0));
let block1 = random_block(&mut rng, 1, Some(genesis.hash()), None, Some(0));
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1].into_iter(),
);
let _engine = spawn_consensus_engine(consensus_engine);
@ -2352,10 +2407,11 @@ mod tests {
mod new_payload {
use super::*;
use reth_db::test_utils::create_test_static_files_dir;
use reth_interfaces::test_utils::generators::random_block;
use reth_primitives::{
genesis::{Genesis, GenesisAllocator},
Hardfork,
Hardfork, U256,
};
use reth_provider::test_utils::blocks::BlockChainTestData;
@ -2426,8 +2482,12 @@ mod tests {
let block1 = random_block(&mut rng, 1, Some(genesis.hash()), None, Some(0));
let block2 = random_block(&mut rng, 2, Some(block1.hash()), None, Some(0));
insert_blocks(
env.db.as_ref(),
chain_spec.clone(),
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1, &block2].into_iter(),
);
@ -2492,7 +2552,15 @@ mod tests {
// TODO: add transactions that transfer from the alloc accounts, generating the new
// block tx and state root
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis, &block1].into_iter(),
);
let mut engine_rx = spawn_consensus_engine(consensus_engine);
@ -2530,7 +2598,15 @@ mod tests {
let genesis = random_block(&mut rng, 0, None, None, Some(0));
insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis].into_iter());
insert_blocks(
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&genesis].into_iter(),
);
let mut engine_rx = spawn_consensus_engine(consensus_engine);
@ -2589,8 +2665,12 @@ mod tests {
.build();
insert_blocks(
env.db.as_ref(),
chain_spec.clone(),
ProviderFactory::new(
env.db.as_ref(),
chain_spec.clone(),
create_test_static_files_dir(),
)
.expect("create provider factory with static_files"),
[&data.genesis, &block1].into_iter(),
);

13
crates/consensus/beacon/src/engine/sync.rs
View File

@ -398,13 +398,14 @@ mod tests {
use reth_interfaces::{p2p::either::EitherDownloader, test_utils::TestFullBlockClient};
use reth_primitives::{
constants::ETHEREUM_BLOCK_GAS_LIMIT, stage::StageCheckpoint, BlockBody, ChainSpecBuilder,
Header, SealedHeader, MAINNET,
Header, PruneModes, SealedHeader, MAINNET,
};
use reth_provider::{
test_utils::{create_test_provider_factory_with_chain_spec, TestExecutorFactory},
BundleStateWithReceipts,
};
use reth_stages::{test_utils::TestStages, ExecOutput, StageError};
use reth_static_file::StaticFileProducer;
use reth_tasks::TokioTaskExecutor;
use std::{collections::VecDeque, future::poll_fn, ops::Range};
use tokio::sync::watch;
@ -465,7 +466,15 @@ mod tests {
pipeline = pipeline.with_max_block(max_block);
}
pipeline.build(create_test_provider_factory_with_chain_spec(chain_spec))
let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec);
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
pipeline.build(provider_factory, static_file_producer)
}
}

48
crates/consensus/beacon/src/engine/test_utils.rs
View File

@ -6,10 +6,7 @@ use crate::{
use reth_blockchain_tree::{
config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree,
};
use reth_db::{
test_utils::{create_test_rw_db, TempDatabase},
DatabaseEnv as DE,
};
use reth_db::{test_utils::TempDatabase, DatabaseEnv as DE};
type DatabaseEnv = TempDatabase<DE>;
use reth_downloaders::{
bodies::bodies::BodiesDownloaderBuilder,
@ -24,10 +21,11 @@ use reth_interfaces::{
};
use reth_node_ethereum::{EthEngineTypes, EthEvmConfig};
use reth_payload_builder::test_utils::spawn_test_payload_service;
use reth_primitives::{BlockNumber, ChainSpec, B256};
use reth_primitives::{BlockNumber, ChainSpec, PruneModes, B256};
use reth_provider::{
providers::BlockchainProvider, test_utils::TestExecutorFactory, BundleStateWithReceipts,
ExecutorFactory, HeaderSyncMode, ProviderFactory, PrunableBlockExecutor,
providers::BlockchainProvider,
test_utils::{create_test_provider_factory_with_chain_spec, TestExecutorFactory},
BundleStateWithReceipts, ExecutorFactory, HeaderSyncMode, PrunableBlockExecutor,
};
use reth_prune::Pruner;
use reth_revm::EvmProcessorFactory;
@ -35,6 +33,7 @@ use reth_rpc_types::engine::{
CancunPayloadFields, ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus,
};
use reth_stages::{sets::DefaultStages, test_utils::TestStages, ExecOutput, Pipeline, StageError};
use reth_static_file::StaticFileProducer;
use reth_tasks::TokioTaskExecutor;
use std::{collections::VecDeque, sync::Arc};
use tokio::sync::{oneshot, watch};
@ -348,9 +347,8 @@ where
/// Builds the test consensus engine into a `TestConsensusEngine` and `TestEnv`.
pub fn build(self) -> (TestBeaconConsensusEngine<Client>, TestEnv<Arc<DatabaseEnv>>) {
reth_tracing::init_test_tracing();
let db = create_test_rw_db();
let provider_factory =
ProviderFactory::new(db.clone(), self.base_config.chain_spec.clone());
create_test_provider_factory_with_chain_spec(self.base_config.chain_spec.clone());
let consensus: Arc<dyn Consensus> = match self.base_config.consensus {
TestConsensusConfig::Real => {
@ -380,6 +378,12 @@ where
)),
};
let static_file_producer = StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
PruneModes::default(),
);
// Setup pipeline
let (tip_tx, tip_rx) = watch::channel(B256::default());
let mut pipeline = match self.base_config.pipeline_config {
@ -395,14 +399,17 @@ where
.build(client.clone(), consensus.clone(), provider_factory.clone())
.into_task();
Pipeline::builder().add_stages(DefaultStages::new(
ProviderFactory::new(db.clone(), self.base_config.chain_spec.clone()),
HeaderSyncMode::Tip(tip_rx.clone()),
Arc::clone(&consensus),
header_downloader,
body_downloader,
executor_factory.clone(),
))
Pipeline::builder().add_stages(
DefaultStages::new(
provider_factory.clone(),
HeaderSyncMode::Tip(tip_rx.clone()),
Arc::clone(&consensus),
header_downloader,
body_downloader,
executor_factory.clone(),
)
.expect("should build"),
)
}
};
@ -410,7 +417,7 @@ where
pipeline = pipeline.with_max_block(max_block);
}
let pipeline = pipeline.build(provider_factory.clone());
let pipeline = pipeline.build(provider_factory.clone(), static_file_producer);
// Setup blockchain tree
let externals = TreeExternals::new(provider_factory.clone(), consensus, executor_factory);
@ -423,12 +430,11 @@ where
BlockchainProvider::with_latest(provider_factory.clone(), tree, latest);
let pruner = Pruner::new(
provider_factory,
provider_factory.clone(),
vec![],
5,
self.base_config.chain_spec.prune_delete_limit,
config.max_reorg_depth() as usize,
watch::channel(None).1,
);
let mut hooks = EngineHooks::new();
@ -453,7 +459,7 @@ where
engine.sync.set_max_block(max_block)
}
(engine, TestEnv::new(db, tip_rx, handle))
(engine, TestEnv::new(provider_factory.db_ref().clone(), tip_rx, handle))
}
}

4
crates/consensus/common/src/validation.rs
View File

@ -439,7 +439,7 @@ mod tests {
gas_price: 0x28f000fff,
gas_limit: 10,
to: TransactionKind::Call(Address::default()),
value: 3_u64.into(),
value: U256::from(3_u64),
input: Bytes::from(vec![1, 2]),
access_list: Default::default(),
});
@ -461,7 +461,7 @@ mod tests {
max_fee_per_blob_gas: 0x7,
gas_limit: 10,
to: TransactionKind::Call(Address::default()),
value: 3_u64.into(),
value: U256::from(3_u64),
input: Bytes::from(vec![1, 2]),
access_list: Default::default(),
blob_versioned_hashes: std::iter::repeat_with(|| rng.gen()).take(num_blobs).collect(),

17
crates/etl/Cargo.toml Normal file
View File

@ -0,0 +1,17 @@
[package]
name = "reth-etl"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
exclude.workspace = true
[dependencies]
tempfile.workspace = true
reth-db.workspace = true
rayon.workspace = true
[dev-dependencies]
reth-primitives.workspace = true

264
crates/etl/src/lib.rs Normal file
View File

@ -0,0 +1,264 @@
//! ETL data collector.
//!
//! This crate is useful for dumping unsorted data into temporary files and iterating on their
//! sorted representation later on.
//!
//! This has multiple uses, such as optimizing database inserts (for Btree based databases) and
//! memory management (as it moves the buffer to disk instead of memory).
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
)]
#![warn(missing_debug_implementations, missing_docs, unreachable_pub, rustdoc::all)]
#![deny(unused_must_use, rust_2018_idioms)]
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
use std::{
cmp::Reverse,
collections::BinaryHeap,
io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
path::Path,
sync::Arc,
};
use rayon::prelude::*;
use reth_db::table::{Compress, Encode, Key, Value};
use tempfile::{NamedTempFile, TempDir};
/// An ETL (extract, transform, load) data collector.
///
/// Data is pushed (extract) to the collector which internally flushes the data in a sorted
/// (transform) manner to files of some specified capacity.
///
/// The data can later be iterated over (load) in a sorted manner.
#[derive(Debug)]
pub struct Collector<K, V>
where
K: Encode + Ord,
V: Compress,
<K as Encode>::Encoded: std::fmt::Debug,
<V as Compress>::Compressed: std::fmt::Debug,
{
/// Directory for temporary file storage
dir: Arc<TempDir>,
/// Collection of temporary ETL files
files: Vec<EtlFile>,
/// Current buffer size in bytes
buffer_size_bytes: usize,
/// Maximum buffer capacity in bytes, triggers flush when reached
buffer_capacity_bytes: usize,
/// In-memory buffer storing encoded and compressed key-value pairs
buffer: Vec<(<K as Encode>::Encoded, <V as Compress>::Compressed)>,
/// Total number of elements in the collector, including all files
len: usize,
}
impl<K, V> Collector<K, V>
where
K: Key,
V: Value,
<K as Encode>::Encoded: Ord + std::fmt::Debug,
<V as Compress>::Compressed: Ord + std::fmt::Debug,
{
/// Create a new collector in a specific temporary directory with some capacity.
///
/// Once the capacity (in bytes) is reached, the data is sorted and flushed to disk.
pub fn new(dir: Arc<TempDir>, buffer_capacity_bytes: usize) -> Self {
Self {
dir,
buffer_size_bytes: 0,
files: Vec::new(),
buffer_capacity_bytes,
buffer: Vec::new(),
len: 0,
}
}
/// Returns number of elements currently in the collector.
pub fn len(&self) -> usize {
self.len
}
/// Returns `true` if there are currently no elements in the collector.
pub fn is_empty(&self) -> bool {
self.len == 0
}
/// Insert an entry into the collector.
pub fn insert(&mut self, key: K, value: V) {
let key = key.encode();
let value = value.compress();
self.buffer_size_bytes += key.as_ref().len() + value.as_ref().len();
self.buffer.push((key, value));
if self.buffer_size_bytes > self.buffer_capacity_bytes {
self.flush();
}
self.len += 1;
}
fn flush(&mut self) {
self.buffer_size_bytes = 0;
self.buffer.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
let mut buf = Vec::with_capacity(self.buffer.len());
std::mem::swap(&mut buf, &mut self.buffer);
self.files.push(EtlFile::new(self.dir.path(), buf).expect("could not flush data to disk"))
}
/// Returns an iterator over the collector data.
///
/// The items of the iterator are sorted across all underlying files.
///
/// # Note
///
/// The keys and values have been pre-encoded, meaning they *SHOULD NOT* be encoded or
/// compressed again.
pub fn iter(&mut self) -> std::io::Result<EtlIter<'_>> {
// Flush the remaining items to disk
if self.buffer_size_bytes > 0 {
self.flush();
}
let mut heap = BinaryHeap::new();
for (current_id, file) in self.files.iter_mut().enumerate() {
if let Some((current_key, current_value)) = file.read_next()? {
heap.push((Reverse((current_key, current_value)), current_id));
}
}
Ok(EtlIter { heap, files: &mut self.files })
}
}
/// `EtlIter` is an iterator for traversing through sorted key-value pairs in a collection of ETL
/// files. These files are created using the [`Collector`] and contain data where keys are encoded
/// and values are compressed.
///
/// This iterator returns each key-value pair in ascending order based on the key.
/// It is particularly designed to efficiently handle large datasets by employing a binary heap for
/// managing the iteration order.
#[derive(Debug)]
pub struct EtlIter<'a> {
/// Heap managing the next items to be iterated.
#[allow(clippy::type_complexity)]
heap: BinaryHeap<(Reverse<(Vec<u8>, Vec<u8>)>, usize)>,
/// Reference to the vector of ETL files being iterated over.
files: &'a mut Vec<EtlFile>,
}
impl<'a> EtlIter<'a> {
/// Peeks into the next element
pub fn peek(&self) -> Option<&(Vec<u8>, Vec<u8>)> {
self.heap.peek().map(|(Reverse(entry), _)| entry)
}
}
impl<'a> Iterator for EtlIter<'a> {
type Item = std::io::Result<(Vec<u8>, Vec<u8>)>;
fn next(&mut self) -> Option<Self::Item> {
// Get the next sorted entry from the heap
let (Reverse(entry), id) = self.heap.pop()?;
// Populate the heap with the next entry from the same file
match self.files[id].read_next() {
Ok(Some((key, value))) => {
self.heap.push((Reverse((key, value)), id));
Some(Ok(entry))
}
Ok(None) => Some(Ok(entry)),
err => err.transpose(),
}
}
}
/// A temporary ETL file.
#[derive(Debug)]
struct EtlFile {
file: BufReader<NamedTempFile>,
len: usize,
}
impl EtlFile {
/// Create a new file with the given data (which should be pre-sorted) at the given path.
///
/// The file will be a temporary file.
pub(crate) fn new<K, V>(dir: &Path, buffer: Vec<(K, V)>) -> std::io::Result<Self>
where
Self: Sized,
K: AsRef<[u8]>,
V: AsRef<[u8]>,
{
let file = NamedTempFile::new_in(dir)?;
let mut w = BufWriter::new(file);
for entry in &buffer {
let k = entry.0.as_ref();
let v = entry.1.as_ref();
w.write_all(&k.len().to_be_bytes())?;
w.write_all(&v.len().to_be_bytes())?;
w.write_all(k)?;
w.write_all(v)?;
}
let mut file = BufReader::new(w.into_inner()?);
file.seek(SeekFrom::Start(0))?;
let len = buffer.len();
Ok(Self { file, len })
}
/// Read the next entry in the file.
///
/// Can return error if it reaches EOF before filling the internal buffers.
pub(crate) fn read_next(&mut self) -> std::io::Result<Option<(Vec<u8>, Vec<u8>)>> {
if self.len == 0 {
return Ok(None)
}
let mut buffer_key_length = [0; 8];
let mut buffer_value_length = [0; 8];
self.file.read_exact(&mut buffer_key_length)?;
self.file.read_exact(&mut buffer_value_length)?;
let key_length = usize::from_be_bytes(buffer_key_length);
let value_length = usize::from_be_bytes(buffer_value_length);
let mut key = vec![0; key_length];
let mut value = vec![0; value_length];
self.file.read_exact(&mut key)?;
self.file.read_exact(&mut value)?;
self.len -= 1;
Ok(Some((key, value)))
}
}
#[cfg(test)]
mod tests {
use reth_primitives::{TxHash, TxNumber};
use super::*;
#[test]
fn etl_hashes() {
let mut entries: Vec<_> =
(0..10_000).map(|id| (TxHash::random(), id as TxNumber)).collect();
let mut collector = Collector::new(Arc::new(TempDir::new().unwrap()), 1024);
for (k, v) in entries.clone() {
collector.insert(k, v);
}
entries.sort_unstable_by_key(|entry| entry.0);
for (id, entry) in collector.iter().unwrap().enumerate() {
let expected = entries[id];
assert_eq!(
entry.unwrap(),
(expected.0.encode().to_vec(), expected.1.compress().to_vec())
);
}
}
}

23
crates/interfaces/src/provider.rs
View File

@ -1,5 +1,5 @@
use reth_primitives::{
Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, SnapshotSegment,
Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, StaticFileSegment,
TxHashOrNumber, TxNumber, B256, U256,
};
use std::path::PathBuf;
@ -113,15 +113,18 @@ pub enum ProviderError {
/// Provider does not support this particular request.
#[error("this provider does not support this request")]
UnsupportedProvider,
/// Snapshot file is not found at specified path.
#[error("not able to find {0} snapshot file at {1}")]
MissingSnapshotPath(SnapshotSegment, PathBuf),
/// Snapshot file is not found for requested block.
#[error("not able to find {0} snapshot file for block number {1}")]
MissingSnapshotBlock(SnapshotSegment, BlockNumber),
/// Snapshot file is not found for requested transaction.
#[error("not able to find {0} snapshot file for transaction id {1}")]
MissingSnapshotTx(SnapshotSegment, TxNumber),
/// Static File is not found at specified path.
#[error("not able to find {0} static file at {1}")]
MissingStaticFilePath(StaticFileSegment, PathBuf),
/// Static File is not found for requested block.
#[error("not able to find {0} static file for block number {1}")]
MissingStaticFileBlock(StaticFileSegment, BlockNumber),
/// Static File is not found for requested transaction.
#[error("unable to find {0} static file for transaction id {1}")]
MissingStaticFileTx(StaticFileSegment, TxNumber),
/// Static File is finalized and cannot be written to.
#[error("unable to write block #{1} to finalized static file {0}")]
FinalizedStaticFile(StaticFileSegment, BlockNumber),
/// Error encountered when the block number conversion from U256 to u64 causes an overflow.
#[error("failed to convert block number U256 to u64: {0}")]
BlockNumberOverflow(U256),

15
crates/interfaces/src/test_utils/generators.rs
View File

@ -80,7 +80,7 @@ pub fn random_tx<R: Rng>(rng: &mut R) -> Transaction {
gas_price: rng.gen::<u16>().into(),
gas_limit: rng.gen::<u16>().into(),
to: TransactionKind::Call(rng.gen()),
value: U256::from(rng.gen::<u16>()).into(),
value: U256::from(rng.gen::<u16>()),
input: Bytes::default(),
})
}
@ -324,12 +324,9 @@ pub fn random_eoa_account<R: Rng>(rng: &mut R) -> (Address, Account) {
}
/// Generate random Externally Owned Accounts
pub fn random_eoa_account_range<R: Rng>(
rng: &mut R,
acc_range: Range<u64>,
) -> Vec<(Address, Account)> {
let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize);
for _ in acc_range {
pub fn random_eoa_accounts<R: Rng>(rng: &mut R, accounts_num: usize) -> Vec<(Address, Account)> {
let mut accounts = Vec::with_capacity(accounts_num);
for _ in 0..accounts_num {
accounts.push(random_eoa_account(rng))
}
accounts
@ -399,7 +396,7 @@ mod tests {
nonce: 0x42,
gas_limit: 44386,
to: TransactionKind::Call(hex!("6069a6c32cf691f5982febae4faf8a6f3ab2f0f6").into()),
value: 0_u64.into(),
value: U256::from(0_u64),
input: hex!("a22cb4650000000000000000000000005eee75727d804a2b13038928d36f8b188945a57a0000000000000000000000000000000000000000000000000000000000000000").into(),
max_fee_per_gas: 0x4a817c800,
max_priority_fee_per_gas: 0x3b9aca00,
@ -431,7 +428,7 @@ mod tests {
gas_price: 20 * 10_u128.pow(9),
gas_limit: 21000,
to: TransactionKind::Call(hex!("3535353535353535353535353535353535353535").into()),
value: 10_u128.pow(18).into(),
value: U256::from(10_u128.pow(18)),
input: Bytes::default(),
});

14
crates/net/downloaders/src/bodies/bodies.rs
View File

@ -596,7 +596,7 @@ mod tests {
test_utils::{generate_bodies, TestBodiesClient},
};
use assert_matches::assert_matches;
use reth_db::test_utils::create_test_rw_db;
use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir};
use reth_interfaces::test_utils::{generators, generators::random_block_range, TestConsensus};
use reth_primitives::{BlockBody, B256, MAINNET};
use reth_provider::ProviderFactory;
@ -618,7 +618,7 @@ mod tests {
let mut downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
downloader.set_download_range(0..=19).expect("failed to set download range");
@ -657,7 +657,7 @@ mod tests {
BodiesDownloaderBuilder::default().with_request_limit(request_limit).build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
downloader.set_download_range(0..=199).expect("failed to set download range");
@ -686,7 +686,7 @@ mod tests {
.build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
let mut range_start = 0;
@ -716,7 +716,7 @@ mod tests {
let mut downloader = BodiesDownloaderBuilder::default().with_stream_batch_size(100).build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
// Set and download the first range
@ -756,7 +756,7 @@ mod tests {
.build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
// Set and download the entire range
@ -787,7 +787,7 @@ mod tests {
.build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(),
);
// Download the requested range

13
crates/net/downloaders/src/bodies/task.rs
View File

@ -169,20 +169,18 @@ mod tests {
test_utils::{generate_bodies, TestBodiesClient},
};
use assert_matches::assert_matches;
use reth_db::test_utils::create_test_rw_db;
use reth_interfaces::{p2p::error::DownloadError, test_utils::TestConsensus};
use reth_primitives::MAINNET;
use reth_provider::ProviderFactory;
use reth_provider::test_utils::create_test_provider_factory;
use std::sync::Arc;
#[tokio::test(flavor = "multi_thread")]
async fn download_one_by_one_on_task() {
reth_tracing::init_test_tracing();
let db = create_test_rw_db();
let factory = create_test_provider_factory();
let (headers, mut bodies) = generate_bodies(0..=19);
insert_headers(db.db(), &headers);
insert_headers(factory.db_ref().db(), &headers);
let client = Arc::new(
TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
@ -190,7 +188,7 @@ mod tests {
let downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
factory,
);
let mut downloader = TaskDownloader::spawn(downloader);
@ -208,11 +206,10 @@ mod tests {
async fn set_download_range_error_returned() {
reth_tracing::init_test_tracing();
let db = create_test_rw_db();
let downloader = BodiesDownloaderBuilder::default().build(
Arc::new(TestBodiesClient::default()),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
create_test_provider_factory(),
);
let mut downloader = TaskDownloader::spawn(downloader);

17
crates/net/downloaders/src/file_client.rs
View File

@ -241,7 +241,6 @@ mod tests {
};
use assert_matches::assert_matches;
use futures_util::stream::StreamExt;
use reth_db::test_utils::create_test_rw_db;
use reth_interfaces::{
p2p::{
bodies::downloader::BodyDownloader,
@ -249,17 +248,17 @@ mod tests {
},
test_utils::TestConsensus,
};
use reth_primitives::{SealedHeader, MAINNET};
use reth_provider::ProviderFactory;
use reth_primitives::SealedHeader;
use reth_provider::test_utils::create_test_provider_factory;
use std::sync::Arc;
#[tokio::test]
async fn streams_bodies_from_buffer() {
// Generate some random blocks
let db = create_test_rw_db();
let factory = create_test_provider_factory();
let (headers, mut bodies) = generate_bodies(0..=19);
insert_headers(db.db(), &headers);
insert_headers(factory.db_ref().db(), &headers);
// create an empty file
let file = tempfile::tempfile().unwrap();
@ -269,7 +268,7 @@ mod tests {
let mut downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
factory,
);
downloader.set_download_range(0..=19).expect("failed to set download range");
@ -337,19 +336,19 @@ mod tests {
#[tokio::test]
async fn test_download_bodies_from_file() {
// Generate some random blocks
let db = create_test_rw_db();
let factory = create_test_provider_factory();
let (file, headers, mut bodies) = generate_bodies_file(0..=19).await;
// now try to read them back
let client = Arc::new(FileClient::from_file(file).await.unwrap());
// insert headers in db for the bodies downloader
insert_headers(db.db(), &headers);
insert_headers(factory.db_ref().db(), &headers);
let mut downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
ProviderFactory::new(db, MAINNET.clone()),
factory,
);
downloader.set_download_range(0..=19).expect("failed to set download range");

2
crates/net/eth-wire/Cargo.toml
View File

@ -25,7 +25,7 @@ reth-metrics.workspace = true
metrics.workspace = true
bytes.workspace = true
derive_more = "0.99.17"
derive_more.workspace = true
thiserror.workspace = true
serde = { workspace = true, optional = true }
tokio = { workspace = true, features = ["full"] }

8
crates/net/eth-wire/src/types/blocks.rs
View File

@ -386,7 +386,7 @@ mod tests {
gas_price: 0x4a817c808,
gas_limit: 0x2e248u64,
to: TransactionKind::Call(hex!("3535353535353535353535353535353535353535").into()),
value: 0x200u64.into(),
value: U256::from(0x200u64),
input: Default::default(),
}),
Signature {
@ -401,7 +401,7 @@ mod tests {
gas_price: 0x4a817c809,
gas_limit: 0x33450u64,
to: TransactionKind::Call(hex!("3535353535353535353535353535353535353535").into()),
value: 0x2d9u64.into(),
value: U256::from(0x2d9u64),
input: Default::default(),
}), Signature {
odd_y_parity: false,
@ -458,7 +458,7 @@ mod tests {
gas_price: 0x4a817c808,
gas_limit: 0x2e248u64,
to: TransactionKind::Call(hex!("3535353535353535353535353535353535353535").into()),
value: 0x200u64.into(),
value: U256::from(0x200u64),
input: Default::default(),
}),
Signature {
@ -474,7 +474,7 @@ mod tests {
gas_price: 0x4a817c809,
gas_limit: 0x33450u64,
to: TransactionKind::Call(hex!("3535353535353535353535353535353535353535").into()),
value: 0x2d9u64.into(),
value: U256::from(0x2d9u64),
input: Default::default(),
}),
Signature {

28
crates/net/eth-wire/src/types/transactions.rs
View File

@ -130,7 +130,7 @@ mod tests {
to: TransactionKind::Call(
hex!("3535353535353535353535353535353535353535").into(),
),
value: 0x200u64.into(),
value: U256::from(0x200u64),
input: Default::default(),
}),
Signature {
@ -154,7 +154,7 @@ mod tests {
to: TransactionKind::Call(
hex!("3535353535353535353535353535353535353535").into(),
),
value: 0x2d9u64.into(),
value: U256::from(0x2d9u64),
input: Default::default(),
}),
Signature {
@ -192,7 +192,7 @@ mod tests {
to: TransactionKind::Call(
hex!("3535353535353535353535353535353535353535").into(),
),
value: 0x200u64.into(),
value: U256::from(0x200u64),
input: Default::default(),
}),
Signature {
@ -216,7 +216,7 @@ mod tests {
to: TransactionKind::Call(
hex!("3535353535353535353535353535353535353535").into(),
),
value: 0x2d9u64.into(),
value: U256::from(0x2d9u64),
input: Default::default(),
}),
Signature {
@ -257,7 +257,7 @@ mod tests {
to: TransactionKind::Call(
hex!("cf7f9e66af820a19257a2108375b180b0ec49167").into(),
),
value: 1234u64.into(),
value: U256::from(1234u64),
input: Default::default(),
}),
Signature {
@ -282,7 +282,7 @@ mod tests {
to: TransactionKind::Call(
hex!("61815774383099e24810ab832a5b2a5425c154d5").into(),
),
value: 3000000000000000000u64.into(),
value: U256::from(3000000000000000000u64),
input: Default::default(),
access_list: Default::default(),
}),
@ -307,7 +307,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 1000000000000000u64.into(),
value: U256::from(1000000000000000u64),
input: Default::default(),
}),
Signature {
@ -331,7 +331,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 693361000000000u64.into(),
value: U256::from(693361000000000u64),
input: Default::default(),
}),
Signature {
@ -355,7 +355,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 1000000000000000u64.into(),
value: U256::from(1000000000000000u64),
input: Default::default(),
}),
Signature {
@ -400,7 +400,7 @@ mod tests {
to: TransactionKind::Call(
hex!("cf7f9e66af820a19257a2108375b180b0ec49167").into(),
),
value: 1234u64.into(),
value: U256::from(1234u64),
input: Default::default(),
}),
Signature {
@ -425,7 +425,7 @@ mod tests {
to: TransactionKind::Call(
hex!("61815774383099e24810ab832a5b2a5425c154d5").into(),
),
value: 3000000000000000000u64.into(),
value: U256::from(3000000000000000000u64),
input: Default::default(),
access_list: Default::default(),
}),
@ -450,7 +450,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 1000000000000000u64.into(),
value: U256::from(1000000000000000u64),
input: Default::default(),
}),
Signature {
@ -474,7 +474,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 693361000000000u64.into(),
value: U256::from(693361000000000u64),
input: Default::default(),
}),
Signature {
@ -498,7 +498,7 @@ mod tests {
to: TransactionKind::Call(
hex!("d3e8763675e4c425df46cc3b5c0f6cbdac396046").into(),
),
value: 1000000000000000u64.into(),
value: U256::from(1000000000000000u64),
input: Default::default(),
}),
Signature {

2
crates/net/network/tests/it/requests.rs
View File

@ -26,7 +26,7 @@ pub fn rng_transaction(rng: &mut impl rand::RngCore) -> TransactionSigned {
gas_price: rng.gen(),
gas_limit: rng.gen(),
to: TransactionKind::Create,
value: rng.gen::<u128>().into(),
value: U256::from(rng.gen::<u128>()),
input: Bytes::from(vec![1, 2]),
access_list: Default::default(),
});

2
crates/node-builder/Cargo.toml
View File

@ -29,7 +29,7 @@ reth-transaction-pool.workspace = true
reth-tasks.workspace = true
reth-tracing.workspace = true
reth-interfaces.workspace = true
reth-snapshot.workspace = true
reth-static-file.workspace = true
reth-prune.workspace = true
reth-stages.workspace = true
reth-config.workspace = true

26
crates/node-builder/src/builder.rs
View File

@ -350,22 +350,22 @@ where
info!(target: "reth::cli", "Database opened");
let mut provider_factory =
ProviderFactory::new(database.clone(), Arc::clone(&config.chain));
// configure snapshotter
let snapshotter = reth_snapshot::Snapshotter::new(
provider_factory.clone(),
data_dir.snapshots_path(),
config.chain.snapshot_block_interval,
let provider_factory = ProviderFactory::new(
database.clone(),
Arc::clone(&config.chain),
data_dir.static_files_path(),
)?;
provider_factory = provider_factory
.with_snapshots(data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver())?;
// configure static_file_producer
let static_file_producer = reth_static_file::StaticFileProducer::new(
provider_factory.clone(),
provider_factory.static_file_provider(),
config.prune_config()?.unwrap_or_default().segments,
);
debug!(target: "reth::cli", chain=%config.chain.chain, genesis=?config.chain.genesis_hash(), "Initializing genesis");
let genesis_hash = init_genesis(database.clone(), config.chain.clone())?;
let genesis_hash = init_genesis(provider_factory.clone())?;
info!(target: "reth::cli", "{}", config.chain.display_hardforks());
@ -471,6 +471,7 @@ where
sync_metrics_tx,
prune_config.clone(),
max_block,
static_file_producer,
evm_config,
)
.await?;
@ -492,6 +493,7 @@ where
sync_metrics_tx,
prune_config.clone(),
max_block,
static_file_producer,
evm_config,
)
.await?;
@ -508,7 +510,7 @@ where
let mut pruner = PrunerBuilder::new(prune_config.clone())
.max_reorg_depth(tree_config.max_reorg_depth() as usize)
.prune_delete_limit(config.chain.prune_delete_limit)
.build(provider_factory, snapshotter.highest_snapshot_receiver());
.build(provider_factory);
let events = pruner.events();
hooks.add(PruneHook::new(pruner, Box::new(executor.clone())));

3
crates/node-core/Cargo.toml
View File

@ -44,7 +44,7 @@ reth-stages.workspace = true
reth-prune.workspace = true
reth-blockchain-tree.workspace = true
revm-inspectors.workspace = true
reth-snapshot.workspace = true
reth-static-file.workspace = true
reth-eth-wire.workspace = true
# `optimism` feature
@ -69,6 +69,7 @@ thiserror.workspace = true
const-str = "0.5.6"
rand.workspace = true
pin-project.workspace = true
derive_more.workspace = true
# io
dirs-next = "2.0.0"

7
crates/node-core/src/args/stage_args.rs
View File

@ -1,9 +1,10 @@
//! Shared arguments related to stages
use derive_more::Display;
/// Represents a specific stage within the data pipeline.
///
/// Different stages within the pipeline have dedicated functionalities and operations.
#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, clap::ValueEnum)]
#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, clap::ValueEnum, Display)]
pub enum StageEnum {
/// The headers stage within the pipeline.
///
@ -49,8 +50,4 @@ pub enum StageEnum {
///
/// Manages historical data related to storage.
StorageHistory,
/// The total difficulty stage within the pipeline.
///
/// Handles computations and data related to total difficulty.
TotalDifficulty,
}

6
crates/node-core/src/dirs.rs
View File

@ -282,9 +282,9 @@ impl<D> ChainPath<D> {
self.0.join("db").into()
}
/// Returns the path to the snapshots directory for this chain.
pub fn snapshots_path(&self) -> PathBuf {
self.0.join("snapshots").into()
/// Returns the path to the static_files directory for this chain.
pub fn static_files_path(&self) -> PathBuf {
self.0.join("static_files").into()
}
/// Returns the path to the reth p2p secret key for this chain.

26
crates/node-core/src/events/node.rs
View File

@ -14,6 +14,7 @@ use reth_primitives::{
};
use reth_prune::PrunerEvent;
use reth_stages::{ExecOutput, PipelineEvent};
use reth_static_file::StaticFileProducerEvent;
use std::{
fmt::{Display, Formatter},
future::Future,
@ -233,11 +234,25 @@ impl<DB> NodeState<DB> {
fn handle_pruner_event(&self, event: PrunerEvent) {
match event {
PrunerEvent::Started { tip_block_number } => {
info!(tip_block_number, "Pruner started");
}
PrunerEvent::Finished { tip_block_number, elapsed, stats } => {
info!(tip_block_number, ?elapsed, ?stats, "Pruner finished");
}
}
}
fn handle_static_file_producer_event(&self, event: StaticFileProducerEvent) {
match event {
StaticFileProducerEvent::Started { targets } => {
info!(?targets, "Static File Producer started");
}
StaticFileProducerEvent::Finished { targets, elapsed } => {
info!(?targets, ?elapsed, "Static File Producer finished");
}
}
}
}
impl<DB: DatabaseMetadata> NodeState<DB> {
@ -282,6 +297,8 @@ pub enum NodeEvent {
ConsensusLayerHealth(ConsensusLayerHealthEvent),
/// A pruner event
Pruner(PrunerEvent),
/// A static_file_producer event
StaticFileProducer(StaticFileProducerEvent),
}
impl From<NetworkEvent> for NodeEvent {
@ -314,6 +331,12 @@ impl From<PrunerEvent> for NodeEvent {
}
}
impl From<StaticFileProducerEvent> for NodeEvent {
fn from(event: StaticFileProducerEvent) -> Self {
NodeEvent::StaticFileProducer(event)
}
}
/// Displays relevant information to the user from components of the node, and periodically
/// displays the high-level status of the node.
pub async fn handle_events<E, DB>(
@ -430,6 +453,9 @@ where
NodeEvent::Pruner(event) => {
this.state.handle_pruner_event(event);
}
NodeEvent::StaticFileProducer(event) => {
this.state.handle_static_file_producer_event(event);
}
}
}

113
crates/node-core/src/init.rs
View File

@ -1,19 +1,20 @@
//! Reth genesis initialization utility functions.
use reth_db::{
cursor::DbCursorRO,
database::Database,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_interfaces::{db::DatabaseError, provider::ProviderResult};
use reth_primitives::{
stage::StageId, Account, Bytecode, ChainSpec, Receipts, StorageEntry, B256, U256,
stage::StageId, Account, Bytecode, ChainSpec, Receipts, StaticFileSegment, StorageEntry, B256,
U256,
};
use reth_provider::{
bundle_state::{BundleStateInit, RevertsInit},
BundleStateWithReceipts, DatabaseProviderRW, HashingWriter, HistoryWriter, OriginalValuesKnown,
ProviderError, ProviderFactory,
providers::{StaticFileProvider, StaticFileWriter},
BlockHashReader, BundleStateWithReceipts, ChainSpecProvider, DatabaseProviderRW, HashingWriter,
HistoryWriter, OriginalValuesKnown, ProviderError, ProviderFactory,
};
use std::{
collections::{BTreeMap, HashMap},
@ -46,49 +47,49 @@ impl From<DatabaseError> for InitDatabaseError {
}
/// Write the genesis block if it has not already been written
pub fn init_genesis<DB: Database>(
db: DB,
chain: Arc<ChainSpec>,
) -> Result<B256, InitDatabaseError> {
let genesis = chain.genesis();
pub fn init_genesis<DB: Database>(factory: ProviderFactory<DB>) -> Result<B256, InitDatabaseError> {
let chain = factory.chain_spec();
let genesis = chain.genesis();
let hash = chain.genesis_hash();
let tx = db.tx()?;
if let Some((_, db_hash)) = tx.cursor_read::<tables::CanonicalHeaders>()?.first()? {
if db_hash == hash {
debug!("Genesis already written, skipping.");
return Ok(hash)
}
// Check if we already have the genesis header or if we have the wrong one.
match factory.block_hash(0) {
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {}
Ok(Some(block_hash)) => {
if block_hash == hash {
debug!("Genesis already written, skipping.");
return Ok(hash)
}
return Err(InitDatabaseError::GenesisHashMismatch {
chainspec_hash: hash,
database_hash: db_hash,
})
return Err(InitDatabaseError::GenesisHashMismatch {
chainspec_hash: hash,
database_hash: block_hash,
})
}
Err(e) => return Err(dbg!(e).into()),
}
drop(tx);
debug!("Writing genesis block.");
// use transaction to insert genesis header
let factory = ProviderFactory::new(&db, chain.clone());
let provider_rw = factory.provider_rw()?;
insert_genesis_hashes(&provider_rw, genesis)?;
insert_genesis_history(&provider_rw, genesis)?;
provider_rw.commit()?;
// Insert header
let tx = db.tx_mut()?;
insert_genesis_header::<DB>(&tx, chain.clone())?;
let tx = provider_rw.into_tx();
insert_genesis_header::<DB>(&tx, factory.static_file_provider(), chain.clone())?;
insert_genesis_state::<DB>(&tx, genesis)?;
// insert sync stage
for stage in StageId::ALL.iter() {
tx.put::<tables::SyncStage>(stage.to_string(), Default::default())?;
tx.put::<tables::StageCheckpoints>(stage.to_string(), Default::default())?;
}
tx.commit()?;
Ok(hash)
}
@ -153,14 +154,14 @@ pub fn insert_genesis_state<DB: Database>(
0,
);
bundle.write_to_db(tx, OriginalValuesKnown::Yes)?;
bundle.write_to_storage(tx, None, OriginalValuesKnown::Yes)?;
Ok(())
}
/// Inserts hashes for the genesis state.
pub fn insert_genesis_hashes<DB: Database>(
provider: &DatabaseProviderRW<&DB>,
provider: &DatabaseProviderRW<DB>,
genesis: &reth_primitives::Genesis,
) -> ProviderResult<()> {
// insert and hash accounts to hashing table
@ -187,7 +188,7 @@ pub fn insert_genesis_hashes<DB: Database>(
/// Inserts history indices for genesis accounts and storage.
pub fn insert_genesis_history<DB: Database>(
provider: &DatabaseProviderRW<&DB>,
provider: &DatabaseProviderRW<DB>,
genesis: &reth_primitives::Genesis,
) -> ProviderResult<()> {
let account_transitions =
@ -208,15 +209,24 @@ pub fn insert_genesis_history<DB: Database>(
/// Inserts header for the genesis state.
pub fn insert_genesis_header<DB: Database>(
tx: &<DB as Database>::TXMut,
static_file_provider: StaticFileProvider,
chain: Arc<ChainSpec>,
) -> ProviderResult<()> {
let (header, block_hash) = chain.sealed_genesis_header().split();
tx.put::<tables::CanonicalHeaders>(0, block_hash)?;
match static_file_provider.block_hash(0) {
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {
let (difficulty, hash) = (header.difficulty, block_hash);
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
writer.append_header(header, difficulty, hash)?;
writer.commit()?;
}
Ok(Some(_)) => {}
Err(e) => return Err(e),
}
tx.put::<tables::HeaderNumbers>(block_hash, 0)?;
tx.put::<tables::BlockBodyIndices>(0, Default::default())?;
tx.put::<tables::HeaderTD>(0, header.difficulty.into())?;
tx.put::<tables::Headers>(0, header)?;
Ok(())
}
@ -226,15 +236,16 @@ mod tests {
use super::*;
use reth_db::{
cursor::DbCursorRO,
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
table::{Table, TableRow},
test_utils::create_test_rw_db,
DatabaseEnv,
};
use reth_primitives::{
Address, Chain, ForkTimestamps, Genesis, GenesisAccount, IntegerList, GOERLI,
GOERLI_GENESIS_HASH, MAINNET, MAINNET_GENESIS_HASH, SEPOLIA, SEPOLIA_GENESIS_HASH,
};
use reth_provider::test_utils::create_test_provider_factory_with_chain_spec;
fn collect_table_entries<DB, T>(
tx: &<DB as Database>::TX,
@ -248,8 +259,8 @@ mod tests {
#[test]
fn success_init_genesis_mainnet() {
let db = create_test_rw_db();
let genesis_hash = init_genesis(db, MAINNET.clone()).unwrap();
let genesis_hash =
init_genesis(create_test_provider_factory_with_chain_spec(MAINNET.clone())).unwrap();
// actual, expected
assert_eq!(genesis_hash, MAINNET_GENESIS_HASH);
@ -257,8 +268,8 @@ mod tests {
#[test]
fn success_init_genesis_goerli() {
let db = create_test_rw_db();
let genesis_hash = init_genesis(db, GOERLI.clone()).unwrap();
let genesis_hash =
init_genesis(create_test_provider_factory_with_chain_spec(GOERLI.clone())).unwrap();
// actual, expected
assert_eq!(genesis_hash, GOERLI_GENESIS_HASH);
@ -266,8 +277,8 @@ mod tests {
#[test]
fn success_init_genesis_sepolia() {
let db = create_test_rw_db();
let genesis_hash = init_genesis(db, SEPOLIA.clone()).unwrap();
let genesis_hash =
init_genesis(create_test_provider_factory_with_chain_spec(SEPOLIA.clone())).unwrap();
// actual, expected
assert_eq!(genesis_hash, SEPOLIA_GENESIS_HASH);
@ -275,11 +286,19 @@ mod tests {
#[test]
fn fail_init_inconsistent_db() {
let db = create_test_rw_db();
init_genesis(db.clone(), SEPOLIA.clone()).unwrap();
let factory = create_test_provider_factory_with_chain_spec(SEPOLIA.clone());
let static_file_provider = factory.static_file_provider();
init_genesis(factory.clone()).unwrap();
// Try to init db with a different genesis block
let genesis_hash = init_genesis(db, MAINNET.clone());
let genesis_hash = init_genesis(
ProviderFactory::new(
factory.into_db(),
MAINNET.clone(),
static_file_provider.path().into(),
)
.unwrap(),
);
assert_eq!(
genesis_hash.unwrap_err(),
@ -321,13 +340,15 @@ mod tests {
..Default::default()
});
let db = create_test_rw_db();
init_genesis(db.clone(), chain_spec).unwrap();
let factory = create_test_provider_factory_with_chain_spec(chain_spec);
init_genesis(factory.clone()).unwrap();
let tx = db.tx().expect("failed to init tx");
let provider = factory.provider().unwrap();
let tx = provider.tx_ref();
assert_eq!(
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountHistory>(&tx)
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx)
.expect("failed to collect"),
vec![
(ShardedKey::new(address_with_balance, u64::MAX), IntegerList::new([0]).unwrap()),
@ -336,7 +357,7 @@ mod tests {
);
assert_eq!(
collect_table_entries::<Arc<DatabaseEnv>, tables::StorageHistory>(&tx)
collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx)
.expect("failed to collect"),
vec![(
StorageShardedKey::new(address_with_storage, storage_key, u64::MAX),

16
crates/node-core/src/node_config.rs
View File

@ -57,10 +57,11 @@ use reth_stages::{
stages::{
AccountHashingStage, ExecutionStage, ExecutionStageThresholds, IndexAccountHistoryStage,
IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, StorageHashingStage,
TotalDifficultyStage, TransactionLookupStage,
TransactionLookupStage,
},
MetricEvent,
};
use reth_static_file::StaticFileProducer;
use reth_tasks::TaskExecutor;
use reth_transaction_pool::{
blobstore::{DiskFileBlobStore, DiskFileBlobStoreConfig},
@ -546,6 +547,7 @@ impl NodeConfig {
metrics_tx: reth_stages::MetricEventsSender,
prune_config: Option<PruneConfig>,
max_block: Option<BlockNumber>,
static_file_producer: StaticFileProducer<DB>,
evm_config: EvmConfig,
) -> eyre::Result<Pipeline<DB>>
where
@ -573,6 +575,7 @@ impl NodeConfig {
self.debug.continuous,
metrics_tx,
prune_config,
static_file_producer,
evm_config,
)
.await?;
@ -794,6 +797,7 @@ impl NodeConfig {
continuous: bool,
metrics_tx: reth_stages::MetricEventsSender,
prune_config: Option<PruneConfig>,
static_file_producer: StaticFileProducer<DB>,
evm_config: EvmConfig,
) -> eyre::Result<Pipeline<DB>>
where
@ -843,11 +847,7 @@ impl NodeConfig {
header_downloader,
body_downloader,
factory.clone(),
)
.set(
TotalDifficultyStage::new(consensus)
.with_commit_threshold(stage_config.total_difficulty.commit_threshold),
)
)?
.set(SenderRecoveryStage {
commit_threshold: stage_config.sender_recovery.commit_threshold,
})
@ -879,7 +879,7 @@ impl NodeConfig {
))
.set(MerkleStage::new_execution(stage_config.merkle.clean_threshold))
.set(TransactionLookupStage::new(
stage_config.transaction_lookup.commit_threshold,
stage_config.transaction_lookup.chunk_size,
prune_modes.transaction_lookup,
))
.set(IndexAccountHistoryStage::new(
@ -891,7 +891,7 @@ impl NodeConfig {
prune_modes.storage_history,
)),
)
.build(provider_factory);
.build(provider_factory, static_file_producer);
Ok(pipeline)
}

11
crates/primitives/Cargo.toml
View File

@ -41,7 +41,7 @@ tracing.workspace = true
bytes.workspace = true
byteorder = "1"
clap = { workspace = true, features = ["derive"], optional = true }
derive_more = "0.99"
derive_more.workspace = true
itertools.workspace = true
modular-bitfield = "0.11.2"
num_enum = "0.7"
@ -50,10 +50,10 @@ rayon.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2 = "0.10.7"
sucds = "~0.6"
tempfile.workspace = true
thiserror.workspace = true
zstd = { version = "0.12", features = ["experimental"] }
roaring = "0.10.2"
cfg-if = "1.0.0"
# `test-utils` feature
@ -83,6 +83,9 @@ triehash = "0.8"
hash-db = "~0.15"
plain_hasher = "0.2"
sucds = "0.8.1"
anyhow = "1.0.75"
# necessary so we don't hit a "undeclared 'std'":
# https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198
criterion.workspace = true
@ -126,3 +129,7 @@ harness = false
name = "trie_root"
required-features = ["arbitrary", "test-utils"]
harness = false
[[bench]]
name = "integer_list"
harness = false

250
crates/primitives/benches/integer_list.rs Normal file
View File

@ -0,0 +1,250 @@
#![allow(missing_docs)]
use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
use rand::prelude::*;
pub fn new_pre_sorted(c: &mut Criterion) {
let mut group = c.benchmark_group("new_pre_sorted");
for delta in [1, 100, 1000, 10000] {
let integers_usize = generate_integers(2000, delta);
assert_eq!(integers_usize.len(), 2000);
let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::<Vec<_>>();
assert_eq!(integers_u64.len(), 2000);
group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| {
b.iter(|| elias_fano::IntegerList::new_pre_sorted(black_box(&integers_usize)));
});
group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| {
b.iter(|| reth_primitives::IntegerList::new_pre_sorted(black_box(&integers_u64)));
});
}
}
pub fn rank_select(c: &mut Criterion) {
let mut group = c.benchmark_group("rank + select");
for delta in [1, 100, 1000, 10000] {
let integers_usize = generate_integers(2000, delta);
assert_eq!(integers_usize.len(), 2000);
let integers_u64 = integers_usize.iter().map(|v| *v as u64).collect::<Vec<_>>();
assert_eq!(integers_u64.len(), 2000);
group.bench_function(BenchmarkId::new("Elias-Fano", delta), |b| {
b.iter_batched(
|| {
let (index, element) =
integers_usize.iter().enumerate().choose(&mut thread_rng()).unwrap();
(elias_fano::IntegerList::new_pre_sorted(&integers_usize).0, index, *element)
},
|(list, index, element)| {
let list = list.enable_rank();
list.rank(element);
list.select(index);
},
BatchSize::PerIteration,
);
});
group.bench_function(BenchmarkId::new("Roaring Bitmaps", delta), |b| {
b.iter_batched(
|| {
let (index, element) =
integers_u64.iter().enumerate().choose(&mut thread_rng()).unwrap();
(
reth_primitives::IntegerList::new_pre_sorted(&integers_u64),
index as u64,
*element,
)
},
|(list, index, element)| {
list.rank(element);
list.select(index);
},
BatchSize::PerIteration,
);
});
}
}
fn generate_integers(n: usize, delta: usize) -> Vec<usize> {
(0..n).fold(Vec::new(), |mut vec, _| {
vec.push(vec.last().map_or(0, |last| {
last + thread_rng().gen_range(delta - delta / 2..=delta + delta / 2)
}));
vec
})
}
criterion_group! {
name = benches;
config = Criterion::default();
targets = new_pre_sorted, rank_select
}
criterion_main!(benches);
/// Implementation from https://github.com/paradigmxyz/reth/blob/cda5d4e7c53ccc898b7725eb5d3b46c35e4da7f8/crates/primitives/src/integer_list.rs
/// adapted to work with `sucds = "0.8.1"`
#[allow(unused, unreachable_pub)]
mod elias_fano {
use std::{fmt, ops::Deref};
use sucds::{mii_sequences::EliasFano, Serializable};
#[derive(Clone, PartialEq, Eq, Default)]
pub struct IntegerList(pub EliasFano);
impl Deref for IntegerList {
type Target = EliasFano;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl fmt::Debug for IntegerList {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let vec: Vec<usize> = self.0.iter(0).collect();
write!(f, "IntegerList {:?}", vec)
}
}
impl IntegerList {
/// Creates an IntegerList from a list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
///
/// # Returns
///
/// Returns an error if the list is empty or not pre-sorted.
pub fn new<T: AsRef<[usize]>>(list: T) -> Result<Self, EliasFanoError> {
let mut builder = EliasFanoBuilder::new(
list.as_ref().iter().max().map_or(0, |max| max + 1),
list.as_ref().len(),
)?;
builder.extend(list.as_ref().iter().copied());
Ok(Self(builder.build()))
}
// Creates an IntegerList from a pre-sorted list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
///
/// # Panics
///
/// Panics if the list is empty or not pre-sorted.
pub fn new_pre_sorted<T: AsRef<[usize]>>(list: T) -> Self {
Self::new(list).expect("IntegerList must be pre-sorted and non-empty.")
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_bytes(&self) -> Vec<u8> {
let mut vec = Vec::with_capacity(self.0.size_in_bytes());
self.0.serialize_into(&mut vec).expect("not able to encode integer list.");
vec
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_mut_bytes<B: bytes::BufMut>(&self, buf: &mut B) {
let len = self.0.size_in_bytes();
let mut vec = Vec::with_capacity(len);
self.0.serialize_into(&mut vec).unwrap();
buf.put_slice(vec.as_slice());
}
/// Deserializes a sequence of bytes into a proper [`IntegerList`].
pub fn from_bytes(data: &[u8]) -> Result<Self, EliasFanoError> {
Ok(Self(
EliasFano::deserialize_from(data).map_err(|_| EliasFanoError::FailedDeserialize)?,
))
}
}
macro_rules! impl_uint {
($($w:tt),+) => {
$(
impl From<Vec<$w>> for IntegerList {
fn from(v: Vec<$w>) -> Self {
let v: Vec<usize> = v.iter().map(|v| *v as usize).collect();
Self::new(v.as_slice()).expect("could not create list.")
}
}
)+
};
}
impl_uint!(usize, u64, u32, u8, u16);
impl Serialize for IntegerList {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let vec = self.0.iter(0).collect::<Vec<usize>>();
let mut seq = serializer.serialize_seq(Some(self.len()))?;
for e in vec {
seq.serialize_element(&e)?;
}
seq.end()
}
}
struct IntegerListVisitor;
impl<'de> Visitor<'de> for IntegerListVisitor {
type Value = IntegerList;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a usize array")
}
fn visit_seq<E>(self, mut seq: E) -> Result<Self::Value, E::Error>
where
E: SeqAccess<'de>,
{
let mut list = Vec::new();
while let Some(item) = seq.next_element()? {
list.push(item);
}
IntegerList::new(list)
.map_err(|_| serde::de::Error::invalid_value(Unexpected::Seq, &self))
}
}
impl<'de> Deserialize<'de> for IntegerList {
fn deserialize<D>(deserializer: D) -> Result<IntegerList, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_byte_buf(IntegerListVisitor)
}
}
#[cfg(any(test, feature = "arbitrary"))]
use arbitrary::{Arbitrary, Unstructured};
use serde::{
de::{SeqAccess, Unexpected, Visitor},
ser::SerializeSeq,
Deserialize, Deserializer, Serialize, Serializer,
};
use sucds::mii_sequences::EliasFanoBuilder;
#[cfg(any(test, feature = "arbitrary"))]
impl<'a> Arbitrary<'a> for IntegerList {
fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
let mut nums: Vec<usize> = Vec::arbitrary(u)?;
nums.sort();
Self::new(&nums).map_err(|_| arbitrary::Error::IncorrectFormat)
}
}
/// Primitives error type.
#[derive(Debug, thiserror::Error)]
pub enum EliasFanoError {
/// The provided input is invalid.
#[error(transparent)]
InvalidInput(#[from] anyhow::Error),
/// Failed to deserialize data into type.
#[error("failed to deserialize data into type")]
FailedDeserialize,
}
}

10
crates/primitives/src/chain/spec.rs
View File

@ -67,7 +67,6 @@ pub static MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 3500,
snapshot_block_interval: 500_000,
}
.into()
});
@ -111,7 +110,6 @@ pub static GOERLI: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 1700,
snapshot_block_interval: 1_000_000,
}
.into()
});
@ -159,7 +157,6 @@ pub static SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 1700,
snapshot_block_interval: 1_000_000,
}
.into()
});
@ -202,7 +199,6 @@ pub static HOLESKY: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
)),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: 1700,
snapshot_block_interval: 1_000_000,
}
.into()
});
@ -296,7 +292,6 @@ pub static BASE_SEPOLIA: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
.into(),
),
prune_delete_limit: 1700,
snapshot_block_interval: 1_000_000,
..Default::default()
}
.into()
@ -351,7 +346,6 @@ pub static BASE_MAINNET: Lazy<Arc<ChainSpec>> = Lazy::new(|| {
.into(),
),
prune_delete_limit: 1700,
snapshot_block_interval: 1_000_000,
..Default::default()
}
.into()
@ -502,9 +496,6 @@ pub struct ChainSpec {
/// data coming in.
#[serde(default)]
pub prune_delete_limit: usize,
/// The block interval for creating snapshots. Each snapshot will have that much blocks in it.
pub snapshot_block_interval: u64,
}
impl Default for ChainSpec {
@ -519,7 +510,6 @@ impl Default for ChainSpec {
deposit_contract: Default::default(),
base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()),
prune_delete_limit: MAINNET.prune_delete_limit,
snapshot_block_interval: Default::default(),
}
}
}

10
crates/primitives/src/header.rs
View File

@ -18,6 +18,7 @@ use proptest::prelude::*;
use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact};
use serde::{Deserialize, Serialize};
use std::{mem, ops::Deref};
/// Errors that can occur during header sanity checks.
#[derive(Debug, PartialEq)]
pub enum HeaderError {
@ -574,13 +575,14 @@ pub enum HeaderValidationError {
/// A [`Header`] that is sealed at a precalculated hash, use [`SealedHeader::unseal()`] if you want
/// to modify header.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[add_arbitrary_tests(rlp)]
#[main_codec(no_arbitrary)]
#[add_arbitrary_tests(rlp, compact)]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SealedHeader {
/// Locked Header fields.
header: Header,
/// Locked Header hash.
hash: BlockHash,
/// Locked Header fields.
header: Header,
}
impl SealedHeader {

65
crates/primitives/src/integer_list.rs
View File

@ -1,18 +1,19 @@
use bytes::BufMut;
use roaring::RoaringTreemap;
use serde::{
de::{SeqAccess, Unexpected, Visitor},
ser::SerializeSeq,
Deserialize, Deserializer, Serialize, Serializer,
};
use std::{fmt, ops::Deref};
use sucds::{EliasFano, Searial};
/// Uses EliasFano to hold a list of integers. It provides really good compression with the
/// Uses Roaring Bitmaps to hold a list of integers. It provides really good compression with the
/// capability to access its elements without decoding it.
#[derive(Clone, PartialEq, Eq, Default)]
pub struct IntegerList(pub EliasFano);
#[derive(Clone, PartialEq, Default)]
pub struct IntegerList(pub RoaringTreemap);
impl Deref for IntegerList {
type Target = EliasFano;
type Target = RoaringTreemap;
fn deref(&self) -> &Self::Target {
&self.0
@ -21,53 +22,54 @@ impl Deref for IntegerList {
impl fmt::Debug for IntegerList {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let vec: Vec<usize> = self.0.iter(0).collect();
let vec: Vec<u64> = self.0.iter().collect();
write!(f, "IntegerList {:?}", vec)
}
}
impl IntegerList {
/// Creates an IntegerList from a list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
/// Creates an IntegerList from a list of integers.
///
/// # Returns
///
/// Returns an error if the list is empty or not pre-sorted.
pub fn new<T: AsRef<[usize]>>(list: T) -> Result<Self, EliasFanoError> {
Ok(Self(EliasFano::from_ints(list.as_ref()).map_err(|_| EliasFanoError::InvalidInput)?))
pub fn new<T: AsRef<[u64]>>(list: T) -> Result<Self, RoaringBitmapError> {
Ok(Self(
RoaringTreemap::from_sorted_iter(list.as_ref().iter().copied())
.map_err(|_| RoaringBitmapError::InvalidInput)?,
))
}
// Creates an IntegerList from a pre-sorted list of integers. `usize` is safe to use since
/// [`sucds::EliasFano`] restricts its compilation to 64bits.
// Creates an IntegerList from a pre-sorted list of integers.
///
/// # Panics
///
/// Panics if the list is empty or not pre-sorted.
pub fn new_pre_sorted<T: AsRef<[usize]>>(list: T) -> Self {
pub fn new_pre_sorted<T: AsRef<[u64]>>(list: T) -> Self {
Self(
EliasFano::from_ints(list.as_ref())
.expect("IntegerList must be pre-sorted and non-empty."),
RoaringTreemap::from_sorted_iter(list.as_ref().iter().copied())
.expect("IntegerList must be pre-sorted and non-empty"),
)
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_bytes(&self) -> Vec<u8> {
let mut vec = Vec::with_capacity(self.0.size_in_bytes());
self.0.serialize_into(&mut vec).expect("not able to encode integer list.");
let mut vec = Vec::with_capacity(self.0.serialized_size());
self.0.serialize_into(&mut vec).expect("not able to encode IntegerList");
vec
}
/// Serializes a [`IntegerList`] into a sequence of bytes.
pub fn to_mut_bytes<B: bytes::BufMut>(&self, buf: &mut B) {
let len = self.0.size_in_bytes();
let mut vec = Vec::with_capacity(len);
self.0.serialize_into(&mut vec).unwrap();
buf.put_slice(vec.as_slice());
self.0.serialize_into(buf.writer()).unwrap();
}
/// Deserializes a sequence of bytes into a proper [`IntegerList`].
pub fn from_bytes(data: &[u8]) -> Result<Self, EliasFanoError> {
Ok(Self(EliasFano::deserialize_from(data).map_err(|_| EliasFanoError::FailedDeserialize)?))
pub fn from_bytes(data: &[u8]) -> Result<Self, RoaringBitmapError> {
Ok(Self(
RoaringTreemap::deserialize_from(data)
.map_err(|_| RoaringBitmapError::FailedToDeserialize)?,
))
}
}
@ -76,8 +78,7 @@ macro_rules! impl_uint {
$(
impl From<Vec<$w>> for IntegerList {
fn from(v: Vec<$w>) -> Self {
let v: Vec<usize> = v.iter().map(|v| *v as usize).collect();
Self(EliasFano::from_ints(v.as_slice()).expect("could not create list."))
Self::new_pre_sorted(v.iter().map(|v| *v as u64).collect::<Vec<_>>())
}
}
)+
@ -91,8 +92,8 @@ impl Serialize for IntegerList {
where
S: Serializer,
{
let vec = self.0.iter(0).collect::<Vec<usize>>();
let mut seq = serializer.serialize_seq(Some(self.len()))?;
let vec = self.0.iter().collect::<Vec<u64>>();
let mut seq = serializer.serialize_seq(Some(self.len() as usize))?;
for e in vec {
seq.serialize_element(&e)?;
}
@ -136,21 +137,21 @@ use arbitrary::{Arbitrary, Unstructured};
#[cfg(any(test, feature = "arbitrary"))]
impl<'a> Arbitrary<'a> for IntegerList {
fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, arbitrary::Error> {
let mut nums: Vec<usize> = Vec::arbitrary(u)?;
let mut nums: Vec<u64> = Vec::arbitrary(u)?;
nums.sort();
Ok(Self(EliasFano::from_ints(&nums).map_err(|_| arbitrary::Error::IncorrectFormat)?))
Self::new(nums).map_err(|_| arbitrary::Error::IncorrectFormat)
}
}
/// Primitives error type.
#[derive(Debug, thiserror::Error)]
pub enum EliasFanoError {
pub enum RoaringBitmapError {
/// The provided input is invalid.
#[error("the provided input is invalid")]
InvalidInput,
/// Failed to deserialize data into type.
#[error("failed to deserialize data into type")]
FailedDeserialize,
FailedToDeserialize,
}
#[cfg(test)]
@ -161,7 +162,7 @@ mod tests {
fn test_integer_list() {
let original_list = [1, 2, 3];
let ef_list = IntegerList::new(original_list).unwrap();
assert_eq!(ef_list.iter(0).collect::<Vec<usize>>(), original_list);
assert_eq!(ef_list.iter().collect::<Vec<_>>(), original_list);
}
#[test]

10
crates/primitives/src/lib.rs
View File

@ -37,8 +37,8 @@ mod receipt;
/// Helpers for working with revm
pub mod revm;
pub mod serde_helper;
pub mod snapshot;
pub mod stage;
pub mod static_file;
mod storage;
/// Helpers for working with transactions
pub mod transaction;
@ -73,11 +73,11 @@ pub use net::{
};
pub use peer::{PeerId, WithPeerId};
pub use prune::{
PruneCheckpoint, PruneMode, PruneModes, PruneProgress, PruneSegment, PruneSegmentError,
ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE,
PruneCheckpoint, PruneMode, PruneModes, PruneProgress, PrunePurpose, PruneSegment,
PruneSegmentError, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE,
};
pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts};
pub use snapshot::SnapshotSegment;
pub use static_file::StaticFileSegment;
pub use storage::StorageEntry;
#[cfg(feature = "c-kzg")]
@ -92,7 +92,7 @@ pub use transaction::{
AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction,
InvalidTransactionError, Signature, Transaction, TransactionKind, TransactionMeta,
TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, TxEip1559, TxEip2930,
TxEip4844, TxHashOrNumber, TxLegacy, TxType, TxValue, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID,
TxEip4844, TxHashOrNumber, TxLegacy, TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID,
EIP4844_TX_TYPE_ID, LEGACY_TX_TYPE_ID,
};
pub use withdrawal::{Withdrawal, Withdrawals};

3
crates/primitives/src/proofs.rs
View File

@ -5,9 +5,8 @@ use crate::{
keccak256,
trie::{HashBuilder, Nibbles, TrieAccount},
Address, Header, Receipt, ReceiptWithBloom, ReceiptWithBloomRef, TransactionSigned, Withdrawal,
B256,
B256, U256,
};
use alloy_primitives::U256;
use alloy_rlp::Encodable;
use bytes::{BufMut, BytesMut};
use itertools::Itertools;

6
crates/primitives/src/prune/mod.rs
View File

@ -6,7 +6,7 @@ mod target;
use crate::{Address, BlockNumber};
pub use checkpoint::PruneCheckpoint;
pub use mode::PruneMode;
pub use segment::{PruneSegment, PruneSegmentError};
pub use segment::{PrunePurpose, PruneSegment, PruneSegmentError};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE};
@ -53,7 +53,7 @@ impl ReceiptsLogPruneConfig {
// Reminder, that we increment because the [`BlockNumber`] key of the new map should be
// viewed as `PruneMode::Before(block)`
let block = (pruned_block + 1).max(
mode.prune_target_block(tip, PruneSegment::ContractLogs)?
mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)?
.map(|(block, _)| block)
.unwrap_or_default() +
1,
@ -76,7 +76,7 @@ impl ReceiptsLogPruneConfig {
for (_, mode) in self.0.iter() {
if let PruneMode::Distance(_) = mode {
if let Some((block, _)) =
mode.prune_target_block(tip, PruneSegment::ContractLogs)?
mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)?
{
lowest = Some(lowest.unwrap_or(u64::MAX).min(block));
}

Some files were not shown because too many files have changed in this diff Show More