diff --git a/crates/consensus/consensus/src/lib.rs b/crates/consensus/consensus/src/lib.rs index b434272a4..2dee6b124 100644 --- a/crates/consensus/consensus/src/lib.rs +++ b/crates/consensus/consensus/src/lib.rs @@ -259,6 +259,13 @@ pub enum ConsensusError { HeaderValidationError(#[from] HeaderValidationError), } +impl ConsensusError { + /// Returns `true` if the error is a state root error. + pub fn is_state_root_error(&self) -> bool { + matches!(self, ConsensusError::BodyStateRootDiff(_)) + } +} + /// `HeaderConsensusError` combines a `ConsensusError` with the `SealedHeader` it relates to. #[derive(thiserror::Error, Debug)] #[error("Consensus error: {0}, Invalid header: {1:?}")] diff --git a/crates/interfaces/src/blockchain_tree/error.rs b/crates/interfaces/src/blockchain_tree/error.rs index 34e018835..44f1f50bc 100644 --- a/crates/interfaces/src/blockchain_tree/error.rs +++ b/crates/interfaces/src/blockchain_tree/error.rs @@ -243,6 +243,36 @@ impl InsertBlockErrorKind { matches!(self, InsertBlockErrorKind::Consensus(_)) } + /// Returns true if this error is a state root error + pub fn is_state_root_error(&self) -> bool { + // we need to get the state root errors inside of the different variant branches + match self { + InsertBlockErrorKind::Execution(err) => { + matches!( + err, + BlockExecutionError::Validation(BlockValidationError::StateRoot { .. }) + ) + } + InsertBlockErrorKind::Canonical(err) => { + matches!( + err, + CanonicalError::Validation(BlockValidationError::StateRoot { .. }) | + CanonicalError::Provider( + ProviderError::StateRootMismatch(_) | + ProviderError::UnwindStateRootMismatch(_) + ) + ) + } + InsertBlockErrorKind::Provider(err) => { + matches!( + err, + ProviderError::StateRootMismatch(_) | ProviderError::UnwindStateRootMismatch(_) + ) + } + _ => false, + } + } + /// Returns true if the error is caused by an invalid block /// /// This is intended to be used to determine if the block should be marked as invalid. diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 25e2f5710..e8f7f40b1 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -153,4 +153,9 @@ impl BlockExecutionError { pub fn is_fatal(&self) -> bool { matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) } + + /// Returns `true` if the error is a state root error. + pub fn is_state_root_error(&self) -> bool { + matches!(self, Self::Validation(BlockValidationError::StateRoot(_))) + } } diff --git a/crates/stages-api/src/error.rs b/crates/stages-api/src/error.rs index 3d7ae1d72..37fe2b3fd 100644 --- a/crates/stages-api/src/error.rs +++ b/crates/stages-api/src/error.rs @@ -20,6 +20,16 @@ pub enum BlockErrorKind { Execution(#[from] executor::BlockExecutionError), } +impl BlockErrorKind { + /// Returns `true` if the error is a state root error. + pub fn is_state_root_error(&self) -> bool { + match self { + BlockErrorKind::Validation(err) => err.is_state_root_error(), + BlockErrorKind::Execution(err) => err.is_state_root_error(), + } + } +} + /// A stage execution error. #[derive(Error, Debug)] pub enum StageError { diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index bfdb9782b..562cff183 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -21,6 +21,24 @@ use reth_trie::{IntermediateStateRootState, StateRoot, StateRootProgress}; use std::fmt::Debug; use tracing::*; +// TODO: automate the process outlined below so the user can just send in a debugging package +/// The error message that we include in invalid state root errors to tell users what information +/// they should include in a bug report, since true state root errors can be impossible to debug +/// with just basic logs. +pub const INVALID_STATE_ROOT_ERROR_MESSAGE: &str = r#" +Invalid state root error on new payload! +This is an error that likely requires a report to the reth team with additional information. +Please include the following information in your report: + * This error message + * The state root of the block that was rejected + * The output of `reth db stats --checksum` from the database that was being used. This will take a long time to run! + * 50-100 lines of logs before and after the first occurrence of this log message. Please search your log output for the first observed occurrence of MAGIC_STATE_ROOT. + * The debug logs from __the same time period__. To find the default location for these logs, run: + `reth --help | grep -A 4 'log.file.directory'` + +Once you have this information, please submit a github issue at https://github.com/paradigmxyz/reth/issues/new +"#; + /// The default threshold (in number of blocks) for switching from incremental trie building /// of changes to whole rebuild. pub const MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD: u64 = 5_000; @@ -196,7 +214,10 @@ impl Stage for MerkleStage { let progress = StateRoot::from_tx(tx) .with_intermediate_state(checkpoint.map(IntermediateStateRootState::from)) .root_with_progress() - .map_err(|e| StageError::Fatal(Box::new(e)))?; + .map_err(|e| { + error!(target: "sync::stages::merkle", %e, ?current_block_number, ?to_block, "State root with progress failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); + StageError::Fatal(Box::new(e)) + })?; match progress { StateRootProgress::Progress(state, hashed_entries_walked, updates) => { updates.flush(tx)?; @@ -230,7 +251,10 @@ impl Stage for MerkleStage { debug!(target: "sync::stages::merkle::exec", current = ?current_block_number, target = ?to_block, "Updating trie"); let (root, updates) = StateRoot::incremental_root_with_updates(provider.tx_ref(), range) - .map_err(|e| StageError::Fatal(Box::new(e)))?; + .map_err(|e| { + error!(target: "sync::stages::merkle", %e, ?current_block_number, ?to_block, "Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); + StageError::Fatal(Box::new(e)) + })?; updates.flush(provider.tx_ref())?; let total_hashed_entries = (provider.count_entries::()? + @@ -325,7 +349,7 @@ fn validate_state_root( if got == expected.state_root { Ok(()) } else { - warn!(target: "sync::stages::merkle", ?target_block, ?got, ?expected, "Failed to verify block state root"); + error!(target: "sync::stages::merkle", ?target_block, ?got, ?expected, "Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); Err(StageError::Block { error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff( GotExpected { got, expected: expected.state_root }.into(),