feat: add thorough error message to state root error (#7607)

crates/consensus/consensus/src/lib.rs

@@ -259,6 +259,13 @@ pub enum ConsensusError {
     HeaderValidationError(#[from] HeaderValidationError),
 }
 
+impl ConsensusError {
+    /// Returns `true` if the error is a state root error.
+    pub fn is_state_root_error(&self) -> bool {
+        matches!(self, ConsensusError::BodyStateRootDiff(_))
+    }
+}
+
 /// `HeaderConsensusError` combines a `ConsensusError` with the `SealedHeader` it relates to.
 #[derive(thiserror::Error, Debug)]
 #[error("Consensus error: {0}, Invalid header: {1:?}")]

crates/interfaces/src/blockchain_tree/error.rs

@@ -243,6 +243,36 @@ impl InsertBlockErrorKind {
         matches!(self, InsertBlockErrorKind::Consensus(_))
     }
 
+    /// Returns true if this error is a state root error
+    pub fn is_state_root_error(&self) -> bool {
+        // we need to get the state root errors inside of the different variant branches
+        match self {
+            InsertBlockErrorKind::Execution(err) => {
+                matches!(
+                    err,
+                    BlockExecutionError::Validation(BlockValidationError::StateRoot { .. })
+                )
+            }
+            InsertBlockErrorKind::Canonical(err) => {
+                matches!(
+                    err,
+                    CanonicalError::Validation(BlockValidationError::StateRoot { .. }) |
+                        CanonicalError::Provider(
+                            ProviderError::StateRootMismatch(_) |
+                                ProviderError::UnwindStateRootMismatch(_)
+                        )
+                )
+            }
+            InsertBlockErrorKind::Provider(err) => {
+                matches!(
+                    err,
+                    ProviderError::StateRootMismatch(_) | ProviderError::UnwindStateRootMismatch(_)
+                )
+            }
+            _ => false,
+        }
+    }
+
     /// Returns true if the error is caused by an invalid block
     ///
     /// This is intended to be used to determine if the block should be marked as invalid.

crates/interfaces/src/executor.rs

@@ -153,4 +153,9 @@ impl BlockExecutionError {
     pub fn is_fatal(&self) -> bool {
         matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. })
     }
+
+    /// Returns `true` if the error is a state root error.
+    pub fn is_state_root_error(&self) -> bool {
+        matches!(self, Self::Validation(BlockValidationError::StateRoot(_)))
+    }
 }

crates/stages-api/src/error.rs

@@ -20,6 +20,16 @@ pub enum BlockErrorKind {
     Execution(#[from] executor::BlockExecutionError),
 }
 
+impl BlockErrorKind {
+    /// Returns `true` if the error is a state root error.
+    pub fn is_state_root_error(&self) -> bool {
+        match self {
+            BlockErrorKind::Validation(err) => err.is_state_root_error(),
+            BlockErrorKind::Execution(err) => err.is_state_root_error(),
+        }
+    }
+}
+
 /// A stage execution error.
 #[derive(Error, Debug)]
 pub enum StageError {

crates/stages/src/stages/merkle.rs

@@ -21,6 +21,24 @@ use reth_trie::{IntermediateStateRootState, StateRoot, StateRootProgress};
 use std::fmt::Debug;
 use tracing::*;
 
+// TODO: automate the process outlined below so the user can just send in a debugging package
+/// The error message that we include in invalid state root errors to tell users what information
+/// they should include in a bug report, since true state root errors can be impossible to debug
+/// with just basic logs.
+pub const INVALID_STATE_ROOT_ERROR_MESSAGE: &str = r#"
+Invalid state root error on new payload!
+This is an error that likely requires a report to the reth team with additional information.
+Please include the following information in your report:
+ * This error message
+ * The state root of the block that was rejected
+ * The output of `reth db stats --checksum` from the database that was being used. This will take a long time to run!
+ * 50-100 lines of logs before and after the first occurrence of this log message. Please search your log output for the first observed occurrence of MAGIC_STATE_ROOT.
+ * The debug logs from __the same time period__. To find the default location for these logs, run:
+   `reth --help | grep -A 4 'log.file.directory'`
+
+Once you have this information, please submit a github issue at https://github.com/paradigmxyz/reth/issues/new
+"#;
+
 /// The default threshold (in number of blocks) for switching from incremental trie building
 /// of changes to whole rebuild.
 pub const MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD: u64 = 5_000;
@@ -196,7 +214,10 @@ impl<DB: Database> Stage<DB> for MerkleStage {
             let progress = StateRoot::from_tx(tx)
                 .with_intermediate_state(checkpoint.map(IntermediateStateRootState::from))
                 .root_with_progress()
-                .map_err(|e| StageError::Fatal(Box::new(e)))?;
+                .map_err(|e| {
+                    error!(target: "sync::stages::merkle", %e, ?current_block_number, ?to_block, "State root with progress failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}");
+                    StageError::Fatal(Box::new(e))
+                })?;
             match progress {
                 StateRootProgress::Progress(state, hashed_entries_walked, updates) => {
                     updates.flush(tx)?;
@@ -230,7 +251,10 @@ impl<DB: Database> Stage<DB> for MerkleStage {
             debug!(target: "sync::stages::merkle::exec", current = ?current_block_number, target = ?to_block, "Updating trie");
             let (root, updates) =
                 StateRoot::incremental_root_with_updates(provider.tx_ref(), range)
-                    .map_err(|e| StageError::Fatal(Box::new(e)))?;
+                    .map_err(|e| {
+                        error!(target: "sync::stages::merkle", %e, ?current_block_number, ?to_block, "Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}");
+                        StageError::Fatal(Box::new(e))
+                    })?;
             updates.flush(provider.tx_ref())?;
 
             let total_hashed_entries = (provider.count_entries::<tables::HashedAccounts>()? +
@@ -325,7 +349,7 @@ fn validate_state_root(
     if got == expected.state_root {
         Ok(())
     } else {
-        warn!(target: "sync::stages::merkle", ?target_block, ?got, ?expected, "Failed to verify block state root");
+        error!(target: "sync::stages::merkle", ?target_block, ?got, ?expected, "Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}");
         Err(StageError::Block {
             error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff(
                 GotExpected { got, expected: expected.state_root }.into(),