feat(engine): save original files for witness invalid block hook (#11132)

Cargo.lock

@@ -7457,6 +7457,7 @@ dependencies = [
  "reth-rpc-api",
  "reth-tracing",
  "reth-trie",
+ "serde",
  "serde_json",
 ]
 

crates/engine/invalid-block-hooks/Cargo.toml

@@ -17,7 +17,7 @@ reth-engine-primitives.workspace = true
 reth-evm.workspace = true
 reth-primitives.workspace = true
 reth-provider.workspace = true
-reth-revm.workspace = true
+reth-revm = { workspace = true, features = ["serde"] }
 reth-rpc-api = { workspace = true, features = ["client"] }
 reth-tracing.workspace = true
 reth-trie = { workspace = true, features = ["serde"] }
@@ -34,4 +34,5 @@ futures.workspace = true
 eyre.workspace = true
 jsonrpsee.workspace = true
 pretty_assertions = "1.4"
+serde.workspace = true
 serde_json.workspace = true

crates/engine/invalid-block-hooks/src/witness.rs

@@ -21,6 +21,7 @@ use reth_revm::{
 use reth_rpc_api::DebugApiClient;
 use reth_tracing::tracing::warn;
 use reth_trie::{updates::TrieUpdates, HashedPostState, HashedStorage};
+use serde::Serialize;
 
 /// Generates a witness for the given block and saves it to a file.
 #[derive(Debug)]
@@ -158,37 +159,10 @@ where
 
         // Write the witness to the output directory.
         let response = ExecutionWitness { state, keys: Some(state_preimages) };
-        File::create_new(self.output_directory.join(format!(
+        let re_executed_witness_path = self.save_file(
-            "{}_{}.json",
+            format!("{}_{}.witness.re_executed.json", block.number, block.hash()),
-            block.number,
+            &response,
-            block.hash()
+        )?;
-        )))?
-        .write_all(serde_json::to_string(&response)?.as_bytes())?;
-
-        // The bundle state after re-execution should match the original one.
-        if bundle_state != output.state {
-            let filename = format!("{}_{}.bundle_state.diff", block.number, block.hash());
-            let path = self.save_diff(filename, &bundle_state, &output.state)?;
-            warn!(target: "engine::invalid_block_hooks::witness", path = %path.display(), "Bundle state mismatch after re-execution");
-        }
-
-        // Calculate the state root and trie updates after re-execution. They should match
-        // the original ones.
-        let (state_root, trie_output) = state_provider.state_root_with_updates(hashed_state)?;
-        if let Some(trie_updates) = trie_updates {
-            if state_root != trie_updates.1 {
-                let filename = format!("{}_{}.state_root.diff", block.number, block.hash());
-                let path = self.save_diff(filename, &state_root, &trie_updates.1)?;
-                warn!(target: "engine::invalid_block_hooks::witness", path = %path.display(), "State root mismatch after re-execution");
-            }
-
-            if &trie_output != trie_updates.0 {
-                let filename = format!("{}_{}.trie_updates.diff", block.number, block.hash());
-                let path = self.save_diff(filename, &trie_output, trie_updates.0)?;
-                warn!(target: "engine::invalid_block_hooks::witness", path = %path.display(), "Trie updates mismatch after re-execution");
-            }
-        }
-
         if let Some(healthy_node_client) = &self.healthy_node_client {
             // Compare the witness against the healthy node.
             let healthy_node_witness = futures::executor::block_on(async move {
@@ -200,19 +174,74 @@ where
                 .await
             })?;
 
-            // Write the healthy node witness to the output directory.
+            let healthy_path = self.save_file(
-            File::create_new(self.output_directory.join(format!(
+                format!("{}_{}.witness.healthy.json", block.number, block.hash()),
-                "{}_{}.healthy_witness.json",
+                &healthy_node_witness,
-                block.number,
+            )?;
-                block.hash()
-            )))?
-            .write_all(serde_json::to_string(&healthy_node_witness)?.as_bytes())?;
 
             // If the witnesses are different, write the diff to the output directory.
             if response != healthy_node_witness {
-                let filename = format!("{}_{}.healthy_witness.diff", block.number, block.hash());
+                let filename = format!("{}_{}.witness.diff", block.number, block.hash());
-                let path = self.save_diff(filename, &response, &healthy_node_witness)?;
+                let diff_path = self.save_diff(filename, &response, &healthy_node_witness)?;
-                warn!(target: "engine::invalid_block_hooks::witness", path = %path.display(), "Witness mismatch against healthy node");
+                warn!(
+                    target: "engine::invalid_block_hooks::witness",
+                    diff_path = %diff_path.display(),
+                    re_executed_path = %re_executed_witness_path.display(),
+                    healthy_path = %healthy_path.display(),
+                    "Witness mismatch against healthy node"
+                );
+            }
+        }
+
+        // The bundle state after re-execution should match the original one.
+        if bundle_state != output.state {
+            let original_path = self.save_file(
+                format!("{}_{}.bundle_state.original.json", block.number, block.hash()),
+                &output.state,
+            )?;
+            let re_executed_path = self.save_file(
+                format!("{}_{}.bundle_state.re_executed.json", block.number, block.hash()),
+                &bundle_state,
+            )?;
+
+            let filename = format!("{}_{}.bundle_state.diff", block.number, block.hash());
+            let diff_path = self.save_diff(filename, &bundle_state, &output.state)?;
+
+            warn!(
+                target: "engine::invalid_block_hooks::witness",
+                diff_path = %diff_path.display(),
+                original_path = %original_path.display(),
+                re_executed_path = %re_executed_path.display(),
+                "Bundle state mismatch after re-execution"
+            );
+        }
+
+        // Calculate the state root and trie updates after re-execution. They should match
+        // the original ones.
+        let (state_root, trie_output) = state_provider.state_root_with_updates(hashed_state)?;
+        if let Some(trie_updates) = trie_updates {
+            if state_root != trie_updates.1 {
+                let filename = format!("{}_{}.state_root.diff", block.number, block.hash());
+                let diff_path = self.save_diff(filename, &state_root, &trie_updates.1)?;
+                warn!(target: "engine::invalid_block_hooks::witness", diff_path = %diff_path.display(), "State root mismatch after re-execution");
+            }
+
+            if &trie_output != trie_updates.0 {
+                // Trie updates are too big to diff, so we just save the original and re-executed
+                let original_path = self.save_file(
+                    format!("{}_{}.trie_updates.original.json", block.number, block.hash()),
+                    trie_updates.0,
+                )?;
+                let re_executed_path = self.save_file(
+                    format!("{}_{}.trie_updates.re_executed.json", block.number, block.hash()),
+                    &trie_output,
+                )?;
+                warn!(
+                    target: "engine::invalid_block_hooks::witness",
+                    original_path = %original_path.display(),
+                    re_executed_path = %re_executed_path.display(),
+                    "Trie updates mismatch after re-execution"
+                );
             }
         }
 
@@ -228,7 +257,14 @@ where
     ) -> eyre::Result<PathBuf> {
         let path = self.output_directory.join(filename);
         let diff = Comparison::new(original, new);
-        File::create_new(&path)?.write_all(diff.to_string().as_bytes())?;
+        File::create(&path)?.write_all(diff.to_string().as_bytes())?;
+
+        Ok(path)
+    }
+
+    fn save_file<T: Serialize>(&self, filename: String, value: &T) -> eyre::Result<PathBuf> {
+        let path = self.output_directory.join(filename);
+        File::create(&path)?.write_all(serde_json::to_string(value)?.as_bytes())?;
 
         Ok(path)
     }

crates/revm/Cargo.toml

@@ -35,3 +35,4 @@ std = []
 c-kzg = ["revm/c-kzg"]
 test-utils = ["dep:reth-trie"]
 optimism = ["revm/optimism"]
+serde = ["revm/serde"]