feat: improve eth test runner (#868)

bin/reth/src/cli.rs

@@ -34,9 +34,6 @@ pub enum Commands {
     /// Start the node
     #[command(name = "node")]
     Node(node::Command),
-    /// Run Ethereum blockchain tests
-    #[command(name = "test-chain")]
-    TestEthChain(test_eth_chain::Command),
     /// Database debugging utilities
     #[command(name = "db")]
     Db(db::Command),
@@ -51,6 +48,9 @@ pub enum Commands {
     /// P2P Debugging utilities
     #[command(name = "p2p")]
     P2P(p2p::Command),
+    /// Run Ethereum blockchain tests
+    #[command(name = "test-chain")]
+    TestEthChain(test_eth_chain::Command),
 }
 
 #[derive(Parser)]

bin/reth/src/test_eth_chain/mod.rs

@@ -2,13 +2,16 @@
 
 use clap::Parser;
 use eyre::eyre;
+use futures::{stream::FuturesUnordered, StreamExt};
 use std::path::PathBuf;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 /// Models for parsing JSON blockchain tests
 pub mod models;
 /// Ethereum blockhain test runner
 pub mod runner;
 
+use runner::TestOutcome;
+
 /// Execute Ethereum blockchain tests by specifying path to json files
 #[derive(Debug, Parser)]
 pub struct Command {
@@ -19,34 +22,37 @@ pub struct Command {
 impl Command {
     /// Execute the command
     pub async fn execute(self) -> eyre::Result<()> {
-        // note the use of `into_iter()` to consume `items`
-        let futs: Vec<_> = self
+        let mut futs: FuturesUnordered<_> = self
             .path
             .iter()
             .flat_map(|item| reth_cli_utils::find_all_files_with_postfix(item, ".json"))
             .map(|file| async { (runner::run_test(file.clone()).await, file) })
             .collect();
 
-        let results = futures::future::join_all(futs).await;
-        // await the tasks for resolve's to complete and give back our test results
-        let mut num_of_failed = 0;
-        let mut num_of_passed = 0;
-        for (result, file) in results {
-            match result {
-                Ok(_) => {
-                    num_of_passed += 1;
+        let mut failed = 0;
+        let mut passed = 0;
+        let mut skipped = 0;
+        while let Some((result, file)) = futs.next().await {
+            match TestOutcome::from(result) {
+                TestOutcome::Passed => {
+                    info!(target: "reth::cli", "[+] Test {file:?} passed.");
+                    passed += 1;
                 }
-                Err(error) => {
-                    num_of_failed += 1;
+                TestOutcome::Skipped => {
+                    warn!(target: "reth::cli", "[=] Test {file:?} skipped.");
+                    skipped += 1;
+                }
+                TestOutcome::Failed(error) => {
                     error!(target: "reth::cli", "Test {file:?} failed:\n{error}");
+                    failed += 1;
                 }
             }
         }
 
-        info!(target: "reth::cli", "{num_of_passed}/{} tests passed\n", num_of_passed + num_of_failed);
+        info!(target: "reth::cli", "{passed}/{0} tests passed, {skipped}/{0} skipped, {failed}/{0} failed.\n", failed + passed + skipped);
 
-        if num_of_failed != 0 {
-            Err(eyre!("Failed {num_of_failed} tests"))
+        if failed != 0 {
+            Err(eyre!("Failed {failed} tests"))
         } else {
             Ok(())
         }

bin/reth/src/test_eth_chain/models.rs

@@ -5,35 +5,35 @@ use reth_primitives::{
 use serde::{self, Deserialize};
 use std::collections::BTreeMap;
 
-/// Blockchain test deserializer.
+/// An Ethereum blockchain test.
 #[derive(Debug, PartialEq, Eq, Deserialize)]
 pub struct Test(pub BTreeMap<String, BlockchainTestData>);
 
-/// Ethereum blockchain test data
+/// Ethereum test data.
 #[derive(Debug, PartialEq, Eq, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct BlockchainTestData {
     /// Genesis block header.
     pub genesis_block_header: Header,
-    /// Genesis rlp.
+    /// RLP encoded genesis block.
     #[serde(rename = "genesisRLP")]
     pub genesis_rlp: Option<Bytes>,
-    /// Blocks.
+    /// Block data.
     pub blocks: Vec<Block>,
-    /// Post state.
+    /// The expected post state.
     pub post_state: Option<RootOrState>,
-    /// Pre state.
+    /// The test pre-state.
     pub pre: State,
-    /// Hash of best block.
+    /// Hash of the best block.
     pub lastblockhash: H256,
-    /// Network.
+    /// Network spec.
     pub network: ForkSpec,
     #[serde(default)]
-    /// Engine
+    /// Engine spec.
     pub self_engine: SealEngine,
 }
 
-/// Ethereum blockchain test data Header.
+/// A block header in an Ethereum blockchain test.
 #[derive(Debug, PartialEq, Eq, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Header {
@@ -99,14 +99,14 @@ impl From<Header> for SealedHeader {
     }
 }
 
-/// Ethereum blockchain test data Block.
+/// A block in an Ethereum blockchain test.
 #[derive(Debug, PartialEq, Eq, Deserialize)]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]
 pub struct Block {
     /// Block header.
     pub block_header: Option<Header>,
-    /// Rlp block bytes
+    /// RLP encoded block bytes
     pub rlp: Bytes,
     /// Transactions
     pub transactions: Option<Vec<Transaction>>,
@@ -126,11 +126,7 @@ pub struct TransactionSequence {
     valid: String,
 }
 
-/// Ethereum blockchain test data State.
-//#[derive(Clone, Debug, Eq, PartialEq, Deserialize)]
-//#[serde(deny_unknown_fields)]
-//pub struct State(pub RootOrState);
-
+/// Ethereum blockchain test data state.
 #[derive(Clone, Debug, Eq, PartialEq, Deserialize)]
 pub struct State(pub BTreeMap<Address, Account>);
 
@@ -144,7 +140,7 @@ pub enum RootOrState {
     State(BTreeMap<Address, Account>),
 }
 
-/// Spec account
+/// An account.
 #[derive(Debug, PartialEq, Eq, Deserialize, Clone)]
 #[serde(deny_unknown_fields)]
 pub struct Account {
@@ -158,7 +154,7 @@ pub struct Account {
     pub storage: BTreeMap<JsonU256, JsonU256>,
 }
 
-/// Fork Spec
+/// Fork specification.
 #[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Deserialize)]
 pub enum ForkSpec {
     /// Frontier
@@ -248,7 +244,7 @@ impl From<ForkSpec> for reth_executor::SpecUpgrades {
     }
 }
 
-/// Json Block test possible engine kind.
+/// Possible seal engines.
 #[derive(Debug, PartialEq, Eq, Default, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum SealEngine {

bin/reth/src/test_eth_chain/runner.rs

@@ -21,7 +21,27 @@ use std::{
     ffi::OsStr,
     path::{Path, PathBuf},
 };
-use tracing::{debug, info, trace};
+use tracing::{debug, trace};
+
+/// The outcome of a test.
+#[derive(Debug)]
+pub enum TestOutcome {
+    /// The test was skipped.
+    Skipped,
+    /// The test passed.
+    Passed,
+    /// The test failed.
+    Failed(eyre::Report),
+}
+
+impl From<eyre::Result<TestOutcome>> for TestOutcome {
+    fn from(v: eyre::Result<TestOutcome>) -> TestOutcome {
+        match v {
+            Ok(outcome) => outcome,
+            Err(err) => TestOutcome::Failed(err),
+        }
+    }
+}
 
 /// Tests are test edge cases that are not possible to happen on mainnet, so we are skipping them.
 pub fn should_skip(path: &Path) -> bool {
@@ -72,15 +92,16 @@ pub fn should_skip(path: &Path) -> bool {
 }
 
 /// Run one JSON-encoded Ethereum blockchain test at the specified path.
-pub async fn run_test(path: PathBuf) -> eyre::Result<()> {
+pub async fn run_test(path: PathBuf) -> eyre::Result<TestOutcome> {
     let path = path.as_path();
     let json_file = std::fs::read(path)?;
     let suites: Test = serde_json::from_reader(&*json_file)?;
 
     if should_skip(path) {
-        return Ok(())
+        return Ok(TestOutcome::Skipped)
     }
-    info!(target: "reth::cli", ?path, "Running test suite");
+
+    debug!(target: "reth::cli", ?path, "Running test suite");
 
     for (name, suite) in suites.0 {
         if matches!(
@@ -189,7 +210,7 @@ pub async fn run_test(path: PathBuf) -> eyre::Result<()> {
         // Validate post state
         match suite.post_state {
             Some(RootOrState::Root(root)) => {
-                info!(target: "reth::cli", "Post-state root: #{root:?}")
+                debug!(target: "reth::cli", "Post-state root: #{root:?}")
             }
             Some(RootOrState::State(state)) => db.view(|tx| -> eyre::Result<()> {
                 let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
@@ -274,8 +295,8 @@ pub async fn run_test(path: PathBuf) -> eyre::Result<()> {
                 }
                 Ok(())
             })??,
-            None => info!(target: "reth::cli", "No post-state"),
+            None => debug!(target: "reth::cli", "No post-state"),
         }
     }
-    Ok(())
+    Ok(TestOutcome::Passed)
 }