chore(ci): improve hive workflow (#9320)

.github/assets/hive/build_simulators.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Create the hive_assets directory
+mkdir hive_assets/
+
+cd hivetests
+go build .
+
+./hive -client reth # first builds and caches the client
+
+# Run each hive command in the background for each simulator and wait
+echo "Building images"
+./hive -client reth --sim "pyspec" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
+./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
+./hive -client reth --sim "smoke/genesis" -sim.timelimit 1s || true &
+./hive -client reth --sim "smoke/network" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true &
+wait
+
+# Run docker save in parallel and wait
+echo "Saving images"
+docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar &
+docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar &
+docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar &
+docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar &
+docker save hive/simulators/ethereum/pyspec:latest -o ../hive_assets/pyspec.tar &
+docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar &
+docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar &
+docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar &
+wait
+
+# Make sure we don't rebuild images on the CI jobs
+git apply ../.github/assets/hive/no_sim_build.diff
+go build .
+mv ./hive ../hive_assets/

.github/assets/hive/expected_failures.yaml

@@ -36,14 +36,6 @@ engine-withdrawals:
   - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
   - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
   - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
-  
-  # https://github.com/paradigmxyz/reth/issues/8304#issuecomment-2208515839
-  - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account - No Transactions (Paris) (reth)
-  - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account (Paris) (reth)
-  - Sync after 2 blocks - Withdrawals on Genesis - Single Withdrawal Account (Paris) (reth)
-  - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth)
-  - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
-  - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
 
 # https://github.com/paradigmxyz/reth/issues/8305
 # https://github.com/paradigmxyz/reth/issues/6217

.github/assets/hive/load_images.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# List of tar files to load
+IMAGES=(
+    "/tmp/hiveproxy.tar"
+    "/tmp/devp2p.tar"
+    "/tmp/engine.tar"
+    "/tmp/rpc_compat.tar"
+    "/tmp/pyspec.tar"
+    "/tmp/smoke_genesis.tar"
+    "/tmp/smoke_network.tar"
+    "/tmp/ethereum_sync.tar"
+    "/tmp/reth_image.tar"
+)
+
+# Loop through the images and load them
+for IMAGE_TAR in "${IMAGES[@]}"; do
+    echo "Loading image $IMAGE_TAR..."
+    docker load -i "$IMAGE_TAR" &
+done
+
+wait
+
+docker image ls -a

.github/assets/hive/no_sim_build.diff

@@ -0,0 +1,53 @@
+diff --git a/internal/libdocker/builder.go b/internal/libdocker/builder.go
+index 4731c9d..d717f52 100644
+--- a/internal/libdocker/builder.go
++++ b/internal/libdocker/builder.go
+@@ -7,9 +7,7 @@ import (
+ 	"fmt"
+ 	"io"
+ 	"io/fs"
+-	"os"
+ 	"path/filepath"
+-	"strings"
+ 
+ 	"github.com/ethereum/hive/internal/libhive"
+ 	docker "github.com/fsouza/go-dockerclient"
+@@ -53,24 +51,8 @@ func (b *Builder) BuildClientImage(ctx context.Context, client libhive.ClientDes
+ 
+ // BuildSimulatorImage builds a docker image of a simulator.
+ func (b *Builder) BuildSimulatorImage(ctx context.Context, name string) (string, error) {
+-	dir := b.config.Inventory.SimulatorDirectory(name)
+-	buildContextPath := dir
+-	buildDockerfile := "Dockerfile"
+-	// build context dir of simulator can be overridden with "hive_context.txt" file containing the desired build path
+-	if contextPathBytes, err := os.ReadFile(filepath.Join(filepath.FromSlash(dir), "hive_context.txt")); err == nil {
+-		buildContextPath = filepath.Join(dir, strings.TrimSpace(string(contextPathBytes)))
+-		if strings.HasPrefix(buildContextPath, "../") {
+-			return "", fmt.Errorf("cannot access build directory outside of Hive root: %q", buildContextPath)
+-		}
+-		if p, err := filepath.Rel(buildContextPath, filepath.Join(filepath.FromSlash(dir), "Dockerfile")); err != nil {
+-			return "", fmt.Errorf("failed to derive relative simulator Dockerfile path: %v", err)
+-		} else {
+-			buildDockerfile = p
+-		}
+-	}
+ 	tag := fmt.Sprintf("hive/simulators/%s:latest", name)
+-	err := b.buildImage(ctx, buildContextPath, buildDockerfile, tag, nil)
+-	return tag, err
++	return tag, nil
+ }
+ 
+ // BuildImage creates a container by archiving the given file system,
+diff --git a/internal/libdocker/proxy.go b/internal/libdocker/proxy.go
+index a53e5af..0bb2ea9 100644
+--- a/internal/libdocker/proxy.go
++++ b/internal/libdocker/proxy.go
+@@ -16,7 +16,7 @@ const hiveproxyTag = "hive/hiveproxy"
+ 
+ // Build builds the hiveproxy image.
+ func (cb *ContainerBackend) Build(ctx context.Context, b libhive.Builder) error {
+-	return b.BuildImage(ctx, hiveproxyTag, hiveproxy.Source)
++	return nil
+ }
+ 
+ // ServeAPI starts the API server.

.github/assets/hive/run_simulator.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# set -x
+
+cd hivetests/
+
+sim="${1}"
+limit="${2}"
+
+run_hive() {
+    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 4 --client reth 2>&1 | tee /tmp/log || true
+}
+
+check_log() {
+    tail -n 1 /tmp/log | sed -r 's/\x1B\[[0-9;]*[mK]//g'
+}
+
+attempt=0
+max_attempts=5
+
+while [ $attempt -lt $max_attempts ]; do
+    run_hive
+
+    # Check if no tests were run. sed removes ansi colors
+    if check_log | grep -q "suites=0"; then
+        echo "no tests were run, retrying in 5 seconds"
+        sleep 5
+        attempt=$((attempt + 1))
+        continue
+    fi
+
+    # Check the last line of the log for "finished", "tests failed", or "test failed"
+    if check_log | grep -Eq "(finished|tests? failed)"; then
+        exit 0
+    else
+        exit 1
+    fi
+done
+exit 1

.github/workflows/hive.yml

@@ -16,7 +16,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  prepare:
+  prepare-reth:
     if: github.repository == 'paradigmxyz/reth'
     timeout-minutes: 45
     runs-on:
@@ -44,6 +44,19 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Upload reth image
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts
+          path: ./artifacts
+
+  prepare-hive:
+    if: github.repository == 'paradigmxyz/reth'
+    timeout-minutes: 45
+    runs-on:
+      group: Reth
+    steps:
+      - uses: actions/checkout@v4
       - name: Checkout hive tests
         uses: actions/checkout@v4
         with:
@@ -55,18 +68,15 @@ jobs:
         with:
           go-version: "^1.13.1"
       - run: go version
-      - name: Build hive tool
-        run: |
-          cd hivetests
-          go build .
-          mv ./hive ../artifacts/
 
-      - name: Upload artifacts
+      - name: Build hive assets
+        run: .github/assets/hive/build_simulators.sh
+
+      - name: Upload hive assets
         uses: actions/upload-artifact@v4
         with:
-          name: artifacts
+          name: hive_assets
-          path: ./artifacts
+          path: ./hive_assets
-
   test:
     timeout-minutes: 60
     strategy:
@@ -105,8 +115,6 @@ jobs:
             limit: engine-withdrawals
           - sim: ethereum/engine
             limit: engine-auth
-          - sim: ethereum/engine
-            limit: engine-transition
           - sim: ethereum/engine
             limit: engine-api
           - sim: ethereum/engine
@@ -167,7 +175,9 @@ jobs:
             include: [homestead/]
           - sim: pyspec
             include: [frontier/]
-    needs: prepare
+    needs: 
+      - prepare-reth
+      - prepare-hive
     name: run
     runs-on:
       group: Reth
@@ -178,16 +188,21 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download artifacts
+      - name: Download hive assets
+        uses: actions/download-artifact@v4
+        with:
+          name: hive_assets
+          path: /tmp
+
+      - name: Download reth image
         uses: actions/download-artifact@v4
         with:
           name: artifacts
           path: /tmp
 
-      - name: Load Docker image
+      - name: Load Docker images
-        run: |
+        run: .github/assets/hive/load_images.sh
-          docker load --input /tmp/reth_image.tar
+
-          docker image ls -a
       - name: Move hive binary
         run: |
           mv /tmp/hive /usr/local/bin
@@ -201,37 +216,12 @@ jobs:
           path: hivetests
 
       - name: Run ${{ matrix.sim }} simulator
-        run: |
+        run: .github/assets/hive/run_simulator.sh "${{ matrix.sim }}$" "${{matrix.limit}}/${{join(matrix.include, '|')}}"
-          cd hivetests
-          hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --sim.parallelism 2 --client reth || true
 
       - name: Parse hive output
         run: |
           find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml
 
-      - name: Create github issue if sim failed
-        env:
-          GH_TOKEN: ${{ github.token }}
-        if: ${{ failure() }}
-        run: |
-          echo "Simulator failed, creating issue"
-          # Check if issue already exists
-          # get all issues with the label C-hivetest, loop over each page and check if the issue already exists
-
-          existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title')
-          if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then
-            echo "Issue already exists"
-            exit 0
-          fi
-          gh api \
-            --method POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            /repos/${{ github.repository }}/issues \
-            -f title='Hive Test Failure: ${{ matrix.sim }}' \
-            -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!<br /><br />The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.<br /><br />[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \
-            -f "labels[]=C-hivetest"
-
       - name: Print simulator output
         if: ${{ failure() }}
         run: |