diff --git a/.github/scripts/check_no_std.sh b/.github/assets/check_no_std.sh similarity index 100% rename from .github/scripts/check_no_std.sh rename to .github/assets/check_no_std.sh diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml new file mode 100644 index 000000000..831cb966f --- /dev/null +++ b/.github/assets/hive/expected_failures.yaml @@ -0,0 +1,128 @@ +# https://github.com/paradigmxyz/reth/issues/7015 +# https://github.com/paradigmxyz/reth/issues/6332 +rpc-compat: + - debug_getRawBlock/get-invalid-number (reth) + - debug_getRawHeader/get-invalid-number (reth) + - debug_getRawReceipts/get-invalid-number (reth) + - debug_getRawTransaction/get-invalid-hash (reth) + + - eth_call/call-callenv (reth) + - eth_createAccessList/create-al-contract-eip1559 (reth) + - eth_createAccessList/create-al-contract (reth) + - eth_feeHistory/fee-history (reth) + - eth_getStorageAt/get-storage-invalid-key-too-large (reth) + - eth_getStorageAt/get-storage-invalid-key (reth) + - eth_getTransactionReceipt/get-access-list (reth) + - eth_getTransactionReceipt/get-blob-tx (reth) + - eth_getTransactionReceipt/get-dynamic-fee (reth) + +# https://github.com/paradigmxyz/reth/issues/8732 +engine-withdrawals: + - Withdrawals Fork On Genesis (Paris) (reth) + - Withdrawals Fork on Block 1 (Paris) (reth) + - Withdrawals Fork on Block 2 (Paris) (reth) + - Withdrawals Fork on Block 3 (Paris) (reth) + - Withdraw to a single account (Paris) (reth) + - Withdraw to two accounts (Paris) (reth) + - Withdraw many accounts (Paris) (reth) + - Withdraw zero amount (Paris) (reth) + - Empty Withdrawals (Paris) (reth) + - Corrupted Block Hash Payload (INVALID) (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) + + # https://github.com/paradigmxyz/reth/issues/8304#issuecomment-2208515839 + - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account - No Transactions (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Genesis - Single Withdrawal Account (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth) + - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +engine-api: + - Inconsistent Head in ForkchoiceState (Paris) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) + - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + + # Hive issue + # https://github.com/ethereum/hive/issues/1135 + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +# https://github.com/paradigmxyz/reth/issues/8306 +# https://github.com/paradigmxyz/reth/issues/7144 +engine-cancun: + - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) + - Inconsistent Head in ForkchoiceState (Cancun) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) + - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + + # Hive issue + # https://github.com/ethereum/hive/issues/1135 + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + +# https://github.com/paradigmxyz/reth/issues/8579 +sync: + - sync reth -> reth \ No newline at end of file diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py new file mode 100644 index 000000000..ee75fdf55 --- /dev/null +++ b/.github/assets/hive/parse.py @@ -0,0 +1,43 @@ +import json +import yaml +import sys +import argparse + +# Argument parser setup +parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") +parser.add_argument("report_json", help="Path to the hive report JSON file.") +parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.") +args = parser.parse_args() + +# Load hive JSON +with open(args.report_json, 'r') as file: + report = json.load(file) + +# Load exclusion YAML +with open(args.exclusion, 'r') as file: + exclusion_data = yaml.safe_load(file) + exclusions = exclusion_data.get(report['name'], []) + +# Collect unexpected failures and passes +unexpected_failures = [] +unexpected_passes = [] + +for test in report['testCases'].values(): + test_name = test['name'] + test_pass = test['summaryResult']['pass'] + if test_name in exclusions: + if test_pass: + unexpected_passes.append(test_name) + else: + if not test_pass: + unexpected_failures.append(test_name) + +# Check if there are any unexpected failures or passes and exit with error +if unexpected_failures or unexpected_passes: + if unexpected_failures: + print("Unexpected Failures:", unexpected_failures) + if unexpected_passes: + print("Unexpected Passes:", unexpected_passes) + sys.exit(1) + +print("Success.") \ No newline at end of file diff --git a/.github/scripts/install_geth.sh b/.github/assets/install_geth.sh similarity index 100% rename from .github/scripts/install_geth.sh rename to .github/assets/install_geth.sh diff --git a/.github/scripts/label_pr.js b/.github/assets/label_pr.js similarity index 100% rename from .github/scripts/label_pr.js rename to .github/assets/label_pr.js diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 421e4b2f5..d18ffd65f 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -72,12 +72,9 @@ jobs: strategy: fail-fast: false matrix: - # TODO: enable etherem/sync once resolved: - # https://github.com/paradigmxyz/reth/issues/8579 # ethereum/rpc to be deprecated: # https://github.com/ethereum/hive/pull/1117 - # sim: [smoke/genesis, smoke/network, ethereum/sync] - sim: [smoke/genesis, smoke/network] + sim: [smoke/genesis, smoke/network, ethereum/sync] include: - sim: devp2p limit: discv4 @@ -104,26 +101,16 @@ jobs: - TestBlobViolations - sim: ethereum/engine limit: engine-exchange-capabilities - # TODO: enable engine-withdrawals once resolved: - # https://github.com/paradigmxyz/reth/issues/8732 - # - sim: ethereum/engine - # limit: engine-withdrawals + - sim: ethereum/engine + limit: engine-withdrawals - sim: ethereum/engine limit: engine-auth - sim: ethereum/engine limit: engine-transition - # TODO: enable engine-api once resolved: - # https://github.com/paradigmxyz/reth/issues/8305 - # https://github.com/paradigmxyz/reth/issues/6217 - # - sim: ethereum/engine - # limit: engine-api - # TODO: enable cancun once resolved: - # https://github.com/paradigmxyz/reth/issues/8305 - # https://github.com/paradigmxyz/reth/issues/6217 - # https://github.com/paradigmxyz/reth/issues/8306 - # https://github.com/paradigmxyz/reth/issues/7144 - # - sim: ethereum/engine - # limit: cancun + - sim: ethereum/engine + limit: engine-api + - sim: ethereum/engine + limit: cancun # eth_ rpc methods - sim: ethereum/rpc-compat include: @@ -144,12 +131,9 @@ jobs: - eth_getTransactionReceipt - eth_sendRawTransaction - eth_syncing - # TODO: enable debug_ rpc-compat once resolved: - # https://github.com/paradigmxyz/reth/issues/7015 - # https://github.com/paradigmxyz/reth/issues/6332 # debug_ rpc methods - # - sim: ethereum/rpc-compat - # include: [debug_] + - sim: ethereum/rpc-compat + include: [debug_] # Pyspec cancun jobs - sim: pyspec include: [cancun/eip4844] @@ -190,6 +174,10 @@ jobs: permissions: issues: write steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -215,7 +203,11 @@ jobs: - name: Run ${{ matrix.sim }} simulator run: | cd hivetests - hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth + hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --sim.parallelism 2 --client reth || true + + - name: Parse hive output + run: | + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml - name: Create github issue if sim failed env: @@ -239,6 +231,7 @@ jobs: -f title='Hive Test Failure: ${{ matrix.sim }}' \ -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ -f "labels[]=C-hivetest" + - name: Print simulator output if: ${{ failure() }} run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b4b494a90..103a87706 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - name: Install Geth - run: .github/scripts/install_geth.sh + run: .github/assets/install_geth.sh - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 with: diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index 857d354a8..077271735 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml @@ -19,5 +19,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const label_pr = require('./.github/scripts/label_pr.js') + const label_pr = require('./.github/assets/label_pr.js') await label_pr({github, context}) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c07cee388..d0329aefc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -58,7 +58,7 @@ jobs: with: cache-on-failure: true - name: Run no_std checks - run: .github/scripts/check_no_std.sh + run: .github/assets/check_no_std.sh crate-checks: runs-on: ubuntu-latest