diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbcb45f77..4ca092401 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,9 +10,12 @@ on: env: REPO_NAME: ${{ github.repository_owner }}/reth IMAGE_NAME: ${{ github.repository_owner }}/reth + RUSTFLAGS: -D warnings + CARGO_TERM_COLOR: always jobs: extract-version: + name: extract version runs-on: ubuntu-latest steps: - name: Extract version @@ -26,8 +29,9 @@ jobs: matrix: arch: [aarch64-unknown-linux-gnu, x86_64-unknown-linux-gnu, - x86_64-apple-darwin] - #x86_64-windows] + x86_64-apple-darwin, + aarch64-apple-darwin, + x86_64-pc-windows-gnu] include: - arch: aarch64-unknown-linux-gnu platform: ubuntu-latest @@ -38,9 +42,12 @@ jobs: - arch: x86_64-apple-darwin platform: macos-latest profile: maxperf - #- arch: x86_64-windows - # platform: windows-2019 - # profile: maxperf + - arch: aarch64-apple-darwin + platform: macos-latest + profile: maxperf + - arch: x86_64-pc-windows-gnu + platform: ubuntu-latest + profile: maxperf runs-on: ${{ matrix.platform }} needs: extract-version @@ -49,81 +56,59 @@ jobs: uses: actions/checkout@v3 - name: Get latest version of stable Rust run: rustup update stable - - # ============================== - # Windows dependencies - # ============================== - - - uses: KyleMayes/install-llvm-action@v1 - if: startsWith(matrix.arch, 'x86_64-windows') + - name: Install target + run: rustup target add ${{ matrix.arch }} + - uses: Swatinem/rust-cache@v2 with: - version: "13.0" - directory: ${{ runner.temp }}/llvm - - name: Set LIBCLANG_PATH - if: startsWith(matrix.arch, 'x86_64-windows') - run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + cache-on-failure: true + + # ============================== + # Apple M1 SDK setup + # ============================== + + - name: Apple M1 setup + if: ${{ matrix.job.target == 'aarch64-apple-darwin' }} + run: | + echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV # ============================== # Builds # ============================== - - name: Build reth for aarch64-unknown-linux-gnu - if: matrix.arch == 'aarch64-unknown-linux-gnu' + - name: Build reth for ${{ matrix.arch }} run: | cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64 - - - name: Build reth for x86_64-unknown-linux-gnu - if: matrix.arch == 'x86_64-unknown-linux-gnu' - run: | - cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 + env PROFILE=${{ matrix.profile }} make build-${{ matrix.arch }} - name: Move cross-compiled binary - if: startsWith(matrix.arch, 'aarch64') - run: mv target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/reth ~/.cargo/bin/reth + if: matrix.arch != 'x86_64-pc-windows-gnu' + run: | + mkdir artifacts + mv target/${{ matrix.arch }}/${{ matrix.profile }}/reth ./artifacts - - name: Move cross-compiled binary - if: startsWith(matrix.arch, 'x86_64-unknown-linux-gnu') - run: mv target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/reth ~/.cargo/bin/reth + - name: Move cross-compiled binary (Windows) + if: matrix.arch == 'x86_64-pc-windows-gnu' + run: | + mkdir artifacts + mv target/${{ matrix.arch }}/${{ matrix.profile }}/reth.exe ./artifacts - - name: Build reth for x86_64-apple-darwin modern - if: matrix.arch == 'x86_64-apple-darwin' - run: cargo install --path reth --force --locked --profile ${{ matrix.profile }} - - - name: Build reth for Windows modern - if: matrix.arch == 'x86_64-windows' - # NOTE: profile set to release (see above) - run: cargo install --path reth --force --locked --profile release + # ============================== + # Signing + # ============================== - name: Configure GPG and create artifacts - if: startsWith(matrix.arch, 'x86_64-windows') != true env: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | export GPG_TTY=$(tty) - echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --import - mkdir artifacts - mv ~/.cargo/bin/reth ./artifacts + echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import cd artifacts - tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz reth + tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz reth* echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz mv *tar.gz* .. - - - name: Configure GPG and create artifacts Windows - if: startsWith(matrix.arch, 'x86_64-windows') - env: - GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - echo $env:GPG_SIGNING_KEY | base64 --decode | gpg --batch --import - mkdir artifacts - move $env:USERPROFILE/.cargo/bin/reth.exe ./artifacts - cd artifacts - tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz reth.exe - gpg --passphrase "$env:GPG_PASSPHRASE" --batch --pinentry-mode loopback -ab reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz - move *tar.gz* .. + shell: bash # ======================================================================= # Upload artifacts @@ -142,11 +127,14 @@ jobs: path: reth-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc draft-release: - name: Draft Release + name: draft release needs: [build, extract-version] runs-on: ubuntu-latest env: VERSION: ${{ needs.extract-version.outputs.VERSION }} + permissions: + # Required to post the release + contents: write steps: # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts. - name: Checkout sources @@ -163,14 +151,14 @@ jobs: # ============================== # Create release draft # ============================== - - name: Generate Full Changelog + - name: Generate full changelog id: changelog run: | echo "CHANGELOG<> $GITHUB_OUTPUT echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - name: Create Release Draft + - name: Create release draft env: GITHUB_USER: ${{ github.repository_owner }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -227,9 +215,11 @@ jobs: | System | Architecture | Binary | PGP Signature | |:---:|:---:|:---:|:---| - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | | | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | | | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | + | | x86_64 | [reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) | + | | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | + | | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | | | | | | | **System** | **Option** | - | **Resource** | | | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | diff --git a/.gitignore b/.gitignore index 44162106f..96bb78b92 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,11 @@ testdata/micro/db # Generated data for stage benchmarks crates/stages/testdata -# prometheus data dir +# Prometheus data dir data/ -proptest-regressions/ \ No newline at end of file +# Proptest data +proptest-regressions/ + +# Release artifacts +dist/ \ No newline at end of file diff --git a/Makefile b/Makefile index a14f1ccdf..8b164a868 100644 --- a/Makefile +++ b/Makefile @@ -1,42 +1,37 @@ # Heavily inspired by Lighthouse: https://github.com/sigp/lighthouse/blob/693886b94176faa4cb450f024696cb69cda2fe58/Makefile GIT_TAG := $(shell git describe --tags --candidates 1) -BIN_DIR = "bin" +BIN_DIR = "dist/bin" -X86_64_TAG = "x86_64-unknown-linux-gnu" -BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release" -AARCH64_TAG = "aarch64-unknown-linux-gnu" -BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release" +BUILD_PATH = "target" -# List of features to use when building natively. Can be overriden via the environment. +# List of features to use when building. Can be overriden via the environment. # No jemalloc on Windows ifeq ($(OS),Windows_NT) - FEATURES?= + FEATURES ?= else - FEATURES?=jemalloc + FEATURES ?= jemalloc endif -# List of features to use when cross-compiling. Can be overridden via the environment. -CROSS_FEATURES ?= jemalloc - -# Cargo profile for Cross builds. Default is for local builds, CI uses an override. -CROSS_PROFILE ?= release - -# Cargo profile for regular builds. +# Cargo profile for builds. Default is for local builds, CI uses an override. PROFILE ?= release # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS ?= -# Builds the reth binary in release (optimized). +# Builds and installs the reth binary. # -# Binaries will most likely be found in `./target/release` +# The binary will most likely be in `~/.cargo/bin` install: - cargo install --path reth --force --locked \ + cargo install --path bin/reth --bin reth --force --locked \ --features "$(FEATURES)" \ --profile "$(PROFILE)" \ $(CARGO_INSTALL_EXTRA_FLAGS) +# Builds the reth binary natively. +build-native-%: + cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)" + # The following commands use `cross` to build a cross-compile. # # These commands require that: @@ -46,22 +41,33 @@ install: # - The current user is in the `docker` group. # # The resulting binaries will be created in the `target/` directory. -# + +# No jemalloc on Windows +build-x86_64-pc-windows-gnu: FEATURES := $(filter-out jemalloc,$(FEATURES)) + # Note: The additional rustc compiler flags are for intrinsics needed by MDBX. # See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std -build-x86_64 build-aarch64: export RUSTFLAGS=-C link-arg=-lgcc -Clink-arg=-static-libgcc +build-%: + RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \ + cross build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)" -build-x86_64: - cross build --bin reth --target x86_64-unknown-linux-gnu --features "$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" -build-aarch64: - cross build --bin reth --target aarch64-unknown-linux-gnu --features "$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" +# Unfortunately we can't easily use cross to build for Darwin because of licensing issues. +# If we wanted to, we would need to build a custom Docker image with the SDK available. +# +# Note: You must set `SDKROOT` and `MACOSX_DEPLOYMENT_TARGET`. These can be found using `xcrun`. +# +# `SDKROOT=$(xcrun -sdk macosx --show-sdk-path) MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)` +build-x86_64-apple-darwin: + $(MAKE) build-native-x86_64-apple-darwin +build-aarch64-apple-darwin: + $(MAKE) build-native-aarch64-apple-darwin # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary - cp $(1)/reth $(BIN_DIR)/reth + cp $(BUILD_PATH)/$(1)/$(PROFILE)/$(2) $(BIN_DIR)/$(2) cd $(BIN_DIR) && \ - tar -czf reth-$(GIT_TAG)-$(2)$(3).tar.gz reth && \ - rm reth + tar -czf reth-$(GIT_TAG)-$(1)$(3).tar.gz $(2) && \ + rm $(2) endef # Create a series of `.tar.gz` files in the BIN_DIR directory, each containing @@ -69,13 +75,18 @@ endef # # The current git tag will be used as the version in the output file names. You # will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`). +# +# Note: This excludes macOS tarballs because of SDK licensing issues. build-release-tarballs: [ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR) - $(MAKE) build-x86_64 - $(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"") - $(MAKE) build-aarch64 - $(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"") + $(MAKE) build-x86_64-unknown-linux-gnu + $(call tarball_release_binary,"x86_64-unknown-linux-gnu","reth","") + $(MAKE) build-aarch64-unknown-linux-gnu + $(call tarball_release_binary,"aarch64-unknown-linux-gnu","reth","") + $(MAKE) build-x86_64-pc-windows-gnu + $(call tarball_release_binary,"x86_64-pc-windows-gnu","reth.exe","") -# Performs a `cargo` clean +# Performs a `cargo` clean and removes the binary directory clean: - cargo clean \ No newline at end of file + cargo clean + rm -r $(BIN_DIR) \ No newline at end of file