From b97d9b4d2e0498efc61f0d45eee1bc40ed575a8b Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Thu, 23 Jan 2025 18:47:01 +0100 Subject: [PATCH] chore: refine the reproducible builds and add it to the release workflow (#13947) --- .github/workflows/release.yml | 37 ++++++++++++++++++++++++++++++++++- Cargo.toml | 7 ------- Dockerfile.reproducible | 24 ++++------------------- Makefile | 37 ++++++++++++++++++++++++++--------- 4 files changed, 68 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac2be796b..17868201c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,8 @@ env: OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth IMAGE_NAME: ${{ github.repository_owner }}/reth CARGO_TERM_COLOR: always + DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth + DOCKER_REPRODUCIBLE_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth-reproducible jobs: extract-version: @@ -102,9 +104,41 @@ jobs: name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc + build-reproducible: + name: build and push reproducible image + runs-on: ubuntu-latest + needs: extract-version + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push reproducible image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.reproducible + push: true + tags: | + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + draft-release: name: draft release - needs: [build, extract-version] + needs: [build, build-reproducible, extract-version] runs-on: ubuntu-latest env: VERSION: ${{ needs.extract-version.outputs.VERSION }} @@ -188,6 +222,7 @@ jobs: | | | | | | **System** | **Option** | - | **Resource** | | | Docker | | [${{ env.IMAGE_NAME }}](https://github.com/paradigmxyz/reth/pkgs/container/reth) | + | | Docker (Reproducible) | | [${{ env.IMAGE_NAME }}-reproducible](https://github.com/paradigmxyz/reth/pkgs/container/reth-reproducible) | ENDBODY ) assets=() diff --git a/Cargo.toml b/Cargo.toml index 723250382..1eb3881ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -289,13 +289,6 @@ codegen-units = 1 inherits = "release" lto = "fat" -[profile.reproducible] -inherits = "release" -debug = false -panic = "abort" -codegen-units = 1 -overflow-checks = true - [workspace.dependencies] # reth op-reth = { path = "crates/optimism/bin" } diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 89b9305d0..a90600ed2 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -4,30 +4,14 @@ FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af26723705 # Install specific version of libclang-dev RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 -# Clone the repository at the specific branch -RUN git clone https://github.com/paradigmxyz/reth /app +# Copy the project to the container +COPY ./ /app WORKDIR /app -# Get the latest commit timestamp and set SOURCE_DATE_EPOCH -RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment - -# Set environment variables for reproducibility -ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $(pwd)=." -ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ - CARGO_INCREMENTAL=0 \ - LC_ALL=C \ - TZ=UTC \ - RUSTFLAGS="${RUSTFLAGS}" - -# Set the default features if not provided -ARG FEATURES="jemalloc asm-keccak" - # Build the project with the reproducible settings -RUN . /etc/environment && \ - cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu +RUN make build-reproducible -RUN . /etc/environment && mv /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth +RUN mv /app/target/x86_64-unknown-linux-gnu/release/reth /reth # Create a minimal final image with just the binary FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a diff --git a/Makefile b/Makefile index b1908d7b1..d815189d5 100644 --- a/Makefile +++ b/Makefile @@ -62,15 +62,34 @@ install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. build: ## Build the reth binary into `target` directory. cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)" -SOURCE_DATE_EPOCH ?= $(shell git log -1 --pretty=%ct) -.PHONY: reproducible -reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently - SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ - CARGO_INCREMENTAL=0 \ - LC_ALL=C \ - TZ=UTC \ - RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." \ - cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu +# Environment variables for reproducible builds +# Initialize RUSTFLAGS +RUST_BUILD_FLAGS = +# Enable static linking to ensure reproducibility across builds +RUST_BUILD_FLAGS += --C target-feature=+crt-static +# Set the linker to use static libgcc to ensure reproducibility across builds +RUST_BUILD_FLAGS += -Clink-arg=-static-libgcc +# Remove build ID from the binary to ensure reproducibility across builds +RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none +# Remove metadata hash from symbol names to ensure reproducible builds +RUST_BUILD_FLAGS += -C metadata='' +# Set timestamp from last git commit for reproducible builds +SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) +# Disable incremental compilation to avoid non-deterministic artifacts +CARGO_INCREMENTAL_VAL = 0 +# Set C locale for consistent string handling and sorting +LOCALE_VAL = C +# Set UTC timezone for consistent time handling across builds +TZ_VAL = UTC + +.PHONY: build-reproducible +build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ + CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ + LC_ALL=${LOCALE_VAL} \ + TZ=${TZ_VAL} \ + cargo build --bin reth --features "$(FEATURES)" --profile "release" --locked --target x86_64-unknown-linux-gnu .PHONY: build-debug build-debug: ## Build the reth binary into `target/debug` directory.