From cb7f26a4e535cb461173c68513ed2ead47537362 Mon Sep 17 00:00:00 2001 From: Chris Oliver Date: Wed, 9 Sep 2026 13:58:52 -0500 Subject: [PATCH] Skip rebuilds whose inputs have not changed Two things caused today's repeated full rebuilds. Release New Versions decided which versions lack a release from `gh release list --limit 100`; with more than 100 releases in the repository every version whose releases fell outside that window was reported missing and rebuilt, and each rebuild pushed other versions out of the window for the next run. It now pages through every published release. Independently, a dispatched release always built, whether or not anything relevant had changed. bin/build-fingerprint digests everything in the repository that determines a version's output: its recipe, the effective series settings, dependency versions and checksums, the targets, package.rb and build.yml. Each revision release records the fingerprint in its notes, and a new plan job compares it to the newest revision's before building: a match with the floating release in place ends the run after one small job. A `force` input rebuilds regardless, for changes the fingerprint cannot see such as a container pin's contents or the Rust toolchain. The revision number is now determined in plan and handed to the later jobs. Claude-Session: https://claude.ai/code/session_01KeNcsk8V24YRXGfD2HwDZY --- .github/workflows/release-new.yml | 5 +- .github/workflows/release.yml | 116 +++++++++++++++++++++++------- CLAUDE.md | 1 + README.md | 7 ++ bin/build-fingerprint | 3 + libexec/build_fingerprint.rb | 65 +++++++++++++++++ 6 files changed, 171 insertions(+), 26 deletions(-) create mode 100755 bin/build-fingerprint create mode 100755 libexec/build_fingerprint.rb diff --git a/.github/workflows/release-new.yml b/.github/workflows/release-new.yml index e0a8164..af89435 100644 --- a/.github/workflows/release-new.yml +++ b/.github/workflows/release-new.yml @@ -53,7 +53,10 @@ jobs: versions_to_release="$recipe_versions" echo "Replacing all releases" else - existing_releases=$(gh release list --limit 100 --json tagName --jq '.[].tagName' || true) + # Every published release, not just the newest page: with more than 100 releases + # in the repository, a capped list reports old versions as missing and rebuilds + # them, which pushes other versions out of the window on the next run. + existing_releases=$(gh api "repos/${{ github.repository }}/releases" --paginate --jq '.[] | select(.draft | not) | .tag_name' || true) versions_to_release="" for version in $recipe_versions; do if ! echo "$existing_releases" | grep -q "^${version}$"; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ddf5378..f799bda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,11 @@ on: required: false type: boolean default: false + force: + description: 'Rebuild even if nothing that affects the build has changed' + required: false + type: boolean + default: false # Prevent multiple releases of the same version from running simultaneously concurrency: @@ -26,7 +31,85 @@ permissions: actions: read # For SLSA provenance jobs: + # Decide whether this version needs building at all. A rebuild is skipped when the + # newest revision release records the same build fingerprint (bin/build-fingerprint) + # as this commit and the floating release is in place; `force` overrides that. + plan: + runs-on: ubuntu-latest + outputs: + build: ${{ steps.decide.outputs.build }} + fingerprint: ${{ steps.fingerprint.outputs.fingerprint }} + revision_tag: ${{ steps.revision.outputs.tag }} + env: + VERSION: ${{ inputs.version }} + GH_REPO: ${{ github.repository }} + GITHUB_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + + - name: Compute build fingerprint + id: fingerprint + run: | + fingerprint="$(bin/build-fingerprint "$VERSION")" + echo "fingerprint=$fingerprint" >> "$GITHUB_OUTPUT" + echo "Build fingerprint: $fingerprint" + + - name: Determine build revision + id: revision + run: | + # Find existing revision tags (e.g., 3.4.5-1, 3.4.5-2, ...) + # Use git ls-remote to get all tags — not truncated like gh release list + last_revision=$(git ls-remote --tags origin "refs/tags/${VERSION}-*" | \ + sed 's|.*refs/tags/||' | \ + { grep -E "^${VERSION}-[0-9]+$" || true; } | \ + sed "s/^${VERSION}-//" | \ + sort -n | tail -1) + if [[ -z "$last_revision" ]]; then + revision=1 + echo "last_tag=" >> "$GITHUB_OUTPUT" + else + revision=$((last_revision + 1)) + echo "last_tag=${VERSION}-${last_revision}" >> "$GITHUB_OUTPUT" + fi + echo "tag=${VERSION}-${revision}" >> "$GITHUB_OUTPUT" + echo "Build revision: ${VERSION}-${revision}" + + - name: Decide whether to build + id: decide + env: + FINGERPRINT: ${{ steps.fingerprint.outputs.fingerprint }} + LAST_TAG: ${{ steps.revision.outputs.last_tag }} + run: | + build=true + if [[ "${{ inputs.force }}" == "true" ]]; then + echo "Building: forced" + elif [[ -z "$LAST_TAG" ]]; then + echo "Building: no previous revision release" + else + previous="$(gh release view "$LAST_TAG" --json body --jq .body 2>/dev/null | sed -n 's/^\*\*Build fingerprint:\*\* //p' | head -1)" + if [[ "$previous" != "$FINGERPRINT" ]]; then + echo "Building: fingerprint changed (${LAST_TAG} was built from ${previous:-an unrecorded fingerprint})" + elif ! gh api "repos/${GH_REPO}/releases/tags/${VERSION}" --jq '.assets | length' > /dev/null 2>&1; then + echo "Building: floating release ${VERSION} is missing" + else + build=false + echo "Skipping: ${LAST_TAG} was built from this same fingerprint and ${VERSION} is in place" + fi + fi + echo "build=$build" >> "$GITHUB_OUTPUT" + + - name: Mark existing release as latest + if: steps.decide.outputs.build == 'false' && inputs.latest + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN || github.token }} + run: gh release edit "${VERSION}" --latest + build: + needs: plan + if: needs.plan.outputs.build == 'true' uses: ./.github/workflows/build.yml with: version: ${{ inputs.version }} @@ -51,10 +134,10 @@ jobs: echo "hashes=$(sha256sum *.tar.gz | base64 -w0)" >> "$GITHUB_OUTPUT" create-release: - needs: build + needs: [plan, build] runs-on: ubuntu-latest outputs: - revision_tag: ${{ steps.revision.outputs.tag }} + revision_tag: ${{ needs.plan.outputs.revision_tag }} permissions: contents: write env: @@ -62,26 +145,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Determine build revision - id: revision - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # Find existing revision tags (e.g., 3.4.5-1, 3.4.5-2, ...) - # Use git ls-remote to get all tags — not truncated like gh release list - last_revision=$(git ls-remote --tags origin "refs/tags/${VERSION}-*" | \ - sed 's|.*refs/tags/||' | \ - { grep -E "^${VERSION}-[0-9]+$" || true; } | \ - sed "s/^${VERSION}-//" | \ - sort -n | tail -1) - if [[ -z "$last_revision" ]]; then - revision=1 - else - revision=$((last_revision + 1)) - fi - echo "tag=${VERSION}-${revision}" >> "$GITHUB_OUTPUT" - echo "Build revision: ${VERSION}-${revision}" - - name: Get Ruby release info id: ruby-release env: @@ -118,7 +181,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN || github.token }} RUBY_RELEASE_DATE: ${{ steps.ruby-release.outputs.date }} HAS_CHANGELOG: ${{ steps.ruby-release.outputs.has_changelog }} - REVISION_TAG: ${{ steps.revision.outputs.tag }} + REVISION_TAG: ${{ needs.plan.outputs.revision_tag }} + FINGERPRINT: ${{ needs.plan.outputs.fingerprint }} run: | # Mark as prerelease if version contains preview or rc if echo "${VERSION}" | grep -qiE '(preview|rc)'; then @@ -150,6 +214,8 @@ jobs: # leaves the floating release missing. echo "" >> /tmp/release_notes.md echo "**Build revision:** ${REVISION_TAG}" >> /tmp/release_notes.md + # Read back by the plan job of later runs to skip rebuilding unchanged versions. + echo "**Build fingerprint:** ${FINGERPRINT}" >> /tmp/release_notes.md # Drop leftover drafts for this revision from an earlier run that failed before # publishing; a draft has no tag, so the revision number can repeat until published. @@ -348,9 +414,9 @@ jobs: -f sha="${{ github.sha }}" -F force=true > /dev/null notify: - needs: [build, combine-hashes, create-release, provenance, upload-assets, publish-release] + needs: [plan, build, combine-hashes, create-release, provenance, upload-assets, publish-release] runs-on: ubuntu-latest - if: always() + if: always() && needs.plan.outputs.build == 'true' env: RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} steps: diff --git a/CLAUDE.md b/CLAUDE.md index a3995af..c411f3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,6 +41,7 @@ YJIT builds require `rustup` or `rustc` in `PATH`. Set `JDX_RUBY_RUSTUP_HOME` to - `bin/update-ruby-recipe`: Adds or updates a Ruby entry in `recipes/rubies.yml`; used by autobump. - `bin/package-linux`: Runs `bin/package` for a Linux target inside its pinned manylinux2014 container, mirroring the build workflow, for local Linux builds. - `bin/recipe-info`: Prints a version's effective series settings (`macos`, `legacy`, ...) for workflows. +- `bin/build-fingerprint`: Prints a digest of everything in the repo that affects a version's build; the release workflow skips rebuilding when it matches the newest revision release's. ### Old series diff --git a/README.md b/README.md index 645594a..6b93f43 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,13 @@ These Rubies use the first available certificate source in this order: [An automated release workflow is available to use](https://github.com/hatchboxio/precompiled-ruby/actions/workflows/release.yml). Dispatch the workflow with a Ruby version and it will build, upload SLSA provenance, publish an immutable build revision release (e.g. `3.4.7-2`), and then re-point the floating release (e.g. `3.4.7`) at that build. The floating release is updated in place rather than recreated, so its download URLs keep working while a rebuild is in flight or if one fails; new assets are renamed over the old ones rather than re-uploaded in place. +A dispatched release only builds when something that affects the build has changed. Each +revision release records a build fingerprint (`bin/build-fingerprint VERSION`: the version's +recipe, its effective series settings, dependency versions and checksums, the targets, the +packaging script and the build workflow), and a run whose fingerprint matches the newest +revision's exits after its first job. Tick `force` to rebuild anyway, for example when a +pinned container or the Rust toolchain is the reason. + [Release New Versions](https://github.com/hatchboxio/precompiled-ruby/actions/workflows/release-new.yml) dispatches that for every recipe that has no release yet; it runs on a schedule, when `recipes/rubies.yml` changes on `main`, or by hand, where its `only` input releases exactly diff --git a/bin/build-fingerprint b/bin/build-fingerprint new file mode 100755 index 0000000..7f53528 --- /dev/null +++ b/bin/build-fingerprint @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +exec ruby "$(dirname "$0")/../libexec/build_fingerprint.rb" "$@" diff --git a/libexec/build_fingerprint.rb b/libexec/build_fingerprint.rb new file mode 100755 index 0000000..df041f3 --- /dev/null +++ b/libexec/build_fingerprint.rb @@ -0,0 +1,65 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Print a digest of everything in this repository that determines what a Ruby version's +# build produces, so the release workflow can skip a rebuild when none of it changed. +# Covered: the version's recipe, its effective series settings (same merge as package.rb), +# every dependency's version and checksum, the release targets, the packaging script and +# the build workflow. Not covered: what the pinned containers, the Rust toolchain and the +# bootstrap Ruby resolve to at build time, so pass `force` to the release workflow when +# one of those is the reason to rebuild. +require "digest" +require "json" +require "yaml" + +ROOT = File.expand_path("..", __dir__) + +# Bump when the inputs below change shape, so older fingerprints never compare equal. +FORMAT = 1 + +HASHED_FILES = %w[ + libexec/package.rb + .github/workflows/build.yml +].freeze + +def load_yaml(path) + YAML.safe_load_file(File.join(ROOT, path), permitted_classes: [], aliases: false) +end + +def deep_merge(left, right) + left.merge(right) do |_key, old_value, new_value| + old_value.is_a?(Hash) && new_value.is_a?(Hash) ? deep_merge(old_value, new_value) : new_value + end +end + +def canonical(value) + case value + when Hash then value.sort_by { |key, _| key.to_s }.to_h { |key, item| [key.to_s, canonical(item)] } + when Array then value.map { |item| canonical(item) } + else value + end +end + +version = ARGV.fetch(0) { abort "Usage: bin/build-fingerprint VERSION" } + +rubies = load_yaml("recipes/rubies.yml").fetch("rubies") +recipe = rubies.fetch(version) { abort "Unknown Ruby version: #{version}" } +series_doc = load_yaml("recipes/series.yml") +series = deep_merge(series_doc.fetch("defaults"), series_doc.fetch("series").fetch(recipe.fetch("series")) || {}) +dependencies = load_yaml("recipes/dependencies.yml").fetch("dependencies").transform_values do |dependency| + dependency.slice("version", "sha256") +end +targets = load_yaml("recipes/targets.yml").fetch("targets") +files = HASHED_FILES.to_h { |path| [path, Digest::SHA256.file(File.join(ROOT, path)).hexdigest] } + +inputs = { + "format" => FORMAT, + "version" => version, + "recipe" => recipe, + "series" => series, + "dependencies" => dependencies, + "targets" => targets, + "files" => files +} + +puts "v#{FORMAT}-#{Digest::SHA256.hexdigest(JSON.generate(canonical(inputs)))}"