diff --git a/.github/workflows/release-new.yml b/.github/workflows/release-new.yml index e0a8164..af89435 100644 --- a/.github/workflows/release-new.yml +++ b/.github/workflows/release-new.yml @@ -53,7 +53,10 @@ jobs: versions_to_release="$recipe_versions" echo "Replacing all releases" else - existing_releases=$(gh release list --limit 100 --json tagName --jq '.[].tagName' || true) + # Every published release, not just the newest page: with more than 100 releases + # in the repository, a capped list reports old versions as missing and rebuilds + # them, which pushes other versions out of the window on the next run. + existing_releases=$(gh api "repos/${{ github.repository }}/releases" --paginate --jq '.[] | select(.draft | not) | .tag_name' || true) versions_to_release="" for version in $recipe_versions; do if ! echo "$existing_releases" | grep -q "^${version}$"; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ddf5378..f799bda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,11 @@ on: required: false type: boolean default: false + force: + description: 'Rebuild even if nothing that affects the build has changed' + required: false + type: boolean + default: false # Prevent multiple releases of the same version from running simultaneously concurrency: @@ -26,7 +31,85 @@ permissions: actions: read # For SLSA provenance jobs: + # Decide whether this version needs building at all. A rebuild is skipped when the + # newest revision release records the same build fingerprint (bin/build-fingerprint) + # as this commit and the floating release is in place; `force` overrides that. + plan: + runs-on: ubuntu-latest + outputs: + build: ${{ steps.decide.outputs.build }} + fingerprint: ${{ steps.fingerprint.outputs.fingerprint }} + revision_tag: ${{ steps.revision.outputs.tag }} + env: + VERSION: ${{ inputs.version }} + GH_REPO: ${{ github.repository }} + GITHUB_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + + - name: Compute build fingerprint + id: fingerprint + run: | + fingerprint="$(bin/build-fingerprint "$VERSION")" + echo "fingerprint=$fingerprint" >> "$GITHUB_OUTPUT" + echo "Build fingerprint: $fingerprint" + + - name: Determine build revision + id: revision + run: | + # Find existing revision tags (e.g., 3.4.5-1, 3.4.5-2, ...) + # Use git ls-remote to get all tags — not truncated like gh release list + last_revision=$(git ls-remote --tags origin "refs/tags/${VERSION}-*" | \ + sed 's|.*refs/tags/||' | \ + { grep -E "^${VERSION}-[0-9]+$" || true; } | \ + sed "s/^${VERSION}-//" | \ + sort -n | tail -1) + if [[ -z "$last_revision" ]]; then + revision=1 + echo "last_tag=" >> "$GITHUB_OUTPUT" + else + revision=$((last_revision + 1)) + echo "last_tag=${VERSION}-${last_revision}" >> "$GITHUB_OUTPUT" + fi + echo "tag=${VERSION}-${revision}" >> "$GITHUB_OUTPUT" + echo "Build revision: ${VERSION}-${revision}" + + - name: Decide whether to build + id: decide + env: + FINGERPRINT: ${{ steps.fingerprint.outputs.fingerprint }} + LAST_TAG: ${{ steps.revision.outputs.last_tag }} + run: | + build=true + if [[ "${{ inputs.force }}" == "true" ]]; then + echo "Building: forced" + elif [[ -z "$LAST_TAG" ]]; then + echo "Building: no previous revision release" + else + previous="$(gh release view "$LAST_TAG" --json body --jq .body 2>/dev/null | sed -n 's/^\*\*Build fingerprint:\*\* //p' | head -1)" + if [[ "$previous" != "$FINGERPRINT" ]]; then + echo "Building: fingerprint changed (${LAST_TAG} was built from ${previous:-an unrecorded fingerprint})" + elif ! gh api "repos/${GH_REPO}/releases/tags/${VERSION}" --jq '.assets | length' > /dev/null 2>&1; then + echo "Building: floating release ${VERSION} is missing" + else + build=false + echo "Skipping: ${LAST_TAG} was built from this same fingerprint and ${VERSION} is in place" + fi + fi + echo "build=$build" >> "$GITHUB_OUTPUT" + + - name: Mark existing release as latest + if: steps.decide.outputs.build == 'false' && inputs.latest + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN || github.token }} + run: gh release edit "${VERSION}" --latest + build: + needs: plan + if: needs.plan.outputs.build == 'true' uses: ./.github/workflows/build.yml with: version: ${{ inputs.version }} @@ -51,10 +134,10 @@ jobs: echo "hashes=$(sha256sum *.tar.gz | base64 -w0)" >> "$GITHUB_OUTPUT" create-release: - needs: build + needs: [plan, build] runs-on: ubuntu-latest outputs: - revision_tag: ${{ steps.revision.outputs.tag }} + revision_tag: ${{ needs.plan.outputs.revision_tag }} permissions: contents: write env: @@ -62,26 +145,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Determine build revision - id: revision - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # Find existing revision tags (e.g., 3.4.5-1, 3.4.5-2, ...) - # Use git ls-remote to get all tags — not truncated like gh release list - last_revision=$(git ls-remote --tags origin "refs/tags/${VERSION}-*" | \ - sed 's|.*refs/tags/||' | \ - { grep -E "^${VERSION}-[0-9]+$" || true; } | \ - sed "s/^${VERSION}-//" | \ - sort -n | tail -1) - if [[ -z "$last_revision" ]]; then - revision=1 - else - revision=$((last_revision + 1)) - fi - echo "tag=${VERSION}-${revision}" >> "$GITHUB_OUTPUT" - echo "Build revision: ${VERSION}-${revision}" - - name: Get Ruby release info id: ruby-release env: @@ -118,7 +181,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN || github.token }} RUBY_RELEASE_DATE: ${{ steps.ruby-release.outputs.date }} HAS_CHANGELOG: ${{ steps.ruby-release.outputs.has_changelog }} - REVISION_TAG: ${{ steps.revision.outputs.tag }} + REVISION_TAG: ${{ needs.plan.outputs.revision_tag }} + FINGERPRINT: ${{ needs.plan.outputs.fingerprint }} run: | # Mark as prerelease if version contains preview or rc if echo "${VERSION}" | grep -qiE '(preview|rc)'; then @@ -150,6 +214,8 @@ jobs: # leaves the floating release missing. echo "" >> /tmp/release_notes.md echo "**Build revision:** ${REVISION_TAG}" >> /tmp/release_notes.md + # Read back by the plan job of later runs to skip rebuilding unchanged versions. + echo "**Build fingerprint:** ${FINGERPRINT}" >> /tmp/release_notes.md # Drop leftover drafts for this revision from an earlier run that failed before # publishing; a draft has no tag, so the revision number can repeat until published. @@ -348,9 +414,9 @@ jobs: -f sha="${{ github.sha }}" -F force=true > /dev/null notify: - needs: [build, combine-hashes, create-release, provenance, upload-assets, publish-release] + needs: [plan, build, combine-hashes, create-release, provenance, upload-assets, publish-release] runs-on: ubuntu-latest - if: always() + if: always() && needs.plan.outputs.build == 'true' env: RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} steps: diff --git a/CLAUDE.md b/CLAUDE.md index 253e650..4366af4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ YJIT builds require `rustup` or `rustc` in `PATH`. Set `JDX_RUBY_RUSTUP_HOME` to - `bin/update-ruby-recipe`: Adds or updates a Ruby entry in `recipes/rubies.yml`; used by autobump. - `bin/package-linux`: Runs `bin/package` for a Linux target inside its pinned manylinux2014 container, mirroring the build workflow, for local Linux builds. - `bin/recipe-info`: Prints a version's effective series settings (`legacy`, `yjit`, ...) for workflows. +- `bin/build-fingerprint`: Prints a digest of everything in the repo that affects a version's build; the release workflow skips rebuilding when it matches the newest revision release's. ### Old series diff --git a/README.md b/README.md index 6bff797..3640bd6 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,13 @@ These Rubies use the first available certificate source in this order: [An automated release workflow is available to use](https://github.com/hatchboxio/precompiled-ruby/actions/workflows/release.yml). Dispatch the workflow with a Ruby version and it will build, upload SLSA provenance, publish an immutable build revision release (e.g. `3.4.7-2`), and then re-point the floating release (e.g. `3.4.7`) at that build. The floating release is updated in place rather than recreated, so its download URLs keep working while a rebuild is in flight or if one fails; new assets are renamed over the old ones rather than re-uploaded in place. +A dispatched release only builds when something that affects the build has changed. Each +revision release records a build fingerprint (`bin/build-fingerprint VERSION`: the version's +recipe, its effective series settings, dependency versions and checksums, the targets, the +packaging script and the build workflow), and a run whose fingerprint matches the newest +revision's exits after its first job. Tick `force` to rebuild anyway, for example when a +pinned container or the Rust toolchain is the reason. + [Release New Versions](https://github.com/hatchboxio/precompiled-ruby/actions/workflows/release-new.yml) dispatches that for every recipe that has no release yet; it runs on a schedule, when `recipes/rubies.yml` changes on `main`, or by hand, where its `only` input releases exactly diff --git a/bin/build-fingerprint b/bin/build-fingerprint new file mode 100755 index 0000000..7f53528 --- /dev/null +++ b/bin/build-fingerprint @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +exec ruby "$(dirname "$0")/../libexec/build_fingerprint.rb" "$@" diff --git a/libexec/build_fingerprint.rb b/libexec/build_fingerprint.rb new file mode 100755 index 0000000..df041f3 --- /dev/null +++ b/libexec/build_fingerprint.rb @@ -0,0 +1,65 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Print a digest of everything in this repository that determines what a Ruby version's +# build produces, so the release workflow can skip a rebuild when none of it changed. +# Covered: the version's recipe, its effective series settings (same merge as package.rb), +# every dependency's version and checksum, the release targets, the packaging script and +# the build workflow. Not covered: what the pinned containers, the Rust toolchain and the +# bootstrap Ruby resolve to at build time, so pass `force` to the release workflow when +# one of those is the reason to rebuild. +require "digest" +require "json" +require "yaml" + +ROOT = File.expand_path("..", __dir__) + +# Bump when the inputs below change shape, so older fingerprints never compare equal. +FORMAT = 1 + +HASHED_FILES = %w[ + libexec/package.rb + .github/workflows/build.yml +].freeze + +def load_yaml(path) + YAML.safe_load_file(File.join(ROOT, path), permitted_classes: [], aliases: false) +end + +def deep_merge(left, right) + left.merge(right) do |_key, old_value, new_value| + old_value.is_a?(Hash) && new_value.is_a?(Hash) ? deep_merge(old_value, new_value) : new_value + end +end + +def canonical(value) + case value + when Hash then value.sort_by { |key, _| key.to_s }.to_h { |key, item| [key.to_s, canonical(item)] } + when Array then value.map { |item| canonical(item) } + else value + end +end + +version = ARGV.fetch(0) { abort "Usage: bin/build-fingerprint VERSION" } + +rubies = load_yaml("recipes/rubies.yml").fetch("rubies") +recipe = rubies.fetch(version) { abort "Unknown Ruby version: #{version}" } +series_doc = load_yaml("recipes/series.yml") +series = deep_merge(series_doc.fetch("defaults"), series_doc.fetch("series").fetch(recipe.fetch("series")) || {}) +dependencies = load_yaml("recipes/dependencies.yml").fetch("dependencies").transform_values do |dependency| + dependency.slice("version", "sha256") +end +targets = load_yaml("recipes/targets.yml").fetch("targets") +files = HASHED_FILES.to_h { |path| [path, Digest::SHA256.file(File.join(ROOT, path)).hexdigest] } + +inputs = { + "format" => FORMAT, + "version" => version, + "recipe" => recipe, + "series" => series, + "dependencies" => dependencies, + "targets" => targets, + "files" => files +} + +puts "v#{FORMAT}-#{Digest::SHA256.hexdigest(JSON.generate(canonical(inputs)))}"