From 48be3083e2bb1b1ec12c05c1891673980f2f382a Mon Sep 17 00:00:00 2001 From: Alex Southwell Date: Tue, 29 Sep 2026 21:16:57 +1000 Subject: [PATCH] fix(nightly): stop retrying an input that already failed Fork Nightly and its Orchestrator v2 caller run every five minutes and on each upstream dispatch. A conflicting patch stack never produces a release, so every run rebuilt the same failing input: about 500 failure emails a month. A failed build now saves a cache marker keyed by channel and stack fingerprint, and later scheduled or dispatch runs skip that input with a notice. New upstream sources, manifest changes, and manual runs still build. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/DOWNSTREAM_NIGHTLY.md | 2 + .github/workflows/downstream-nightly.yml | 54 ++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/.github/DOWNSTREAM_NIGHTLY.md b/.github/DOWNSTREAM_NIGHTLY.md index 8a9e5a065fd2..45172eb961c4 100644 --- a/.github/DOWNSTREAM_NIGHTLY.md +++ b/.github/DOWNSTREAM_NIGHTLY.md @@ -16,6 +16,8 @@ PR heads and refs are pinned on purpose. If one moves, the workflow stops before If a selected commit is already part of the upstream Nightly, the assembler skips it. A real cherry-pick conflict stops the release and leaves the previous fork Nightly available. +A failed build records a cache marker for its release channel and stack fingerprint. Later scheduled and dispatch-triggered runs skip that same input with a notice instead of failing again. A new upstream source or any manifest change produces a new fingerprint and builds normally. To retry the same input, for example after a flaky runner, run the workflow manually. Markers expire when GitHub evicts the cache, currently after 7 days without access. + Normal scheduled and dispatch-triggered runs are idempotent. To repair artifacts without changing the upstream tag or patch stack, run the workflow manually with `force_rebuild` enabled. The workflow rebuilds every gate and replaces the matching release assets. Fork Nightly currently publishes macOS arm64 and macOS x64 desktop artifacts. The Linux, Windows, and WSL platform-specific steps remain available for later restoration, but they are not on the current release path. Ordinary Fork Nightly is complete when its merged macOS updater manifest is present. OV2 additionally requires both architectures' DMG and ZIP installers, the Apple Silicon server archive, and `SHA256SUMS`. diff --git a/.github/workflows/downstream-nightly.yml b/.github/workflows/downstream-nightly.yml index 87b593506d61..488f7ff5314c 100644 --- a/.github/workflows/downstream-nightly.yml +++ b/.github/workflows/downstream-nightly.yml @@ -82,7 +82,7 @@ jobs: permissions: contents: write outputs: - should_build: ${{ steps.plan.outputs.should_build }} + should_build: ${{ steps.gate.outputs.should_build }} upstream_repository: ${{ steps.plan.outputs.upstream_repository }} upstream_tag: ${{ steps.plan.outputs.upstream_tag }} release_channel: ${{ steps.plan.outputs.release_channel }} @@ -116,8 +116,34 @@ jobs: --force "$FORCE_REBUILD" \ --upstream-tag "$UPSTREAM_TAG_OVERRIDE" + # Scheduled and dispatch runs retry the same failing input every few + # minutes. record_failure stores a marker per input; manual runs retry. + - id: failed_before + name: Look up an earlier failure of this input + if: steps.plan.outputs.should_build == 'true' && github.event_name != 'workflow_dispatch' + uses: actions/cache/restore@v6 + with: + path: fork-nightly-failed + key: fork-nightly-failed-${{ steps.plan.outputs.release_channel }}-${{ steps.plan.outputs.fingerprint }} + lookup-only: true + + - id: gate + name: Skip an input that already failed + shell: bash + env: + SHOULD_BUILD: ${{ steps.plan.outputs.should_build }} + FAILED_BEFORE: ${{ steps.failed_before.outputs.cache-hit }} + TAG: ${{ steps.plan.outputs.tag }} + run: | + if [[ "$SHOULD_BUILD" == "true" && "$FAILED_BEFORE" == "true" ]]; then + echo "::notice::$TAG already failed with this upstream source and patch stack. Change the manifest or run the workflow manually to retry." + echo "should_build=false" >> "$GITHUB_OUTPUT" + else + echo "should_build=$SHOULD_BUILD" >> "$GITHUB_OUTPUT" + fi + - name: Checkout exact upstream source - if: steps.plan.outputs.should_build == 'true' + if: steps.gate.outputs.should_build == 'true' shell: bash env: UPSTREAM_REPOSITORY: ${{ steps.plan.outputs.upstream_repository }} @@ -132,14 +158,14 @@ jobs: - id: apply name: Apply selected patches - if: steps.plan.outputs.should_build == 'true' + if: steps.gate.outputs.should_build == 'true' run: | node automation/.github/scripts/downstream-nightly.mjs apply \ --plan "$RUNNER_TEMP/downstream-nightly-plan.json" \ --source source - name: Push generated source - if: steps.plan.outputs.should_build == 'true' + if: steps.gate.outputs.should_build == 'true' env: GH_TOKEN: ${{ github.token }} GENERATED_BRANCH: ${{ steps.plan.outputs.generated_branch }} @@ -696,3 +722,23 @@ jobs: --notes-file release-notes.md \ --prerelease fi + + record_failure: + name: Remember failed input + needs: [prepare, quality, build, publish] + if: failure() && needs.prepare.outputs.should_build == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Write failure marker + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p fork-nightly-failed + echo "$RUN_URL" > fork-nightly-failed/run-url + + - name: Save failure marker + uses: actions/cache/save@v6 + with: + path: fork-nightly-failed + key: fork-nightly-failed-${{ needs.prepare.outputs.release_channel }}-${{ needs.prepare.outputs.fingerprint }}