From da13a93fffa7d2ec337b32d761dcab14256399f4 Mon Sep 17 00:00:00 2001 From: Sil Date: Sun, 18 Oct 2026 19:55:26 +0200 Subject: [PATCH 1/2] Wait for matching previews before URL test setup --- .github/workflows/test-live.yml | 5 ++ .github/workflows/test.yml | 94 +++++++++++++++++++++------------ README.md | 6 +-- tests/preview.test.mjs | 40 +++++++++++++- 4 files changed, 107 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test-live.yml b/.github/workflows/test-live.yml index 9f066b6..939bed1 100644 --- a/.github/workflows/test-live.yml +++ b/.github/workflows/test-live.yml @@ -17,6 +17,10 @@ on: description: Optional JSON array of production origins and their Netlify config/base paths. Waits for this revision or unchanged build inputs. type: string default: "[]" + deploy-urls: + description: Optional JSON map of selected Netlify status contexts to production origins, passed to the repository tests. + type: string + default: "" http: type: boolean default: true @@ -37,6 +41,7 @@ jobs: pnpm_config_ignore_pnpmfile: "true" BASE_URL: ${{ inputs.base-url }} QA_PROFILE: live + QA_DEPLOY_URLS: ${{ inputs.deploy-urls }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c758c4..447ef56 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ on: type: boolean default: false web-timeout-minutes: - description: Maximum duration including preview waiting, setup and URL tests. + description: Maximum duration for setup and URL tests, after matching previews are ready. type: number default: 30 preview-checks: @@ -64,37 +64,15 @@ jobs: - name: Unit and service tests run: vp run test:unit - web: - name: Web + preview: + name: Preview if: ${{ !github.event.pull_request.draft && (inputs.http || inputs.browser) }} runs-on: ${{ inputs.runner }} - timeout-minutes: ${{ inputs.web-timeout-minutes }} - # Signed-in tests can share preview accounts across browser targets and PRs. - concurrency: - group: qa-web-${{ github.repository }} - cancel-in-progress: false - env: - pnpm_config_ignore_scripts: "true" - pnpm_config_ignore_pnpmfile: "true" - QA_PROFILE: preview + timeout-minutes: 15 + outputs: + base-url: ${{ steps.preview.outputs.base-url }} + deploy-urls: ${{ steps.preview.outputs.deploy-urls }} steps: - - name: Checkout the tested preview revision - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 - with: - version: "1.1.0" - node-version-file: ${{ inputs.node-version-file }} - cache: true - run-install: | - args: - - --frozen-lockfile - - --ignore-scripts - - name: Install desktop browsers - if: ${{ inputs.browser }} - run: vp exec playwright install --with-deps chromium firefox webkit - name: Resolve previews for this PR revision id: preview uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -143,18 +121,61 @@ jobs: await new Promise(resolve => setTimeout(resolve, 15000)); } throw new Error('Matching previews did not complete within 12 minutes.'); + + web: + name: Web + needs: preview + if: ${{ !github.event.pull_request.draft && (inputs.http || inputs.browser) }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.web-timeout-minutes }} + # Signed-in tests can share preview accounts across browser targets and PRs. + concurrency: + group: qa-web-${{ github.repository }} + cancel-in-progress: false + env: + pnpm_config_ignore_scripts: "true" + pnpm_config_ignore_pnpmfile: "true" + QA_PROFILE: preview + steps: + - name: Checkout the tested preview revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 + with: + version: "1.1.0" + node-version-file: ${{ inputs.node-version-file }} + cache: true + run-install: | + args: + - --frozen-lockfile + - --ignore-scripts + - name: Install desktop browsers + if: ${{ inputs.browser }} + run: vp exec playwright install --with-deps chromium firefox webkit + - name: Confirm the PR still matches the tested revision + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const {owner, repo} = context.repo; + const pull = context.payload.pull_request; + const {data: current} = await github.rest.pulls.get({owner, repo, pull_number: pull.number}); + if (current.state !== 'open' || current.draft || current.head.sha !== pull.head.sha) { + throw new Error('PR changed or returned to draft; discard this test run.'); + } - name: HTTP tests if: ${{ inputs.http }} env: - BASE_URL: ${{ steps.preview.outputs.base-url }} - QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }} + BASE_URL: ${{ needs.preview.outputs.base-url }} + QA_DEPLOY_URLS: ${{ needs.preview.outputs.deploy-urls }} run: vp run test:http - name: Browser tests id: browser if: ${{ inputs.browser }} env: - BASE_URL: ${{ steps.preview.outputs.base-url }} - QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }} + BASE_URL: ${{ needs.preview.outputs.base-url }} + QA_DEPLOY_URLS: ${{ needs.preview.outputs.deploy-urls }} TESTING_SECRET: ${{ secrets.testing-secret }} # A virtual display on Linux, for repositories that run a browser headed. shell: bash @@ -176,7 +197,7 @@ jobs: test: name: Test if: ${{ always() && !github.event.pull_request.draft }} - needs: [unit, web] + needs: [unit, preview, web] runs-on: ${{ inputs.runner }} timeout-minutes: 2 steps: @@ -185,12 +206,17 @@ jobs: UNIT_ENABLED: ${{ inputs.unit }} WEB_ENABLED: ${{ inputs.http || inputs.browser }} UNIT_RESULT: ${{ needs.unit.result }} + PREVIEW_RESULT: ${{ needs.preview.result }} WEB_RESULT: ${{ needs.web.result }} run: | if [[ "$UNIT_ENABLED" == true && "$UNIT_RESULT" != success ]]; then echo 'Enabled unit tests did not succeed.' >&2 exit 1 fi + if [[ "$WEB_ENABLED" == true && "$PREVIEW_RESULT" != success ]]; then + echo 'Required previews did not succeed.' >&2 + exit 1 + fi if [[ "$WEB_ENABLED" == true && "$WEB_RESULT" != success ]]; then echo 'Enabled URL tests did not succeed.' >&2 exit 1 diff --git a/README.md b/README.md index cec4dd3..cd379aa 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ Callers pin a full commit SHA from `main` after the shared workflow PR is merged New projects take their callers from [astro-template](https://github.com/fulldotdev/astro-template) or [vite-template](https://github.com/fulldotdev/vite-template). - **Explore** experiments locally; **Implement** builds the requested change locally. -- **Draft PR** builds locally, then opens a shared draft and a Netlify preview. Source checks and tests do not delay the preview. +- **Draft PR** saves a coherent checkpoint or provides a shared cloud preview. Reuse the task’s PR and batch local iterations. Source checks and tests do not delay the preview. - **Make PR ready** runs enabled checks and tests locally, then CI confirms them. Source checks and unit tests run alongside the Netlify build. URL tests use the preview of the tested PR head. - **Merge PR** merges the validated head and follows the production build and short, safe live checks. Fulldev, astro-template and vite-template expose `fix`, `check`, `build` and `test`. `check` validates source; `build` includes intrinsic artifact validation. Internal `test:unit`, `test:http` and `test:browser` helpers let CI run the enabled suites in the right environment. Devices stay manual. Do not add empty scripts or test workflows for disabled categories. -These callers use the Check workflow's optional `test-command` to run unit and service tests after source checks with one installation. Their Test caller disables its separate unit job. Browser dependencies install before waiting for the exact PR revision's previews. +These callers use the Check workflow's optional `test-command` to run unit and service tests after source checks with one installation. Their Test caller disables its separate unit job. A separate Preview job waits for the selected PR revision’s deployments before Web installs dependencies or acquires its shared-account lock. Waiting does not consume the Web test budget. -After a push to main, Check runs source checks and unit/service tests again, Netlify runs the same build command, and Test live waits for the published revision before safe live HTTP/browser tests. Its `deployments` input lists HTTPS origins and their repository-relative Netlify config/base paths. Each build publishes `/.well-known/build.json` with its `COMMIT_REF`. For an older production deploy, Test live accepts it only if it is an ancestor of the requested revision and that site's existing Netlify `build.ignore` command confirms unchanged inputs. No Netlify token or second build is needed. +After a push to main, Check runs source checks and unit/service tests again, Netlify runs the same build command, and Test live waits for the published revision before safe live HTTP/browser tests. Its `deployments` input lists HTTPS origins and their repository-relative Netlify config/base paths. Monorepos can supply a selected `deployments` list and `deploy-urls` map from their change detector. Each build publishes `/.well-known/build.json` with its `COMMIT_REF`. For an older production deploy, Test live accepts it only if it is an ancestor of the requested revision and that site's existing Netlify `build.ignore` command confirms unchanged inputs. No Netlify token or second build is needed. Other repositories retain their existing `check:source`, `check:build` and Test caller contracts until explicitly migrated. The reusable defaults remain compatible with them. Non-Node repositories keep their existing language-specific check command with dependency installation disabled. diff --git a/tests/preview.test.mjs b/tests/preview.test.mjs index e47e400..d9743d4 100644 --- a/tests/preview.test.mjs +++ b/tests/preview.test.mjs @@ -1,13 +1,31 @@ import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; import test from 'node:test'; +import {spawnSync} from 'node:child_process'; const source = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') - .split(' script: |\n')[1].split('\n - name: HTTP tests')[0] + .split(' script: |\n')[1].split('\n web:')[0] .split('\n').map(line => line.slice(12)).join('\n'); const execute = new (Object.getPrototypeOf(async function () {}).constructor)( 'github', 'context', 'core', 'process', 'Date', 'setTimeout', source); +const recheckSource = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') + .split(' - name: Confirm the PR still matches the tested revision\n')[1] + .split(' script: |\n')[1].split('\n - name: HTTP tests')[0] + .split('\n').map(line => line.slice(12)).join('\n'); +const recheck = new (Object.getPrototypeOf(async function () {}).constructor)( + 'github', 'context', recheckSource); + +test('URL tests reject a PR changed while the Web job waited for its lock', async () => { + const pull = {number: 7, state: 'open', draft: false, head: {sha: 'tested-head'}}; + const context = {repo: {owner: 'o', repo: 'r'}, payload: {pull_request: pull}}; + const github = current => ({rest: {pulls: {get: async () => ({data: current})}}}); + await recheck(github(pull), context); + for (const changes of [{draft: true}, {state: 'closed'}, {head: {sha: 'new-head'}}]) { + await assert.rejects(() => recheck(github({...pull, ...changes}), context), /PR changed/); + } +}); + async function scenario(options = {}) { const pull = {number: 7, state: 'open', draft: false, head: {sha: 'tested-head'}}; const names = options.names ?? ['netlify/app/deploy-preview']; @@ -62,3 +80,23 @@ test('production, another PR and untrusted URLs are rejected', async () => { } await assert.rejects(scenario({names: []}), /require preview-checks/); }); + +// Exercise the actual aggregate with upstream outcomes: skipped prerequisites +// must never turn an enabled URL suite into a successful merge gate. +const gate = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') + .split(' - name: Require every enabled category\n')[1].split(' run: |\n')[1] + .split('\n').map(line => line.slice(10)).join('\n'); +function aggregate(overrides = {}) { + return spawnSync('bash', ['-e', '-c', gate], {env: {...process.env, + UNIT_ENABLED: 'false', WEB_ENABLED: 'true', UNIT_RESULT: 'skipped', + PREVIEW_RESULT: 'success', WEB_RESULT: 'success', ...overrides}}).status; +} +test('enabled URL tests require successful preview and Web jobs', () => { + assert.equal(aggregate(), 0); + for (const result of ['skipped', 'failure', 'cancelled']) { + assert.notEqual(aggregate({PREVIEW_RESULT: result}), 0); + assert.notEqual(aggregate({WEB_RESULT: result}), 0); + } + assert.equal(aggregate({WEB_ENABLED: 'false', PREVIEW_RESULT: 'skipped', WEB_RESULT: 'skipped'}), 0); + assert.notEqual(aggregate({UNIT_ENABLED: 'true'}), 0); +}); From 215bc28e33379215c62cc168132d8cd11e0e7300 Mon Sep 17 00:00:00 2001 From: Sil Date: Sun, 18 Oct 2026 20:06:04 +0200 Subject: [PATCH 2/2] Select and verify affected previews with native Netlify ignore rules --- .github/workflows/test-live.yml | 5 -- .github/workflows/test.yml | 135 +++++++++++++++++++++++++++++--- README.md | 6 +- tests/preview.test.mjs | 134 ++++++++++++++++++++++++++++--- 4 files changed, 253 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-live.yml b/.github/workflows/test-live.yml index 939bed1..9f066b6 100644 --- a/.github/workflows/test-live.yml +++ b/.github/workflows/test-live.yml @@ -17,10 +17,6 @@ on: description: Optional JSON array of production origins and their Netlify config/base paths. Waits for this revision or unchanged build inputs. type: string default: "[]" - deploy-urls: - description: Optional JSON map of selected Netlify status contexts to production origins, passed to the repository tests. - type: string - default: "" http: type: boolean default: true @@ -41,7 +37,6 @@ jobs: pnpm_config_ignore_pnpmfile: "true" BASE_URL: ${{ inputs.base-url }} QA_PROFILE: live - QA_DEPLOY_URLS: ${{ inputs.deploy-urls }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 447ef56..0c0c43c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,10 @@ on: description: JSON array of Netlify status contexts; the first is BASE_URL. type: string default: "[]" + deployments: + description: Optional JSON array of context, production url, Netlify config and base. Uses each build.ignore to select affected previews and verify unchanged builds. + type: string + default: "[]" node-version-file: type: string default: .node-version @@ -72,23 +76,98 @@ jobs: outputs: base-url: ${{ steps.preview.outputs.base-url }} deploy-urls: ${{ steps.preview.outputs.deploy-urls }} + deploy-revisions: ${{ steps.preview.outputs.deploy-revisions }} + web-enabled: ${{ steps.preview.outputs.web-enabled }} steps: + - name: Read the PR build inputs + if: ${{ inputs.deployments != '[]' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false - name: Resolve previews for this PR revision id: preview uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: PREVIEW_CHECKS: ${{ inputs.preview-checks }} + DEPLOYMENTS: ${{ inputs.deployments }} with: script: | const {owner, repo} = context.repo; const pull = context.payload.pull_request; if (!pull) throw new Error('URL tests require a pull_request event.'); - const names = JSON.parse(process.env.PREVIEW_CHECKS); + const sha = pull.head.sha; + const targets = JSON.parse(process.env.DEPLOYMENTS || '[]'); + if (!Array.isArray(targets)) throw new Error('deployments must be an array.'); + let names = JSON.parse(process.env.PREVIEW_CHECKS); + let selected = []; + let changed; + let compatible; + if (targets.length) { + const {execFileSync, spawnSync} = require('node:child_process'); + const {resolve, sep} = require('node:path'); + const root = process.env.GITHUB_WORKSPACE; + const inside = path => { + const absolute = resolve(root, path); + if (absolute !== root && !absolute.startsWith(root + sep)) throw new Error('Build paths must stay in the checkout.'); + return absolute; + }; + const git = args => execFileSync('git', args, {cwd: root, encoding: 'utf8', timeout: 10000, stdio: ['ignore', 'pipe', 'pipe']}).trim(); + const configs = new Map(); + for (const site of targets) { + if (!/^netlify\/[a-z0-9-]+\/deploy-preview$/.test(site.context) + || !/^https:\/\/[a-z0-9.-]+$/.test(site.url) || configs.has(site.context)) { + throw new Error('deployments require unique Netlify contexts and HTTPS production origins.'); + } + const ignore = execFileSync('python3', ['-c', + 'import sys,tomllib; print(tomllib.load(open(sys.argv[1], "rb")).get("build", {}).get("ignore", ""))', + inside(site.config)], {encoding: 'utf8', timeout: 10000}).trim(); + if (!ignore) throw new Error(`${site.context} needs a build.ignore rule.`); + configs.set(site.context, {ignore, cwd: inside(site.base || '.')}); + } + changed = (site, from, to) => { + const config = configs.get(site.context); + const result = spawnSync('bash', ['-e', '-o', 'pipefail', '-c', config.ignore], { + cwd: config.cwd, timeout: 10000, + env: {...process.env, CACHED_COMMIT_REF: from, COMMIT_REF: to}, + }); + if (![0, 1].includes(result.status)) throw new Error(`${site.context}: build.ignore failed.`); + return result.status === 1; + }; + let baseline; + try { baseline = git(['merge-base', pull.base.sha, sha]); } + catch { core.info('No PR baseline available; test every deployment.'); } + // Test changes must exercise the new tests on every declared target. + const testChanges = !baseline || git(['diff', '--name-only', baseline, sha, '--', 'tests', '.github/workflows/test.yml']); + selected = targets.filter(site => testChanges || changed(site, baseline, sha)); + names = selected.map(site => site.context); + compatible = (site, revision) => { + if (!/^[0-9a-f]{40}$/.test(revision || '')) return false; + if (revision === sha) return true; + try { git(['merge-base', '--is-ancestor', revision, sha]); } + catch { return false; } + return !changed(site, revision, sha); + }; + core.info(`Affected previews: ${names.join(', ') || 'none'}`); + if (!names.length) { + core.setOutput('web-enabled', 'false'); + core.setOutput('deploy-urls', '{}'); + return; + } + } if (!Array.isArray(names) || !names.length || names.some(name => typeof name !== 'string' || !/^netlify\/[a-z0-9-]+\/deploy-preview$/.test(name))) { throw new Error('Enabled URL tests require preview-checks.'); } - const sha = pull.head.sha; + async function revisionAt(origin) { + try { + const response = await fetch(`${origin}/.well-known/build.json?revision=${sha}`, { + headers: {'Cache-Control': 'no-cache'}, redirect: 'error', signal: AbortSignal.timeout(10000), + }); + return response.ok ? (await response.json()).revision : undefined; + } catch { return undefined; } + } const deadline = Date.now() + 12 * 60 * 1000; while (Date.now() < deadline) { const {data: current} = await github.rest.pulls.get({owner, repo, pull_number: pull.number}); @@ -97,24 +176,44 @@ jobs: } const {data} = await github.rest.repos.getCombinedStatusForRef({owner, repo, ref: sha, per_page: 100}); const urls = {}; - for (const name of names) { + const revisions = {}; + await Promise.all(names.map(async name => { const status = data.statuses.find(status => status.context === name); if (['error', 'failure'].includes(status?.state)) { throw new Error(`${name} failed for ${sha}.`); } - if (status?.state !== 'success') continue; + if (status?.state !== 'success') return; const url = new URL(status.target_url); const site = name.split('/')[1]; - if (url.protocol !== 'https:' || url.username || url.password - || url.hostname !== `deploy-preview-${pull.number}--${site}.netlify.app`) { + const origin = `https://deploy-preview-${pull.number}--${site}.netlify.app`; + const built = url.origin === origin; + const skipped = targets.length && url.hostname === 'app.netlify.com' + && /^\/(?:sites|projects)\/[^/]+\/deploys\/[^/]+\/?$/.test(url.pathname); + if (url.protocol !== 'https:' || url.username || url.password || !(built || skipped)) { throw new Error(`${name} did not return this PR's Netlify preview URL.`); } - urls[name] = url.origin; - } + if (!targets.length) { urls[name] = origin; return; } + const target = selected.find(site => site.context === name); + const revision = await revisionAt(origin); + if (built ? revision === sha : compatible(target, revision)) { + urls[name] = origin; + revisions[origin] = revision; + } else if (skipped && target['production-fallback'] === true) { + // Static websites may have no first preview when their build was skipped. + // Signed-in apps must always use their isolated preview. + const production = await revisionAt(target.url); + if (compatible(target, production)) { + urls[name] = target.url; + revisions[target.url] = production; + } + } + })); if (names.every(name => urls[name])) { core.setOutput('base-url', urls[names[0]]); core.setOutput('deploy-urls', JSON.stringify(urls)); + core.setOutput('deploy-revisions', JSON.stringify(revisions)); core.setOutput('sha', sha); + core.setOutput('web-enabled', 'true'); return; } core.info(`Waiting for matching previews: ${names.filter(name => !urls[name]).join(', ')}`); @@ -125,7 +224,7 @@ jobs: web: name: Web needs: preview - if: ${{ !github.event.pull_request.draft && (inputs.http || inputs.browser) }} + if: ${{ !github.event.pull_request.draft && needs.preview.outputs.web-enabled == 'true' }} runs-on: ${{ inputs.runner }} timeout-minutes: ${{ inputs.web-timeout-minutes }} # Signed-in tests can share preview accounts across browser targets and PRs. @@ -156,6 +255,8 @@ jobs: run: vp exec playwright install --with-deps chromium firefox webkit - name: Confirm the PR still matches the tested revision uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + DEPLOY_REVISIONS: ${{ needs.preview.outputs.deploy-revisions }} with: script: | const {owner, repo} = context.repo; @@ -164,6 +265,15 @@ jobs: if (current.state !== 'open' || current.draft || current.head.sha !== pull.head.sha) { throw new Error('PR changed or returned to draft; discard this test run.'); } + const revisions = JSON.parse(process.env.DEPLOY_REVISIONS || '{}'); + await Promise.all(Object.entries(revisions).map(async ([origin, revision]) => { + const response = await fetch(`${origin}/.well-known/build.json?revision=${pull.head.sha}`, { + headers: {'Cache-Control': 'no-cache'}, redirect: 'error', signal: AbortSignal.timeout(10000), + }); + if (!response.ok || (await response.json()).revision !== revision) { + throw new Error(`${origin} changed while Web waited; rerun Test to verify its build inputs.`); + } + })); - name: HTTP tests if: ${{ inputs.http }} env: @@ -205,6 +315,7 @@ jobs: env: UNIT_ENABLED: ${{ inputs.unit }} WEB_ENABLED: ${{ inputs.http || inputs.browser }} + WEB_SELECTED: ${{ needs.preview.outputs.web-enabled }} UNIT_RESULT: ${{ needs.unit.result }} PREVIEW_RESULT: ${{ needs.preview.result }} WEB_RESULT: ${{ needs.web.result }} @@ -217,7 +328,11 @@ jobs: echo 'Required previews did not succeed.' >&2 exit 1 fi - if [[ "$WEB_ENABLED" == true && "$WEB_RESULT" != success ]]; then + if [[ "$WEB_ENABLED" == true && "$WEB_SELECTED" != true && "$WEB_SELECTED" != false ]]; then + echo 'Preview selection did not complete.' >&2 + exit 1 + fi + if [[ "$WEB_ENABLED" == true && "$WEB_SELECTED" == true && "$WEB_RESULT" != success ]]; then echo 'Enabled URL tests did not succeed.' >&2 exit 1 fi diff --git a/README.md b/README.md index cd379aa..d3dfaa2 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,13 @@ Fulldev, astro-template and vite-template expose `fix`, `check`, `build` and `te These callers use the Check workflow's optional `test-command` to run unit and service tests after source checks with one installation. Their Test caller disables its separate unit job. A separate Preview job waits for the selected PR revision’s deployments before Web installs dependencies or acquires its shared-account lock. Waiting does not consume the Web test budget. -After a push to main, Check runs source checks and unit/service tests again, Netlify runs the same build command, and Test live waits for the published revision before safe live HTTP/browser tests. Its `deployments` input lists HTTPS origins and their repository-relative Netlify config/base paths. Monorepos can supply a selected `deployments` list and `deploy-urls` map from their change detector. Each build publishes `/.well-known/build.json` with its `COMMIT_REF`. For an older production deploy, Test live accepts it only if it is an ancestor of the requested revision and that site's existing Netlify `build.ignore` command confirms unchanged inputs. No Netlify token or second build is needed. +Monorepos can give Test a `deployments` array of Netlify `context`, production `url`, repository-relative `config` and optional `base`. Preview selects targets by running each native `build.ignore` against the cumulative PR diff. Changes in `tests/` or `.github/workflows/test.yml` select every target. Netlify uses the same ignore rule against its cached revision. Skipped previews are reusable only when `/.well-known/build.json` proves an ancestor with unchanged build inputs. Static websites may explicitly enable `production-fallback: true` for a skipped first preview; signed-in apps always require an isolated preview. `QA_DEPLOY_URLS` contains only selected targets, and repositories must exclude absent targets from their suites. An explicitly empty selection skips Web and passes Test; missing or failed selection never does. + +After a push to main, Check runs source checks and unit/service tests again, Netlify runs the same build command, and Test live waits for the published revision before safe live HTTP/browser tests. Its `deployments` input lists HTTPS origins and their repository-relative Netlify config/base paths. Each build publishes `/.well-known/build.json` with its `COMMIT_REF`. For an older production deploy, Test live accepts it only if it is an ancestor of the requested revision and that site's existing Netlify `build.ignore` command confirms unchanged inputs. No Netlify token or second build is needed. Other repositories retain their existing `check:source`, `check:build` and Test caller contracts until explicitly migrated. The reusable defaults remain compatible with them. Non-Node repositories keep their existing language-specific check command with dependency installation disabled. -Repositories with `test.yml` require `Test / Test` through the existing `Review` status. Missing, skipped, failed or superseded test results cannot pass that gate. Independent agent review owns code assessment; Ready has no external review-service dependency. +Repositories with `test.yml` require `Test / Test` through the existing `Review` status. Missing, failed or superseded enabled results cannot pass that gate. Skipped URL work passes only for an explicitly empty selection. Independent agent review owns code assessment; Ready has no external review-service dependency. The Ready caller grants `actions: read` so the gate reads the current Test workflow attempt and its jobs, including reruns. diff --git a/tests/preview.test.mjs b/tests/preview.test.mjs index d9743d4..2280c25 100644 --- a/tests/preview.test.mjs +++ b/tests/preview.test.mjs @@ -1,33 +1,48 @@ import assert from 'node:assert/strict'; -import {readFileSync} from 'node:fs'; +import {mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {createRequire} from 'node:module'; import test from 'node:test'; -import {spawnSync} from 'node:child_process'; +import {execFileSync, spawnSync} from 'node:child_process'; const source = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') .split(' script: |\n')[1].split('\n web:')[0] .split('\n').map(line => line.slice(12)).join('\n'); const execute = new (Object.getPrototypeOf(async function () {}).constructor)( - 'github', 'context', 'core', 'process', 'Date', 'setTimeout', source); + 'github', 'context', 'core', 'process', 'Date', 'setTimeout', 'require', 'fetch', source); const recheckSource = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') .split(' - name: Confirm the PR still matches the tested revision\n')[1] .split(' script: |\n')[1].split('\n - name: HTTP tests')[0] .split('\n').map(line => line.slice(12)).join('\n'); const recheck = new (Object.getPrototypeOf(async function () {}).constructor)( - 'github', 'context', recheckSource); + 'github', 'context', 'process', 'fetch', recheckSource); test('URL tests reject a PR changed while the Web job waited for its lock', async () => { const pull = {number: 7, state: 'open', draft: false, head: {sha: 'tested-head'}}; const context = {repo: {owner: 'o', repo: 'r'}, payload: {pull_request: pull}}; const github = current => ({rest: {pulls: {get: async () => ({data: current})}}}); - await recheck(github(pull), context); + await recheck(github(pull), context, {env: {}}); for (const changes of [{draft: true}, {state: 'closed'}, {head: {sha: 'new-head'}}]) { - await assert.rejects(() => recheck(github({...pull, ...changes}), context), /PR changed/); + await assert.rejects(() => recheck(github({...pull, ...changes}), context, {env: {}}), /PR changed/); } }); -async function scenario(options = {}) { +test('URL tests revalidate approved alias and production revisions after waiting for the Web lock', async () => { const pull = {number: 7, state: 'open', draft: false, head: {sha: 'tested-head'}}; + const context = {repo: {owner: 'o', repo: 'r'}, payload: {pull_request: pull}}; + const github = {rest: {pulls: {get: async () => ({data: pull})}}}; + for (const origin of ['https://deploy-preview-7--site.netlify.app', 'https://site.test']) { + const env = {DEPLOY_REVISIONS: JSON.stringify({[origin]: 'approved-revision'})}; + const fetchRevision = revision => async () => ({ok: true, json: async () => ({revision})}); + await recheck(github, context, {env}, fetchRevision('approved-revision')); + await assert.rejects(() => recheck(github, context, {env}, fetchRevision('another-revision')), /changed while Web waited/); + } +}); + +async function scenario(options = {}) { + const pull = {number: 7, state: 'open', draft: false, head: {sha: options.head ?? 'tested-head'}, base: {sha: options.base}}; const names = options.names ?? ['netlify/app/deploy-preview']; const outputs = {}; const refs = []; @@ -42,8 +57,9 @@ async function scenario(options = {}) { }}; await execute(github, {repo: {owner: 'o', repo: 'r'}, payload: {pull_request: pull}}, {info() {}, setOutput: (name, value) => { outputs[name] = value; }}, - {env: {PREVIEW_CHECKS: JSON.stringify(names)}}, - {now: () => clock}, callback => { clock += 60 * 1000; callback(); }); + {env: {...process.env, PREVIEW_CHECKS: JSON.stringify(names), DEPLOYMENTS: JSON.stringify(options.targets ?? []), GITHUB_WORKSPACE: options.root}}, + {now: () => clock}, callback => { clock += 60 * 1000; callback(); }, + createRequire(import.meta.url), async url => ({ok: true, json: async () => ({revision: options.revisions?.[new URL(url).origin] ?? options.head})})); return {outputs, refs}; } @@ -81,6 +97,102 @@ test('production, another PR and untrusted URLs are rejected', async () => { await assert.rejects(scenario({names: []}), /require preview-checks/); }); +// Real git history and native ignore commands protect selection and reuse from +// drifting apart. Nothing from the fixture is published or installed. +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'qa-preview-')); + t.after(() => rmSync(root, {recursive: true, force: true})); + const git = (...args) => execFileSync('git', args, {cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}).trim(); + const write = (path, value) => { + mkdirSync(join(root, path, '..'), {recursive: true}); + writeFileSync(join(root, path), value); + }; + const commit = (path, value) => { + if (path) write(path, value); + git('add', '.'); + git('-c', 'user.name=QA', '-c', 'user.email=qa@example.test', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture'); + return git('rev-parse', 'HEAD'); + }; + git('init', '-q'); + const targets = ['app', 'site'].map(site => { + const paths = site === 'app' ? '. ../../packages/shared ../../root.json' : 'apps/site packages/shared root.json'; + write(`apps/${site}/netlify.toml`, `[build]\nignore = 'test -n "$CACHED_COMMIT_REF" && test "$CACHED_COMMIT_REF" != "$COMMIT_REF" && git diff --quiet "$CACHED_COMMIT_REF" "$COMMIT_REF" -- ${paths}'\n`); + write(`apps/${site}/source.txt`, 'original'); + return {context: `netlify/${site}/deploy-preview`, config: `apps/${site}/netlify.toml`, url: `https://${site}.test`, ...(site === 'site' ? {'production-fallback': true} : {base: 'apps/app'})}; + }); + write('packages/shared/source.txt', 'shared'); + const base = commit(); + return {root, targets, names: targets.map(site => site.context), base, write, commit}; +} + +const skipped = context => ({context, state: 'success', target_url: 'https://app.netlify.com/sites/example/deploys/canceled'}); +const appAlias = 'https://deploy-preview-7--app.netlify.app'; +const siteAlias = 'https://deploy-preview-7--site.netlify.app'; + +test('native ignore rules select the changed site and shared dependents', async t => { + const f = fixture(t); + let head = f.commit('apps/site/source.txt', 'website changed'); + const result = await scenario({...f, head}); + assert.deepEqual(Object.keys(JSON.parse(result.outputs['deploy-urls'])), ['netlify/site/deploy-preview']); + head = f.commit('packages/shared/source.txt', 'shared changed'); + const shared = await scenario({...f, head}); + assert.equal(Object.keys(JSON.parse(shared.outputs['deploy-urls'])).length, 2); +}); + +test('renaming across project folders and deleting source keeps both affected sites selected', async t => { + const f = fixture(t); + rmSync(join(f.root, 'apps/app/source.txt')); + const head = f.commit('apps/site/moved.txt', 'original'); + assert.equal(Object.keys(JSON.parse((await scenario({...f, head})).outputs['deploy-urls'])).length, 2); +}); + +test('a later unrelated push still tests the cumulative PR change on a proven unchanged preview', async t => { + const f = fixture(t); + const revision = f.commit('apps/app/source.txt', 'app changed'); + const head = f.commit('README.md', 'later unrelated push'); + const result = await scenario({...f, head, statuses: [skipped('netlify/app/deploy-preview')], revisions: {[appAlias]: revision}}); + assert.deepEqual(JSON.parse(result.outputs['deploy-urls']), {'netlify/app/deploy-preview': appAlias}); +}); + +test('a skipped or manually canceled changed app cannot reuse stale code or production', async t => { + const f = fixture(t); + const head = f.commit('apps/app/source.txt', 'app changed'); + await assert.rejects(scenario({...f, head, statuses: [skipped('netlify/app/deploy-preview')], revisions: {[appAlias]: f.base}}), /within 12 minutes/); + await assert.rejects(scenario({...f, head, revisions: {[appAlias]: f.base}}), /within 12 minutes/); +}); + +test('only explicitly read-only sites may reuse matching production when no first preview exists', async t => { + const f = fixture(t); + const revision = f.commit('apps/site/source.txt', 'site changed'); + const head = f.commit('README.md', 'later unrelated push'); + const options = {...f, head, statuses: [skipped('netlify/site/deploy-preview')], revisions: {[siteAlias]: 'missing', 'https://site.test': revision}}; + const result = await scenario(options); + assert.equal(JSON.parse(result.outputs['deploy-urls'])['netlify/site/deploy-preview'], 'https://site.test'); + await assert.rejects(scenario({...options, targets: f.targets.map(site => ({...site, 'production-fallback': false}))}), /within 12 minutes/); +}); + +test('changed tests exercise all targets and missing baselines fail safe to every target', async t => { + const f = fixture(t); + const head = f.commit('tests/http/example.spec.ts', 'new test'); + assert.equal(Object.keys(JSON.parse((await scenario({...f, head})).outputs['deploy-urls'])).length, 2); + assert.equal(Object.keys(JSON.parse((await scenario({...f, head, base: 'missing-base'})).outputs['deploy-urls'])).length, 2); +}); + +test('an explicit empty scope skips URL work without waiting for unused statuses', async t => { + const f = fixture(t); + const head = f.commit('README.md', 'documentation only'); + const result = await scenario({...f, head, statuses: []}); + assert.equal(result.outputs['web-enabled'], 'false'); + assert.equal(result.outputs['deploy-urls'], '{}'); + assert.deepEqual(result.refs, []); +}); + +test('native ignore failures cannot silently remove a target from required tests', async t => { + const f = fixture(t); + const head = f.commit('apps/app/netlify.toml', "[build]\nignore = 'exit 2'\n"); + await assert.rejects(scenario({...f, head}), /build.ignore failed/); +}); + // Exercise the actual aggregate with upstream outcomes: skipped prerequisites // must never turn an enabled URL suite into a successful merge gate. const gate = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') @@ -89,7 +201,7 @@ const gate = readFileSync(new URL('../.github/workflows/test.yml', import.meta.u function aggregate(overrides = {}) { return spawnSync('bash', ['-e', '-c', gate], {env: {...process.env, UNIT_ENABLED: 'false', WEB_ENABLED: 'true', UNIT_RESULT: 'skipped', - PREVIEW_RESULT: 'success', WEB_RESULT: 'success', ...overrides}}).status; + PREVIEW_RESULT: 'success', WEB_RESULT: 'success', WEB_SELECTED: 'true', ...overrides}}).status; } test('enabled URL tests require successful preview and Web jobs', () => { assert.equal(aggregate(), 0); @@ -98,5 +210,7 @@ test('enabled URL tests require successful preview and Web jobs', () => { assert.notEqual(aggregate({WEB_RESULT: result}), 0); } assert.equal(aggregate({WEB_ENABLED: 'false', PREVIEW_RESULT: 'skipped', WEB_RESULT: 'skipped'}), 0); + assert.equal(aggregate({WEB_SELECTED: 'false', WEB_RESULT: 'skipped'}), 0); + assert.notEqual(aggregate({WEB_SELECTED: '', WEB_RESULT: 'skipped'}), 0); assert.notEqual(aggregate({UNIT_ENABLED: 'true'}), 0); });