From fd1fb8e2751bd61a0c2848aceb4a0422e1b003d2 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Sat, 8 Aug 2026 15:32:41 +0200 Subject: [PATCH 1/2] fix(e2e): retain-on-failure traces + a globalTimeout under the 45m CI cap Fleet-wide Playwright instrument sweep, ConductionNL/.github#188. Neither change can alter a verdict; both change whether you can see why a verdict happened. `trace: 'on-first-retry'` only writes a trace when a retry actually happens, which makes the trace artifact a function of `retries`. `retain-on-failure` captures every test, keeps only the failures, and does not depend on the retry count. No repo in the fleet set `globalTimeout`. The shared quality.yml Playwright job is `timeout-minutes: 45`, and a job cancelled by that cap produces no verdict and no artifacts: the trace upload is `if: failure()` and the report upload is `if: always()`, and neither runs on a cancelled job, while `gh pr checks` still renders it as "fail". Runs cancelled at ~45m16s have been observed in this fleet. Measured overhead in that job before the `Run Playwright tests` step starts is 2.0-2.4 min, so 38m leaves ~7 min of margin while guaranteeing a tally and its artifacts. --- playwright.config.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index 9ee537c5..76937456 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -24,12 +24,31 @@ export default defineConfig({ forbidOnly: !!process.env.CI, retries: process.env.CI ? 1 : 0, workers: 1, + // The shared quality.yml Playwright job is `timeout-minutes: 45`, and a job + // cancelled by that cap produces NO verdict: Playwright never prints its + // tally, the `if: failure()` trace upload never fires, and the + // `if: always()` report upload does not run on a cancelled job either — the + // run you most need to read is the one that leaves nothing behind, and it + // still renders as "fail" in `gh pr checks` while carrying no information. + // Runs cancelled at ~45m16s have been observed in this fleet. Measured + // overhead before `Run Playwright tests` starts is 2.0-2.4 min and the + // uploads after it take seconds, so 38m keeps ~7 min of margin while + // guaranteeing both a tally and the artifacts that explain it. + globalTimeout: 38 * 60_000, reporter: 'list', outputDir: 'test-results', use: { baseURL: resolveBaseUrl(), storageState: path.resolve(__dirname, 'tests/e2e/.auth/admin.json'), - trace: 'on-first-retry', + // `on-first-retry` writes a trace only when a retry actually happens, so + // the trace artifact is a function of `retries`. Off CI `retries` is 0 + // above, so a local failure has never produced a trace at all; on CI it + // traces the SECOND attempt only, which means the failure that does not + // reproduce — the one actually worth a trace — leaves no record of the + // attempt that failed. `retain-on-failure` traces every attempt and + // keeps the ones that failed: strictly more informative, and + // independent of the retry count. + trace: 'retain-on-failure', screenshot: 'only-on-failure', }, projects: [ From 9809998197d2ea0a01683e3425c37a633d40ffc7 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Sat, 8 Aug 2026 15:42:09 +0200 Subject: [PATCH 2/2] fix(e2e): apply the same fix to the config CI actually loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared quality.yml resolves its config as `${playwright-test-path}/playwright.config.ts` and only falls back to the app-root `playwright.config.ts` when that file is absent (quality.yml ~L2218). This repo ships tests/e2e/playwright.config.ts, so THAT is the file every CI run has been using — the app-root config fixed in the previous commit is the one developers load by hand, not the one the gate reads. Applies the identical `retain-on-failure` + `globalTimeout: 38 * 60_000` change here. ConductionNL/.github#188. --- tests/e2e/playwright.config.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index 3e913471..11487b10 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -71,6 +71,17 @@ export default defineConfig({ forbidOnly: !!process.env.CI, retries: process.env.CI ? 1 : 0, workers: 1, + // The shared quality.yml Playwright job is `timeout-minutes: 45`, and a job + // cancelled by that cap produces NO verdict: Playwright never prints its + // tally, the `if: failure()` trace upload never fires, and the + // `if: always()` report upload does not run on a cancelled job either — the + // run you most need to read is the one that leaves nothing behind, and it + // still renders as "fail" in `gh pr checks` while carrying no information. + // Runs cancelled at ~45m16s have been observed in this fleet. Measured + // overhead before `Run Playwright tests` starts is 2.0-2.4 min and the + // uploads after it take seconds, so 38m keeps ~7 min of margin while + // guaranteeing both a tally and the artifacts that explain it. + globalTimeout: 38 * 60_000, reporter: [ ['html', { open: 'never', outputFolder: path.join(APP_ROOT, 'playwright-report') }], ['list'], @@ -80,7 +91,15 @@ export default defineConfig({ use: { baseURL: BASE_URL, storageState: path.resolve(__dirname, '.auth', 'admin.json'), - trace: 'on-first-retry', + // `on-first-retry` writes a trace only when a retry actually happens, so + // the trace artifact is a function of `retries`. Off CI `retries` is 0 + // above, so a local failure has never produced a trace at all; on CI it + // traces the SECOND attempt only, which means the failure that does not + // reproduce — the one actually worth a trace — leaves no record of the + // attempt that failed. `retain-on-failure` traces every attempt and + // keeps the ones that failed: strictly more informative, and + // independent of the retry count. + trace: 'retain-on-failure', screenshot: 'only-on-failure', },