Skip to content

Commit b4695e3

Browse files
authored
devops: triage labeled issues end to end, up to a draft PR (#43614)
1 parent 7d001dc commit b4695e3

8 files changed

Lines changed: 704 additions & 453 deletions

File tree

‎.claude/skills/playwright-triage/SKILL.md‎

Lines changed: 145 additions & 114 deletions
Large diffs are not rendered by default.

.github/workflows/bot-voice.md renamed to .claude/skills/playwright-triage/bot-voice.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ test('networkidle resolves with an open EventSource', { annotation: { type: 'iss
8383
- variations: headed and headless; `goto({waitUntil:'networkidle'})` and
8484
`waitForLoadState('networkidle')` — both hang; closing the stream server-side lets it resolve
8585
- raw `npx playwright test` output …
86-
- full run: <link to the GitHub Actions workflow run, when available>
8786
</details>
8887
~~~
8988

‎.claude/skills/playwright-triage/triage.md‎

Lines changed: 314 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/pr-ci-triage.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ jobs:
5252
PROMPT=$(cat <<PROMPT_EOF
5353
Come up with a verdict on the CI test failures on pull request #${PR_NUMBER} of ${TARGET_REPO}: are they likely caused by this PR? Follow the pr-ci-triage guide (.github/workflows/pr-ci-triage.md) for the analysis.
5454
55-
Write your verdict to output/triage.md as a PR comment, in the format the guide specifies and the bot voice register (.github/workflows/bot-voice.md).
55+
Write your verdict to output/triage.md as a PR comment, in the format the guide specifies and the bot voice register (.claude/skills/playwright-triage/bot-voice.md).
5656
5757
Do not post anything yourself — a later workflow step posts output/triage.md.
5858
PROMPT_EOF
@@ -63,7 +63,7 @@ jobs:
6363
--no-ask-user \
6464
--enable-all-github-mcp-tools \
6565
--share=output/copilot-session.md \
66-
--model gpt-5.6-sol \
66+
--model gpt-6.1-sol \
6767
--max-ai-credits 500 \
6868
-p "$PROMPT"
6969

‎.github/workflows/triage.yml‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
# Copilot triages and fixes the issue without write credentials. Fresh jobs open
2+
# the completed fix as a draft PR using the app, so CI runs on it, and comment on the issue.
3+
name: Issue triage
4+
5+
on:
6+
# Apply needs-triage-linux, needs-triage-mac or needs-triage-win; re-apply it to trigger another pass.
7+
issues:
8+
types: [labeled]
9+
workflow_dispatch:
10+
inputs:
11+
issue:
12+
description: "Issue number to triage"
13+
type: string
14+
required: true
15+
platform:
16+
description: "Platform to triage on"
17+
type: choice
18+
required: true
19+
default: linux
20+
options:
21+
- linux
22+
- mac
23+
- windows
24+
task:
25+
description: "Optional task to direct the triage, such as what to focus on"
26+
type: string
27+
required: false
28+
default: ""
29+
30+
permissions: {}
31+
32+
jobs:
33+
triage:
34+
name: Triage issue ${{ github.event.issue.number || inputs.issue }} (${{ inputs.platform || github.event.label.name }})
35+
if: >-
36+
github.repository == 'microsoft/playwright' &&
37+
(github.event_name == 'workflow_dispatch' ||
38+
contains(fromJSON('["needs-triage-linux","needs-triage-mac","needs-triage-win"]'), github.event.label.name))
39+
# Falls back to a runner name for unrelated labels, where the job is skipped anyway.
40+
runs-on: ${{ fromJSON('{"linux":"playwright-x64-ubuntu24-64-core","mac":"macos-15-xlarge","windows":"playwright-x64-server2025-64-core"}')[inputs.platform || fromJSON('{"needs-triage-linux":"linux","needs-triage-mac":"mac","needs-triage-win":"windows"}')[github.event.label.name]] || 'ubuntu-latest' }}
41+
timeout-minutes: 240
42+
concurrency:
43+
group: triage-${{ github.event.issue.number || inputs.issue }}
44+
cancel-in-progress: false
45+
defaults:
46+
run:
47+
shell: bash
48+
env:
49+
ISSUE: ${{ github.event.issue.number || inputs.issue }}
50+
TRIAGE_PLATFORM: ${{ inputs.platform || fromJSON('{"needs-triage-linux":"linux","needs-triage-mac":"mac","needs-triage-win":"windows"}')[github.event.label.name] }}
51+
permissions:
52+
copilot-requests: write
53+
outputs:
54+
base: ${{ steps.base.outputs.sha }}
55+
setup: ${{ steps.setup.outcome }}
56+
agent: ${{ steps.agent.outcome }}
57+
has_fix: ${{ steps.save.outputs.has_fix }}
58+
steps:
59+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
60+
with:
61+
ref: main
62+
persist-credentials: false
63+
- name: Record the base and output directory
64+
id: base
65+
run: |
66+
[[ "$ISSUE" =~ ^[1-9][0-9]*$ ]] || { echo "issue must be a positive integer" >&2; exit 1; }
67+
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
68+
echo "OUTPUT_DIR=${RUNNER_TEMP//\\//}/triage" >> "$GITHUB_ENV"
69+
echo "SCRATCH_DIR=${RUNNER_TEMP//\\//}/scratch" >> "$GITHUB_ENV"
70+
echo "PLAYWRIGHT_DIR=${GITHUB_WORKSPACE//\\//}" >> "$GITHUB_ENV"
71+
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
72+
with:
73+
node-version: '24'
74+
- name: Install Playwright dependencies and browsers
75+
id: setup
76+
run: |
77+
mkdir -p "$OUTPUT_DIR" "$SCRATCH_DIR"
78+
npm ci
79+
npm install -g @github/copilot
80+
npm run build
81+
npx playwright install --with-deps chromium firefox webkit
82+
- name: Triage with Copilot CLI
83+
id: agent
84+
# A finished proposal is still useful if the session exhausts its budget.
85+
continue-on-error: true
86+
env:
87+
COPILOT_GITHUB_TOKEN: ${{ github.token }}
88+
TASK: ${{ inputs.task }}
89+
run: |
90+
PROMPT="Follow .claude/skills/playwright-triage/SKILL.md. Read it and the triage.md it
91+
points to in full before starting.
92+
93+
Issue: https://github.com/microsoft/playwright/issues/$ISSUE"
94+
if [[ -n "$TASK" ]]; then
95+
PROMPT="$PROMPT
96+
97+
Task: $TASK"
98+
fi
99+
copilot \
100+
--allow-all-tools \
101+
--allow-all-paths \
102+
--no-ask-user \
103+
--enable-all-github-mcp-tools \
104+
--share="$OUTPUT_DIR/copilot-session.md" \
105+
--model gpt-6.1-sol \
106+
--max-ai-credits 5000 \
107+
-p "$PROMPT"
108+
- name: Save the outcome
109+
id: save
110+
if: always()
111+
env:
112+
AGENT_OUTCOME: ${{ steps.agent.outcome }}
113+
run: |
114+
mkdir -p "$OUTPUT_DIR"
115+
if [[ -f "$OUTPUT_DIR/fix/proposal.json" ]]; then
116+
echo "has_fix=true" >> "$GITHUB_OUTPUT"
117+
else
118+
echo "has_fix=false" >> "$GITHUB_OUTPUT"
119+
fi
120+
echo "Copilot step: $AGENT_OUTCOME" >> "$GITHUB_STEP_SUMMARY"
121+
if [[ -f "$OUTPUT_DIR/summary.md" ]]; then
122+
cat "$OUTPUT_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"
123+
else
124+
echo "No investigation summary was produced; see the session artifact." >> "$GITHUB_STEP_SUMMARY"
125+
fi
126+
- name: Upload investigation and fix
127+
if: always()
128+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
129+
with:
130+
name: triage
131+
path: ${{ env.OUTPUT_DIR }}
132+
if-no-files-found: warn
133+
134+
publish:
135+
name: Open draft pull request
136+
needs: triage
137+
# proposal.json is written last, so a finished proposal is published even when the job failed later.
138+
if: ${{ !cancelled() && needs.triage.outputs.has_fix == 'true' }}
139+
runs-on: ubuntu-latest
140+
outputs:
141+
pr: ${{ steps.publish.outputs.pr }}
142+
steps:
143+
- uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
144+
id: app-token
145+
with:
146+
client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }}
147+
private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }}
148+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
149+
with:
150+
ref: ${{ needs.triage.outputs.base }}
151+
token: ${{ steps.app-token.outputs.token }}
152+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
153+
with:
154+
name: triage
155+
path: ${{ runner.temp }}/triage
156+
# Never execute code or scripts from the proposed changes in the app job.
157+
- name: Validate, push and open draft
158+
id: publish
159+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
160+
env:
161+
BASE_SHA: ${{ needs.triage.outputs.base }}
162+
ISSUE: ${{ github.event.issue.number || inputs.issue }}
163+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
164+
with:
165+
github-token: ${{ steps.app-token.outputs.token }}
166+
script: |
167+
const fs = require('fs');
168+
const path = require('path');
169+
const { execFileSync } = require('child_process');
170+
const { ISSUE, RUNNER_TEMP, RUN_URL } = process.env;
171+
const git = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim();
172+
const dir = path.join(RUNNER_TEMP, 'triage', 'fix');
173+
const { owner, repo } = context.repo;
174+
git('apply', '--index', path.join(dir, 'fix.patch'));
175+
176+
// Workflow changes would run with this repo's secrets in PR CI.
177+
const workflowFiles = git('diff', '--cached', '--name-only', '--no-renames', '-z').split('\0').filter(file => file.startsWith('.github/'));
178+
if (workflowFiles.length)
179+
throw new Error(`Patch changes workflow files, treating it as compromised: ${workflowFiles.join(', ')}`);
180+
181+
const { title } = JSON.parse(fs.readFileSync(path.join(dir, 'proposal.json'), 'utf8'));
182+
const description = fs.readFileSync(path.join(dir, 'pr-body.md'), 'utf8').trim();
183+
const branch = `triage/${ISSUE}/${context.runId}-${process.env.GITHUB_RUN_ATTEMPT}`;
184+
git('config', 'user.name', 'microsoft-playwright-automation[bot]');
185+
git('config', 'user.email', '203992400+microsoft-playwright-automation[bot]@users.noreply.github.com');
186+
git('switch', '-c', branch);
187+
git('commit', '-m', title, '-m', `Fixes: https://github.com/${owner}/${repo}/issues/${ISSUE}`);
188+
git('push', 'origin', `HEAD:refs/heads/${branch}`);
189+
const { data: pr } = await github.rest.pulls.create({
190+
owner, repo, head: branch, base: 'main', title,
191+
body: `${description}\n\n<sub>Prepared by the [triage workflow](${RUN_URL}).</sub>`,
192+
draft: true,
193+
});
194+
core.setOutput('pr', pr.html_url);
195+
196+
comment:
197+
name: Comment on the issue
198+
needs: [triage, publish]
199+
# Runs after every triage run, including failed ones, and whatever publish did.
200+
if: ${{ !cancelled() && needs.triage.result != 'skipped' }}
201+
runs-on: ubuntu-latest
202+
permissions:
203+
issues: write
204+
env:
205+
ISSUE: ${{ github.event.issue.number || inputs.issue }}
206+
PLATFORM_NAME: ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage-linux":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }}
207+
TRIAGE_RESULT: ${{ needs.triage.result }}
208+
SETUP_OUTCOME: ${{ needs.triage.outputs.setup }}
209+
AGENT_OUTCOME: ${{ needs.triage.outputs.agent }}
210+
PUBLISH_RESULT: ${{ needs.publish.result }}
211+
PR_URL: ${{ needs.publish.outputs.pr }}
212+
GH_TOKEN: ${{ github.token }}
213+
GH_REPO: ${{ github.repository }}
214+
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
215+
steps:
216+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
217+
# A failed triage run may not have uploaded anything.
218+
continue-on-error: true
219+
with:
220+
name: triage
221+
path: output
222+
- name: Post triage comment
223+
run: |
224+
mkdir -p output
225+
if [[ ! -s output/comment.md ]]; then
226+
if [[ "$SETUP_OUTCOME" == "failure" ]]; then
227+
reason="installing dependencies, building or downloading browsers failed"
228+
elif [[ "$AGENT_OUTCOME" == "failure" ]]; then
229+
reason="the Copilot session failed or ran out of budget before it wrote a result"
230+
elif [[ "$TRIAGE_RESULT" != "success" ]]; then
231+
reason="the triage job ended with result \`$TRIAGE_RESULT\`, for example after a timeout"
232+
else
233+
reason="the Copilot session ended without writing a result"
234+
fi
235+
echo "Hi, I'm the Playwright bot. I couldn't finish triaging this issue: $reason. The agent run has the details." > output/comment.md
236+
fi
237+
if [[ -n "$PR_URL" ]]; then
238+
printf '\n\nDraft fix: %s\n' "$PR_URL" >> output/comment.md
239+
elif [[ "$PUBLISH_RESULT" == "failure" ]]; then
240+
printf '\n\nThe bot proposed a fix, but opening the pull request failed. The agent run has the details.\n' >> output/comment.md
241+
fi
242+
printf '\n\n<sub>Triaged on %s by the Playwright bot - [agent run](%s)</sub>\n<!-- playwright-ai-triage -->\n' "$PLATFORM_NAME" "$WORKFLOW_URL" >> output/comment.md
243+
gh issue comment "$ISSUE" --body-file output/comment.md

‎.github/workflows/triage_linux.yml‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

0 commit comments

Comments
 (0)