diff --git a/.github/scripts/thread-transfer-report.cjs b/.github/scripts/thread-transfer-report.cjs index 94a02b7806..264a92b1c8 100644 --- a/.github/scripts/thread-transfer-report.cjs +++ b/.github/scripts/thread-transfer-report.cjs @@ -61,7 +61,7 @@ function validateResult(value) { } assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario"); - if (value.scenario.id !== "thread-transfer-v1") { + if (!["thread-transfer-v1", "thread-transfer-v2"].includes(value.scenario.id)) { throw new Error("result.scenario.id is not supported"); } for (const key of SCENARIO_KEYS.slice(1)) { @@ -157,7 +157,7 @@ function renderComment(input) { `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, ); - if (baseline && baseline.providers[provider].ceiling[metric.key] !== ceiling) { + if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) { ceilingChanges.push( `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, ); diff --git a/.github/scripts/thread-transfer-report.test.cjs b/.github/scripts/thread-transfer-report.test.cjs index 4935864e46..3c8ab2578a 100644 --- a/.github/scripts/thread-transfer-report.test.cjs +++ b/.github/scripts/thread-transfer-report.test.cjs @@ -290,3 +290,21 @@ test("preserves a successful result when a same-SHA rerun has no artifact", asyn assert.equal(published, true); assert.equal(updatedComment, false); }); + +test("accepts V2 without comparing it to the V1 scenario", () => { + const current = result(); + current.scenario.id = "thread-transfer-v2"; + current.providers.codex.ceiling = { ...current.providers.codex.ceiling, totalWireBytes: 3000000 }; + assert.equal(validateResult(current), current); + const comment = renderComment({ + current, + baseline: result(), + currentRun: { sha: "bbbbbbbb", conclusion: "success", url: "https://example.com/current" }, + baselineRun: { sha: "aaaaaaaa", matchesBase: true, url: "https://example.com/baseline" }, + }); + assert.match(comment, /fixture changed/); + assert.doesNotMatch(comment, /This PR changes transfer ceilings/); + assert.doesNotMatch(comment, /[+-]\d+\.\d+%/); + current.scenario.id = "unrecognized"; + assert.throws(() => validateResult(current), /not supported/); +});