diff --git a/apps/judge/src/judge-strategies.test.ts b/apps/judge/src/judge-strategies.test.ts index ddf309533..5cf0dfcdc 100644 --- a/apps/judge/src/judge-strategies.test.ts +++ b/apps/judge/src/judge-strategies.test.ts @@ -7,6 +7,8 @@ import { tmpdir } from "os"; import { join } from "path"; import { IndependentStrategy, + BundledStrategy, + TOOL_OUTPUTS_GUIDANCE, isWithinWorkspace, JUDGE_AVAILABLE_TOOLS, JUDGE_EXCLUDED_TOOLS, @@ -21,9 +23,21 @@ class TestableStrategy extends IndependentStrategy { publicCreateFileTools(workspacePath: string) { return this.createFileTools(workspacePath); } + publicCreateToolOutputTools(toolCalls: any[]) { + return this.createToolOutputTools(toolCalls); + } publicBuildSessionConfig(tools: any[], systemPrompt: string) { return this.buildSessionConfig(tools, systemPrompt); } + publicBuildSystemPrompt(hasToolOutputs: boolean) { + return (this as any).buildSystemPrompt(undefined, hasToolOutputs); + } + publicBuildUserPrompt( + criterion: { id: string; prompt: string }, + history: any[] = [] + ) { + return (this as any).buildUserPrompt(criterion, history); + } } function toolMap(workspacePath: string) { @@ -72,6 +86,37 @@ describe("judge file tools", () => { }); }); +describe("judge tool-output tools (issue #1125)", () => { + const strategy = new TestableStrategy("test-model"); + const toolCalls = [ + { + id: "1", + name: "bash", + arguments: { command: "npm run build" }, + response: "build ok", + timestamp: "", + }, + ]; + const tools = strategy.publicCreateToolOutputTools(toolCalls); + + it("exposes read_tool_outputs and get_tool_output", () => { + const names = tools.map((t) => t.name).sort(); + expect(names).toEqual(["get_tool_output", "read_tool_outputs"]); + }); + + // Regression guard for scope #1125: headless, any tool without skipPermission + // is denied at execution time ("could not request permission from user"). When + // this affected read_tool_outputs/get_tool_output, the judge could never see + // the coding agent's captured build/test output and wrongly demanded on-disk + // proof files, forcing the build gate to loop for many iterations. + it("marks every tool-output tool to skip the permission prompt", () => { + expect(tools.length).toBeGreaterThan(0); + for (const tool of tools) { + expect(tool.skipPermission, `${tool.name} must skip the permission prompt`).toBe(true); + } + }); +}); + describe("isWithinWorkspace", () => { it("accepts the root itself and nested paths", () => { expect(isWithinWorkspace("/tmp/ws", "/tmp/ws")).toBe(true); @@ -191,3 +236,161 @@ describe("judge session tool restriction (scope #1117)", () => { expect([...JUDGE_EXCLUDED_TOOLS]).toEqual(["builtin:*", "mcp:*"]); }); }); + +/** + * Exposes the protected `buildSystemPrompt` / `buildUserPrompt` so we can assert + * how captured tool outputs, criteria, and history are surfaced to the judge + * without running a real Copilot session. + */ +class TestableBundledStrategy extends BundledStrategy { + publicBuildSystemPrompt(hasToolOutputs: boolean) { + return (this as any).buildSystemPrompt(undefined, hasToolOutputs); + } + publicBuildUserPrompt( + criteria: { id: string; prompt: string }[] = [ + { id: "c1", prompt: "does the code work" }, + ], + history: any[] = [] + ) { + return (this as any).buildUserPrompt(criteria, history); + } +} + +describe("judge tool-outputs guidance (issue #1125)", () => { + // The headless judge cannot run commands; it must decide from the codebase + // plus the coding agent's captured tool outputs. The guidance must be generic + // (not build/test specific) and must tell the judge to treat captured output + // as authoritative instead of demanding the agent redo or re-prove the work. + it("is generic, not tied to any one command type", () => { + const g = TOOL_OUTPUTS_GUIDANCE.toLowerCase(); + expect(g).not.toMatch(/build\.log|build_proof|\bnpm run build\b/); + }); + + it("tells the judge it cannot run commands itself", () => { + expect(TOOL_OUTPUTS_GUIDANCE.toLowerCase()).toContain("cannot run any commands"); + }); + + it("names the judge's own read-only tools and disambiguates them from the agent's", () => { + expect(TOOL_OUTPUTS_GUIDANCE).toContain("## Your Tools"); + expect(TOOL_OUTPUTS_GUIDANCE).toContain("read_file"); + expect(TOOL_OUTPUTS_GUIDANCE).toContain("read_tool_outputs"); + expect(TOOL_OUTPUTS_GUIDANCE).toContain("get_tool_output"); + expect(TOOL_OUTPUTS_GUIDANCE.toLowerCase()).toContain("codebase"); + }); + + it("frames the codebase and captured outputs as equally authoritative and to be examined together", () => { + expect(TOOL_OUTPUTS_GUIDANCE).toContain("## How to Judge"); + const g = TOOL_OUTPUTS_GUIDANCE.toLowerCase(); + expect(g).toContain("equally authoritative"); + expect(g).toContain("examine both"); + }); + + it("treats captured output as the record of what happened and forbids redundant re-proving", () => { + const g = TOOL_OUTPUTS_GUIDANCE.toLowerCase(); + expect(g).toContain("record of what happened"); + expect(g).toMatch(/redo or re-prove/); + }); + + it("overrides criteria wording that asks the judge to run commands", () => { + const g = TOOL_OUTPUTS_GUIDANCE.toLowerCase(); + expect(g).toContain("criterion"); + expect(g).toMatch(/run, execute, or re-run/); + expect(g).toContain("ignore that instruction"); + }); + + it("includes the guidance in the system prompt when tool outputs are present", () => { + const prompt = new TestableBundledStrategy("test-model").publicBuildSystemPrompt(true); + expect(prompt).toContain(TOOL_OUTPUTS_GUIDANCE); + }); + + it("frames What to Evaluate around the agent's work and points at the user message", () => { + const prompt = new TestableBundledStrategy("test-model").publicBuildSystemPrompt(true); + expect(prompt).toContain("## What to Evaluate"); + expect(prompt).toMatch(/generated code together with the captured outputs of the tools it ran/); + expect(prompt).toMatch(/each criterion provided in the user message/); + }); + + it("omits the guidance when no tool outputs were captured", () => { + const prompt = new TestableBundledStrategy("test-model").publicBuildSystemPrompt(false); + expect(prompt).not.toContain(TOOL_OUTPUTS_GUIDANCE); + }); +}); + +describe("judge system/user prompt split (criteria + history are user data)", () => { + // The system prompt must stay invariant across criteria/iterations: it carries + // only the role, tools, judging method, instructions, and output format. The + // per-request data — the criterion/criteria and previous-iteration history — + // belongs in the user prompt. + + describe("BundledStrategy", () => { + const sys = new TestableBundledStrategy("test-model").publicBuildSystemPrompt(true); + + it("keeps the criteria out of the system prompt", () => { + expect(sys).not.toContain("## Criteria"); + expect(sys).not.toContain("does the code work"); + }); + + it("keeps the previous-iteration history out of the system prompt", () => { + const sysWithHistoryPath = new TestableBundledStrategy("test-model").publicBuildSystemPrompt(false); + expect(sys).not.toContain("## Previous Iterations"); + expect(sysWithHistoryPath).not.toContain("## Previous Iterations"); + }); + + it("puts the criteria in the user prompt", () => { + const user = new TestableBundledStrategy("test-model").publicBuildUserPrompt(); + expect(user).toContain("## Criteria"); + expect(user).toContain("c1: does the code work"); + }); + + it("appends previous-iteration history to the user prompt only when present", () => { + const strat = new TestableBundledStrategy("test-model"); + const noHistory = strat.publicBuildUserPrompt(); + expect(noHistory).not.toContain("## Previous Iterations"); + + const withHistory = strat.publicBuildUserPrompt( + [{ id: "c1", prompt: "does the code work" }], + [ + { + iteration: 1, + codingAgentResponse: "did some work", + judgeFeedback: "needs more", + passed: false, + }, + ] + ); + expect(withHistory).toContain("## Previous Iterations"); + expect(withHistory).toContain("### Iteration 1"); + }); + }); + + describe("IndependentStrategy", () => { + const strat = new TestableStrategy("test-model"); + const sys = strat.publicBuildSystemPrompt(true); + + it("keeps the criterion out of the system prompt", () => { + expect(sys).not.toContain("**Criterion**"); + expect(sys).toMatch(/the criterion provided in the user message/); + }); + + it("keeps the previous-iteration history out of the system prompt", () => { + expect(sys).not.toContain("## Previous Iterations"); + }); + + it("puts the criterion id and prompt in the user prompt", () => { + const user = strat.publicBuildUserPrompt({ id: "build-ok", prompt: "it builds" }); + expect(user).toContain('Evaluate criterion "build-ok": it builds'); + }); + + it("appends previous-iteration history to the user prompt only when present", () => { + const noHistory = strat.publicBuildUserPrompt({ id: "build-ok", prompt: "it builds" }); + expect(noHistory).not.toContain("## Previous Iterations"); + + const withHistory = strat.publicBuildUserPrompt( + { id: "build-ok", prompt: "it builds" }, + [{ iteration: 1, codingAgentResponse: "tried", passed: false }] + ); + expect(withHistory).toContain("## Previous Iterations (for context)"); + expect(withHistory).toContain("### Iteration 1"); + }); + }); +}); diff --git a/apps/judge/src/judge-strategies.ts b/apps/judge/src/judge-strategies.ts index cca6e86ee..19705be5e 100644 --- a/apps/judge/src/judge-strategies.ts +++ b/apps/judge/src/judge-strategies.ts @@ -41,6 +41,25 @@ const DEFAULT_JUDGE_TIMEOUT = 480_000; /** Default number of retries for sendAndWait calls */ const DEFAULT_JUDGE_RETRIES = 3; +/** + * The `## Your Tools` + `## How to Judge` guidance injected into the judge + * system prompt when the coding agent's tool calls/outputs were captured for + * this iteration. + * + * The judge runs headless and cannot run any commands itself — it can only + * inspect the workspace and read what the coding agent already did. This text + * keeps the judge's own read-only tools unambiguous from the coding agent's + * tools/commands, and frames the codebase and captured tool outputs as two + * complementary, equally authoritative sources of evidence so the judge bases + * its decision on actual evidence instead of demanding the agent re-prove work + * it has already done. It is intentionally generic across all criteria. + */ +export const TOOL_OUTPUTS_GUIDANCE = `## Your Tools +You have read-only tools to gather evidence: read_file, list_directory, search_files and file_exists to inspect the workspace, and read_tool_outputs / get_tool_output to review the tool calls the coding agent ran while doing the task. You yourself cannot run any commands or coding-agent tools — you can only read what the agent already did. + +## How to Judge +The codebase and the agent's captured tool outputs are two complementary, equally authoritative sources of evidence — examine both. The files show the resulting state of the code; the captured outputs (logs, results, exit status) show what actually happened when the agent ran a command, which the files alone may not reveal. When a criterion concerns something the agent did or ran, take the agent's captured output and exit status as the record of what happened, rather than asking the agent to redo or re-prove work the evidence already shows. If a criterion's wording tells you to run, execute, or re-run a command, ignore that instruction and judge the outcome from the captured outputs together with the codebase.`; + /** * Tool filter applied to every judge session. * @@ -329,6 +348,13 @@ export abstract class JudgeStrategy { const readToolOutputs = defineTool("read_tool_outputs", { description: "List the tool calls the coding agent made during this iteration (e.g. shell/bash commands and their output). Returns each call's index, name, arguments and a truncated response preview. Use get_tool_output(index) to fetch the full output of a specific call. Consult these to decide whether a command (build, test, run) actually succeeded.", + // Read-only in-memory inspection of already-captured tool calls. Like the + // file tools above, this MUST run without a permission prompt: the judge is + // headless (no TUI), so the v3 runtime would otherwise deny every call with + // "could not request permission from user" — which silently blocks the judge + // from ever seeing the coding agent's build/test output and forces it to + // demand on-disk proof files instead. See scope #1125. + skipPermission: true, parameters: { type: "object", properties: {}, @@ -359,6 +385,8 @@ export abstract class JudgeStrategy { const getToolOutput = defineTool("get_tool_output", { description: "Return the full captured output (response) of a single tool call by its index, as listed by read_tool_outputs.", + // Read-only; same headless permission rationale as read_tool_outputs. See scope #1125. + skipPermission: true, parameters: { type: "object", properties: { @@ -498,72 +526,52 @@ export class BundledStrategy extends JudgeStrategy { criteria, conversationHistory, personaInstructions, - gate, toolCalls, } = context; const systemPrompt = this.buildSystemPrompt( - criteria, - conversationHistory, personaInstructions, - gate, toolCalls && toolCalls.length > 0 ); + const userPrompt = this.buildUserPrompt(criteria, conversationHistory); + const response = await this.runCopilotSession( workspacePath, systemPrompt, - "Evaluate the workspace against ALL criteria. Use the file tools to inspect the code, then provide your verdict in JSON format.", + userPrompt, toolCalls ); return this.parseJsonResponse(response, criteria, context.onProgress); } + /** + * Builds the invariant system prompt (role, persona, tools, judging method, + * instructions, output format). It does NOT contain the criteria or the + * previous-iteration history — those are per-request data carried by the user + * prompt (see {@link buildUserPrompt}) so the system prompt stays identical + * across every criterion and iteration in a run. + */ private buildSystemPrompt( - criteria: CriteriaConfig[], - conversationHistory: ConversationTurn[], personaInstructions?: string, - gate?: GateId, hasToolOutputs?: boolean ): string { - const criteriaList = criteria - .map((c) => ` - ${c.id}: ${c.prompt}`) - .join("\n"); - - const historySection = - conversationHistory.length > 0 - ? `\n## Previous Iterations\n${conversationHistory - .map((t) => { - const car = t.codingAgentResponse ?? "(no response captured)"; - const fb = t.judgeFeedback; - return `### Iteration ${t.iteration}\n- **Coding agent response**: ${car.substring(0, 500)}${car.length > 500 ? "..." : ""}\n- **Your previous feedback**: ${fb.substring(0, 500)}${fb.length > 500 ? "..." : ""}\n- **Passed**: ${t.passed}`; - }) - .join("\n\n")}` - : ""; - const personaSection = personaInstructions ? `\n## Persona\n${personaInstructions}\n` : ""; - const gateLine = - gate && gate !== "select" ? `\nYou are evaluating the **${gate}** gate.\n` : ""; - const toolOutputsSection = hasToolOutputs - ? `\n## Tool outputs\nThe coding agent ran commands during this iteration. Use read_tool_outputs to list those calls and get_tool_output(index) to read full output. To decide whether a command (e.g. build, test, run) succeeded, inspect its output and exit status rather than guessing from the files alone.\n` + ? `\n${TOOL_OUTPUTS_GUIDANCE}\n` : ""; - return `You are an expert code reviewer evaluating whether generated code meets requirements. -${personaSection}${gateLine} -## Your Task -Inspect the workspace using the provided tools (read_file, list_directory, search_files, file_exists) and evaluate whether the code meets each criterion. + return `You are an expert code reviewer evaluating the tool calls, logs and generated code produced by a coding agent. +${personaSection} +## What to Evaluate +Evaluate whether the coding agent's work — its generated code together with the captured outputs of the tools it ran — meets each criterion provided in the user message. ${toolOutputsSection} -## Criteria -${criteriaList} -${historySection} - ## Instructions -1. Use the tools to thoroughly inspect the workspace. +1. Gather evidence from both the workspace and the coding agent's captured tool outputs. 2. Evaluate EACH criterion individually. 3. For each criterion, provide specific feedback about what you found. 4. Be constructive and actionable in your feedback. @@ -582,6 +590,36 @@ Your response MUST be valid JSON with this structure: IMPORTANT: Return ONLY the JSON, no additional text before or after.`; } + /** + * Builds the per-request user prompt: the criteria to evaluate plus any + * previous-iteration context. This is the data the (invariant) system prompt + * refers to. + */ + private buildUserPrompt( + criteria: CriteriaConfig[], + conversationHistory: ConversationTurn[] + ): string { + const criteriaList = criteria + .map((c) => ` - ${c.id}: ${c.prompt}`) + .join("\n"); + + const historySection = + conversationHistory.length > 0 + ? `\n\n## Previous Iterations\n${conversationHistory + .map((t) => { + const car = t.codingAgentResponse ?? "(no response captured)"; + const fb = t.judgeFeedback; + return `### Iteration ${t.iteration}\n- **Coding agent response**: ${car.substring(0, 500)}${car.length > 500 ? "..." : ""}\n- **Your previous feedback**: ${fb.substring(0, 500)}${fb.length > 500 ? "..." : ""}\n- **Passed**: ${t.passed}`; + }) + .join("\n\n")}` + : ""; + + return `Evaluate the workspace against ALL criteria below. Use the file tools to inspect the code, then provide your verdict in JSON format. + +## Criteria +${criteriaList}${historySection}`; + } + private parseJsonResponse( response: string, criteria: CriteriaConfig[], @@ -800,59 +838,18 @@ export class IndependentStrategy extends JudgeStrategy { gate?: GateId, toolCalls?: ToolCall[] ): Promise { - const historySection = - conversationHistory.length > 0 - ? `\n## Previous Iterations (for context)\n${conversationHistory - .map((t) => { - const car = t.codingAgentResponse ?? "(no response captured)"; - return `### Iteration ${t.iteration}\n- **Coding agent response**: ${car.substring(0, 300)}${car.length > 300 ? "..." : ""}\n- **Passed**: ${t.passed}`; - }) - .join("\n\n")}` - : ""; - - const personaSection = personaInstructions - ? `\n## Persona\n${personaInstructions}\n` - : ""; - - const gateLine = - gate && gate !== "select" ? `\nYou are evaluating the **${gate}** gate.\n` : ""; - - const toolOutputsSection = + const systemPrompt = this.buildSystemPrompt( + personaInstructions, toolCalls && toolCalls.length > 0 - ? `\n## Tool outputs\nThe coding agent ran commands during this iteration. Use read_tool_outputs to list those calls and get_tool_output(index) to read full output. To decide whether a command (e.g. build, test, run) succeeded, inspect its output and exit status rather than guessing from the files alone.\n` - : ""; - - const systemPrompt = `You are an expert code reviewer evaluating ONE specific criterion. -${personaSection}${gateLine} -## Your Task -Inspect the workspace using the provided tools and evaluate ONLY this criterion: - -**Criterion**: ${criterion.prompt} -${toolOutputsSection} -${historySection} - -## Instructions -1. Use the file tools to thoroughly inspect the workspace. -2. Determine if this specific criterion is met (PASS) or not met (FAIL). -3. Provide specific feedback about what you found. - -## Output Format -**CRITICAL**: The FIRST LINE of your response MUST be exactly "PASS:" or "FAIL:" (nothing else on that line). -Then provide your explanation on subsequent lines. - -Example: -PASS: -The workspace contains a package.json file with express listed as a dependency (version 4.18.0). + ); -Or: -FAIL: -No package.json file was found in the workspace root.`; + const userPrompt = this.buildUserPrompt(criterion, conversationHistory); try { const response = await this.runCopilotSession( workspacePath, systemPrompt, - `Evaluate criterion "${criterion.id}": ${criterion.prompt}`, + userPrompt, toolCalls ); @@ -878,6 +875,70 @@ No package.json file was found in the workspace root.`; } } + /** + * Builds the invariant system prompt (role, persona, tools, judging method, + * instructions, output format). It does NOT contain the criterion or the + * previous-iteration history — those are per-request data carried by the user + * prompt (see {@link buildUserPrompt}) so the system prompt stays identical + * across every criterion and iteration in a run. + */ + protected buildSystemPrompt( + personaInstructions?: string, + hasToolOutputs?: boolean + ): string { + const personaSection = personaInstructions + ? `\n## Persona\n${personaInstructions}\n` + : ""; + + const toolOutputsSection = hasToolOutputs + ? `\n${TOOL_OUTPUTS_GUIDANCE}\n` + : ""; + + return `You are an expert code reviewer evaluating the tool calls, logs and generated code produced by a coding agent against ONE specific criterion. +${personaSection} +## What to Evaluate +Evaluate the coding agent's work — its generated code together with the captured outputs of the tools it ran — against the criterion provided in the user message. +${toolOutputsSection} +## Instructions +1. Gather evidence from both the workspace and the coding agent's captured tool outputs. +2. Determine if the criterion is met (PASS) or not met (FAIL). +3. Provide specific feedback about what you found. + +## Output Format +**CRITICAL**: The FIRST LINE of your response MUST be exactly "PASS:" or "FAIL:" (nothing else on that line). +Then provide your explanation on subsequent lines. + +Example: +PASS: +The workspace contains a package.json file with express listed as a dependency (version 4.18.0). + +Or: +FAIL: +No package.json file was found in the workspace root.`; + } + + /** + * Builds the per-request user prompt: the single criterion to evaluate plus + * any previous-iteration context. This is the data the (invariant) system + * prompt refers to. + */ + protected buildUserPrompt( + criterion: CriteriaConfig, + conversationHistory: ConversationTurn[] + ): string { + const historySection = + conversationHistory.length > 0 + ? `\n\n## Previous Iterations (for context)\n${conversationHistory + .map((t) => { + const car = t.codingAgentResponse ?? "(no response captured)"; + return `### Iteration ${t.iteration}\n- **Coding agent response**: ${car.substring(0, 300)}${car.length > 300 ? "..." : ""}\n- **Passed**: ${t.passed}`; + }) + .join("\n\n")}` + : ""; + + return `Evaluate criterion "${criterion.id}": ${criterion.prompt}${historySection}`; + } + private detectPassFail(response: string): boolean { const lines = response.trim().split("\n"); const firstLines = lines.slice(0, 3).join("\n").toUpperCase(); diff --git a/docs/design/gates.md b/docs/design/gates.md index 548831981..582438803 100644 --- a/docs/design/gates.md +++ b/docs/design/gates.md @@ -449,9 +449,71 @@ alongside `createFileTools`): (`writeToolCalls` / `toolCallsUrl`), downloaded next to the snapshot. The judge agent's available tools therefore become workspace-scoped -(`read_file`, `list_directory`) **plus** `read_tool_outputs`. The system prompt -for non-Select gates instructs the judge to consult tool outputs (e.g. "to -decide whether the build succeeded, inspect the build command's output"). +(`read_file`, `list_directory`, `search_files`, `file_exists`) **plus** +`read_tool_outputs` / `get_tool_output`. When tool outputs are present, the +system prompt injects a generic guidance block (`TOOL_OUTPUTS_GUIDANCE` in +`judge-strategies.ts`, shared by both the bundled and independent strategies), +which renders as two sections: **`## Your Tools`** and **`## How to Judge`**. + +The judge's opening framing is also gate-agnostic — it presents the judge as +"evaluating the tool calls, logs and generated code produced by a coding agent" +(against one specific criterion, in the independent strategy). The prompt no +longer announces which gate is being evaluated; gate scoping is handled entirely +by *which* criteria are passed in, so the wording stays identical across gates. + +The system prompt is organized into balanced, single-purpose sections: + +- **`## What to Evaluate`** — names the subject under review as the coding + agent's work: *its generated code together with the captured outputs of the + tools it ran*, and points at the criterion/criteria **provided in the user + message** (see the system/user split below). It does not list tools or inline + the criteria, so it stays invariant and does not back-couple to the tooling + section. +- **`## Your Tools`** — names the judge's own read-only tools once (`read_file`, + `list_directory`, `search_files`, `file_exists`, plus `read_tool_outputs` / + `get_tool_output`) and states the hard limit: the judge cannot run any commands + or coding-agent tools. This keeps the judge's tools unambiguous from the + *coding agent's* tools/commands (whose output the judge only reads). +- **`## How to Judge`** — the evidence philosophy (below). + +> **System / user prompt split.** The judge SDK session receives two distinct +> messages, and the division of content matters: +> +> - The **system prompt** carries only the invariant *method/role*: the opening +> framing, `## Persona` (when set), `## Your Tools` + `## How to Judge` (when +> tool outputs were captured), `## Instructions`, and `## Output Format`. It is +> identical for every criterion and iteration in a run. +> - The **user prompt** carries the per-request *data*: the criterion (independent +> strategy: `Evaluate criterion "": `) or the `## Criteria` list +> (bundled strategy), followed by the `## Previous Iterations` context when this +> is not the first iteration. +> +> Both strategies build these via `buildSystemPrompt(persona, hasToolOutputs)` and +> `buildUserPrompt(criteria, history)` in `judge-strategies.ts`. Keeping the +> criterion in the user message (not the system prompt) gives a clean trust +> boundary: the system prompt is the sole authority — including the override +> clause below — while the criterion is *data being evaluated* rather than an +> instruction that competes with the judge's rules. It also keeps the system +> prompt stable across criteria/iterations. + +> **Gate-judge contract (issue #1125).** The guidance is deliberately generic — +> it is not specific to the build, test, or any single gate. It frames the +> **codebase** and the agent's **captured tool outputs** as two complementary, +> **equally authoritative** sources of evidence that must both be examined — +> neither is prioritized over the other, and they are not mutually exclusive. The +> files show the resulting state of the code; the captured outputs (logs, +> results, exit status) show what actually happened when the agent ran a command, +> which the files alone may not reveal. When a criterion concerns something the +> agent did or ran, the judge takes the captured output and exit status as the +> **record of what happened**, rather than asking the agent to **redo or +> re-prove** work the evidence already shows (e.g. it must not demand on-disk +> proof files like `build.log` when the command's captured output already shows +> it succeeded). **If a criterion's own wording tells the judge to +> run/execute/re-run a command, the judge must ignore that instruction** — it has +> no command-running ability — and judge the outcome from the captured outputs +> together with the codebase. This prevents the failure mode where the judge +> withheld a passing verdict for several iterations despite an exit-code-0 build +> being present in the captured outputs. > **Tool isolation (scope #1117).** The judge session is restricted to its own > custom tools only — `createSession` is called with `availableTools: ["custom:*"]` @@ -466,6 +528,22 @@ decide whether the build succeeded, inspect the build command's output"). > command actually succeeded). Locking the session to `custom:*` removes that > escape hatch and forces the judge to evaluate from `read_tool_outputs`. +> **All judge tools must be `skipPermission` (scope #1125).** The same headless +> permission rule that applies to the built-ins above applies to the judge's *own* +> custom tools: under the v3 runtime, any tool the model calls that is not marked +> `skipPermission: true` is denied non-interactively with "Permission denied and +> could not request permission from user". The file-inspection tools +> (`read_file`, `list_directory`, `search_files`, `file_exists`) set this flag, +> but `read_tool_outputs` / `get_tool_output` originally did not — so every judge +> attempt to read the coder's captured build/test output was silently denied. The +> judge then fell back to demanding on-disk proof (a `dist/` directory, a +> `build-output.log`, a README), which forced the coding agent to *fabricate* +> build-evidence files and made the build/test gates loop for many iterations. +> Both tool-output tools are now `skipPermission: true`, so the judge can read the +> captured command output (and trust an exit-code-0 build) on the first iteration. +> A regression test in `judge-strategies.test.ts` asserts every tool returned by +> `createToolOutputTools` skips the permission prompt. + ```mermaid flowchart LR subgraph Iteration