From 59a2ab3d232236af2c6d2162482ff518c3cd6968 Mon Sep 17 00:00:00 2001 From: Miya Date: Tue, 15 Sep 2026 18:54:08 +0200 Subject: [PATCH 1/3] fix(sdk): default Claude steps to Sonnet 4.6 Session-Id: 01a09c40-ce3b-7f11-a7df-b6b7ccab6fd9 --- packages/sdk/src/adapters/base.ts | 3 ++ packages/sdk/src/adapters/claude.ts | 1 + packages/sdk/src/budget-preflight.ts | 16 +++++-- packages/sdk/src/cli-adapter.ts | 6 +++ packages/sdk/src/cli/check.ts | 5 ++- packages/sdk/src/llm-worker.ts | 6 ++- packages/sdk/src/preflight.ts | 45 ++++++++++++------- packages/sdk/src/worker-cli.ts | 12 ++--- packages/sdk/src/worker.ts | 6 ++- packages/sdk/tests/adapters/claude.test.ts | 1 + packages/sdk/tests/budget-preflight.test.ts | 17 +++++-- packages/sdk/tests/cli-adapter.test.ts | 7 +++ .../sdk/tests/flow-executor-chain.test.ts | 40 ++++++++++++++++- packages/sdk/tests/pty-sidechannel.test.ts | 8 ++-- packages/sdk/tests/worker-cli.test.ts | 23 ++++++++++ 15 files changed, 159 insertions(+), 37 deletions(-) diff --git a/packages/sdk/src/adapters/base.ts b/packages/sdk/src/adapters/base.ts index a6daaddd7..da0d1da38 100644 --- a/packages/sdk/src/adapters/base.ts +++ b/packages/sdk/src/adapters/base.ts @@ -32,6 +32,9 @@ export interface HeadlessAdapter { /** Identity of this adapter — matches CliAdapterKind for registry keys. */ readonly kind: string; + /** Stable model used when an author omits one. */ + readonly defaultModel?: string; + /** Shape-check invocation before classifying an auth failure. */ buildIdentification(): CliAdapterIdentification; diff --git a/packages/sdk/src/adapters/claude.ts b/packages/sdk/src/adapters/claude.ts index 8d163118a..187d4c9c6 100644 --- a/packages/sdk/src/adapters/claude.ts +++ b/packages/sdk/src/adapters/claude.ts @@ -10,6 +10,7 @@ const MODEL_PROBE_PROMPT = 'Reply with exactly RELAYFLOWS_MODEL_READY and nothin * pre-#141 inline shape in `cli-adapter.ts` — only the packaging changed. */ export const claudeAdapter: HeadlessAdapter = { kind: 'claude', + defaultModel: 'claude-sonnet-4-6', buildIdentification(): CliAdapterIdentification { return { invocation: { args: ['auth', 'status', '--help'], timeoutMs: 10_000 } }; diff --git a/packages/sdk/src/budget-preflight.ts b/packages/sdk/src/budget-preflight.ts index 449f14633..319c77fc9 100644 --- a/packages/sdk/src/budget-preflight.ts +++ b/packages/sdk/src/budget-preflight.ts @@ -3,7 +3,10 @@ import type { PreflightRefusal } from './preflight.js'; import { MODEL_PRICING } from './model-pricing.js'; /** Legacy explicit envelopes keep their worker-supplied pricing contract. */ -export function budgetDiagnostics(flow: CompiledFlowSpec): PreflightRefusal[] { +export function budgetDiagnostics( + flow: CompiledFlowSpec, + resolvedModels: ReadonlyMap = new Map(), +): PreflightRefusal[] { if (flow.budget?.pricing !== 'frozen') return []; const diagnostics: PreflightRefusal[] = []; const declared = [ @@ -13,12 +16,19 @@ export function budgetDiagnostics(flow: CompiledFlowSpec): PreflightRefusal[] { ]; for (const step of flow.steps) { if (step.type === 'deterministic' || flow.budget.maxDollars === undefined) continue; - const model = step.model ?? (step.type === 'agent' && step.agent !== undefined - ? flow.agents?.[step.agent]?.model : undefined); + const model = resolvedModels.has(step.id) ? resolvedModels.get(step.id) + : step.model ?? (step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent]?.model : undefined); if (model === undefined) diagnostics.push({ severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, message: `Step "${step.id}" needs a declared, priced model for its dollar budget.`, }); + else if (step.model === undefined + && (step.type !== 'agent' || step.agent === undefined || flow.agents?.[step.agent]?.model === undefined) + && !Object.hasOwn(MODEL_PRICING, model)) diagnostics.push({ + severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, model, + message: `Model "${model}" has no frozen price for budget accounting.`, + }); } for (const declaration of declared) { if (Object.hasOwn(MODEL_PRICING, declaration.model)) continue; diff --git a/packages/sdk/src/cli-adapter.ts b/packages/sdk/src/cli-adapter.ts index 0133c552d..a7ac82e2b 100644 --- a/packages/sdk/src/cli-adapter.ts +++ b/packages/sdk/src/cli-adapter.ts @@ -28,6 +28,12 @@ export function cliAdapterKind(executable: string): CliAdapterKind { return resolveAdapterKind(executable); } +/** Resolve only defaults owned by registered raw-CLI adapters. */ +export function resolveCliModel(executable: string, model?: string): string | undefined { + if (model !== undefined) return model; + return registeredAdapters()[resolveAdapterKind(executable)].defaultModel; +} + /** Prove the adapter command shape before classifying an auth failure. */ export function adapterIdentification(kind: CliAdapterKind): CliAdapterIdentification { return registeredAdapters()[kind].buildIdentification(); diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index b96e435c4..f9f56d81d 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -358,7 +358,10 @@ function bindResolvedCliPaths( const resolution = byStep.get(step.id); if (resolution === undefined) return step; const directory = resolution.source === 'project' ? configDirectory : flowDirectory; - return { ...step, cli: canonicalCli(resolution.cli, directory) }; + const declaredModel = step.model ?? (step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent]?.model : undefined); + return { ...step, cli: canonicalCli(resolution.cli, directory), + ...(declaredModel === undefined && resolution.model !== undefined ? { model: resolution.model } : {}) }; }), }; } diff --git a/packages/sdk/src/llm-worker.ts b/packages/sdk/src/llm-worker.ts index b004b947d..cf4002b97 100644 --- a/packages/sdk/src/llm-worker.ts +++ b/packages/sdk/src/llm-worker.ts @@ -5,6 +5,7 @@ import type { JournalClient } from './journal-client.js'; import type { CompletionReason, StepDispatchEvent } from './protocol.js'; import type { KernelLlmStep } from './spec.js'; import { runAgentCli } from './worker-cli.js'; +import { resolveCliModel } from './cli-adapter.js'; import { withWorkerLease } from './worker-lease.js'; import { workerInstruction } from './worker-input.js'; import { jsonSchemaOutputError } from './json-schema.js'; @@ -49,11 +50,12 @@ export class LlmWorker extends EventEmitter { const schema = spec.verification?.json_schema; const prompt = schema === undefined ? spec.prompt : `${spec.prompt}\n\nReturn only a JSON value matching this JSON Schema (no Markdown fences):\n${JSON.stringify(schema)}`; + const effectiveModel = typeof spec.cli === 'string' ? resolveCliModel(spec.cli, spec.model) : spec.model; const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.prompt === 'string' - ? runAgentCli(spec.cli, workerInstruction(prompt, dispatch), dispatch.wake_context, spec.model, undefined, signal, 'llm') + ? runAgentCli(spec.cli, workerInstruction(prompt, dispatch), dispatch.wake_context, effectiveModel, undefined, signal, 'llm') : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'llm step has no declared CLI' })); - const { result, usage } = workerSpend(completed, spec.model); + const { result, usage } = workerSpend(completed, effectiveModel); let reason: CompletionReason = result.exit_code === 0 ? 'success' : 'worker_error'; let output: unknown = result.stdout_tail; let detail = result.stderr_tail; diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 6043c8274..f42a37acd 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -8,6 +8,7 @@ import type { TriggerSource } from '@relayflows/surface'; import { acceptsAnyOutput, inspectStepGate, type StepGateInspection } from './gate-contract.js'; import { compileSpec, CompileError } from './compile.js'; import { helperCall } from './yaml-helpers.js'; +import { resolveCliModel } from './cli-adapter.js'; import { isNamedGate, NAMED_GATE_FAILURE_KINDS, type NamedGateFailureKind } from './named-gates.js'; import { compileScopes, type ScopeInput, type MountRegistry } from './scope-compiler.js'; import { readMountRegistry } from './mount-registry.js'; @@ -208,18 +209,17 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul }; } const diagnostics: PreflightDiagnostic[] = []; + const cliResolutionDiagnostics: PreflightDiagnostic[] = []; const resolutions: CliResolution[] = []; const resolutionByStep = new Map(); const cliProbeResults = new Map(); - diagnostics.push(...unknownModelDiagnostics(compiled, options)); diagnostics.push(...scopeDiagnostics(compiled, options)); for (const server of new Set(options.mcpServers ?? [])) { if (options.mcp !== undefined && Object.hasOwn(options.mcp, server)) continue; diagnostics.push({ severity: 'refusal', kind: 'mcp_undeclared_server', server, message: `MCP server "${server}" is not declared in the nearest flows.json mcp map.` }); } - diagnostics.push(...budgetDiagnostics(compiled)); // Resolve the complete flow before touching any environment fact. A later // statically unresolved CLI makes the whole submission impossible, so no // earlier command, provider/model, or trigger probe may run first. @@ -228,7 +228,7 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul if (step.type === 'agent' && helperCall(step) !== undefined) continue; const resolution = resolveCli(step, compiled, options.projectCli); if (resolution === undefined) { - diagnostics.push({ + cliResolutionDiagnostics.push({ severity: 'refusal', kind: 'cli_unresolved', stepId: step.id, @@ -239,6 +239,12 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul resolutionByStep.set(step.id, resolution); } } + diagnostics.push(...unknownModelDiagnostics(compiled, options, resolutionByStep)); + diagnostics.push(...cliResolutionDiagnostics); + diagnostics.push(...budgetDiagnostics( + compiled, + new Map(resolutions.map(resolution => [resolution.stepId, resolution.model])), + )); if (diagnostics.length > 0) { return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics }; } @@ -311,6 +317,7 @@ function scopeDiagnostics(flow: FlowSpec, options: PreflightOptions): PreflightR function unknownModelDiagnostics( flow: FlowSpec, options: PreflightOptions, + resolutionByStep: ReadonlyMap = new Map(), ): PreflightRefusal[] { const diagnostics: PreflightRefusal[] = []; // model_unknown is a governance check: it exists to enforce a project's @@ -340,17 +347,21 @@ function unknownModelDiagnostics( } for (const step of flow.steps) { - if (step.type === 'deterministic' || step.model === undefined) continue; - if (isKnownModel(step.model, options.models)) continue; + if (step.type === 'deterministic') continue; + const named = step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent] : undefined; + if (step.model === undefined && named !== undefined) continue; + const resolution = resolutionByStep.get(step.id); + const model = step.model ?? resolution?.model; + if (model === undefined || isKnownModel(model, options.models)) continue; if (!enforceRegistry) continue; - const resolution = resolveCli(step, flow, options.projectCli); diagnostics.push({ severity: 'refusal', kind: 'model_unknown', stepId: step.id, ...(resolution === undefined ? {} : { cli: resolution.cli }), - model: step.model, - message: unknownModelMessage(step.id, step.model, resolution?.cli, options.modelRegistryPath), + model, + message: unknownModelMessage(step.id, model, resolution?.cli, options.modelRegistryPath), }); } @@ -403,14 +414,16 @@ function resolveCli( const named = step.type === 'agent' && step.agent !== undefined ? flow.agents?.[step.agent] : undefined; - // Model comes only from the step or its explicitly selected declaration. - // There is deliberately no flow/project or host default. - const effectiveModel = step.model ?? named?.model; - const model = effectiveModel !== undefined ? { model: effectiveModel } : {}; - if (step.cli !== undefined) return { stepId: step.id, cli: step.cli, source: 'step', ...model }; - if (named !== undefined) return { stepId: step.id, cli: named.cli, source: 'named', ...model }; - if (flow.cli !== undefined) return { stepId: step.id, cli: flow.cli, source: 'flow', ...model }; - if (projectCli !== undefined) return { stepId: step.id, cli: projectCli, source: 'project', ...model }; + const declaredModel = step.model ?? named?.model; + const resolved = (cli: string, source: CliResolutionSource): CliResolution => { + const effectiveModel = resolveCliModel(cli, declaredModel); + return { stepId: step.id, cli, source, + ...(effectiveModel === undefined ? {} : { model: effectiveModel }) }; + }; + if (step.cli !== undefined) return resolved(step.cli, 'step'); + if (named !== undefined) return resolved(named.cli, 'named'); + if (flow.cli !== undefined) return resolved(flow.cli, 'flow'); + if (projectCli !== undefined) return resolved(projectCli, 'project'); return undefined; } diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 8297e1108..0752964e2 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -6,6 +6,7 @@ import { agentExecution, llmExecution, cliAdapterKind, + resolveCliModel, type CliInvocation, type CliAdapterKind, } from './cli-adapter.js'; @@ -68,9 +69,10 @@ export async function runAgentCli( throw new Error('Lease-bound agent execution requires macOS or Linux process-group cancellation; Windows is unsupported.'); } const kind = cliAdapterKind(cli); + const effectiveModel = resolveCliModel(cli, model); if (mode === 'agent' && transport === 'relay') { - return runViaAgentRelay(kind, instruction, wakeContext, model, relayContext, cwd, signal); + return runViaAgentRelay(kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal); } if (kind === 'relayflows-wrapper-v1') { @@ -78,17 +80,17 @@ export async function runAgentCli( cli, instruction, wakeContext, - model, + effectiveModel, wrapperEnvironment(process.env), wrapperLimits, signal, - )), model); + )), effectiveModel); } const env: NodeJS.ProcessEnv = { ...process.env }; delete env[WAKE_CONTEXT_ENV]; delete env[MODEL_ENV]; - const invocation = mode === 'llm' ? llmExecution(kind, instruction, model) : agentExecution(kind, instruction, model); + const invocation = mode === 'llm' ? llmExecution(kind, instruction, effectiveModel) : agentExecution(kind, instruction, effectiveModel); if (wakeContext !== undefined) { try { @@ -106,7 +108,7 @@ export async function runAgentCli( // Structured provider output carries the authoritative token counts. const args = [...invocation.args]; args.splice(args.length - 1, 0, ...(kind === 'claude' ? ['--output-format', 'json'] : ['--json'])); - return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal, sidechannel, cwd), kind), model); + return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal, sidechannel, cwd), kind), effectiveModel); } /** Wait under the same worker lease for an authoritative task receipt. */ diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 632e0728d..0dae443e4 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -5,6 +5,7 @@ import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; import type { KernelAgentStep } from './spec.js'; import { runAgentCli } from './worker-cli.js'; +import { resolveCliModel } from './cli-adapter.js'; import { withWorkerLease } from './worker-lease.js'; import { workerInstruction } from './worker-input.js'; import { helperCall } from './yaml-helpers.js'; @@ -105,9 +106,10 @@ export class AgentWorker extends EventEmitter { return; } let humanIntervention = false; + const effectiveModel = typeof spec.cli === 'string' ? resolveCliModel(spec.cli, spec.model) : spec.model; const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.instruction === 'string' - ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, spec.model, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { + ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, effectiveModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { dataDir: this.options.dataDir, runId: dispatch.run_id, stepId: dispatch.step_id, onReady: this.options.onPtyReady, onDrive: () => { humanIntervention = true; }, }, typeof spec.cwd === 'string' ? spec.cwd : undefined, @@ -115,7 +117,7 @@ export class AgentWorker extends EventEmitter { { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema }) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); - const { result, usage } = workerSpend(completed, spec.model); + const { result, usage } = workerSpend(completed, effectiveModel); const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; // Output shape: if the CLI's stdout parses as JSON, promote THAT diff --git a/packages/sdk/tests/adapters/claude.test.ts b/packages/sdk/tests/adapters/claude.test.ts index f56db9d51..c2b2afb6c 100644 --- a/packages/sdk/tests/adapters/claude.test.ts +++ b/packages/sdk/tests/adapters/claude.test.ts @@ -4,6 +4,7 @@ import { claudeAdapter } from '../../src/adapters/claude.js'; describe('claudeAdapter — HeadlessAdapter contract', () => { it('identifies itself as kind "claude"', () => { expect(claudeAdapter.kind).toBe('claude'); + expect(claudeAdapter.defaultModel).toBe('claude-sonnet-4-6'); }); it('buildIdentification uses the auth-status help shape', () => { diff --git a/packages/sdk/tests/budget-preflight.test.ts b/packages/sdk/tests/budget-preflight.test.ts index f6590a959..0a43b6e6c 100644 --- a/packages/sdk/tests/budget-preflight.test.ts +++ b/packages/sdk/tests/budget-preflight.test.ts @@ -41,11 +41,22 @@ describe('budget preflight', () => { expect(preflight(kernelToAuthoring(toKernelSpec(compileSpec(spec('$20/run', 'unknown')))), options()).diagnostics) .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); }); - it('requires a model when declaring a dollar budget', () => { + it('prices and probes the Claude default when a dollar-budgeted step omits model', () => { const input = spec('$20/run'); const { model, ...step } = input.steps[0]!; - expect(preflight({...input, steps:[step]}, options()).diagnostics) - .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); + const o = options(); + const result = preflight({...input, steps:[step]}, o); + expect(result.ok).toBe(true); + expect(result.resolutions).toContainEqual(expect.objectContaining({ + stepId: 'ask', cli: 'claude', model: 'claude-sonnet-4-6', + })); + expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-sonnet-4-6'); + }); + it('still requires a model for an unknown/custom CLI under a dollar budget', () => { + const input = spec('$20/run'); + const { model, ...step } = input.steps[0]!; + expect(preflight({...input, steps:[{...step, cli:'team-wrapper'}]}, options()).diagnostics) + .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price', stepId: 'ask'})])); }); it.each(['$1/week', '-$1/run', '$1.0000001/run', {tokens: -1}, {wallclock: 'soon'}, {dollars: Infinity}, {typo: 2}])('refuses malformed budget %j', budget => { expect(preflight(spec(budget), options()).ok).toBe(false); diff --git a/packages/sdk/tests/cli-adapter.test.ts b/packages/sdk/tests/cli-adapter.test.ts index 6a3811ace..672e4fb34 100644 --- a/packages/sdk/tests/cli-adapter.test.ts +++ b/packages/sdk/tests/cli-adapter.test.ts @@ -5,10 +5,17 @@ import { authenticationProbe, cliAdapterKind, modelReadinessProbe, + resolveCliModel, WRAPPER_IDENTIFY_ARG, } from '../src/cli-adapter.js'; describe('typed CLI adapters', () => { + it('resolves only Claude to its stable default and preserves explicit models', () => { + expect(resolveCliModel('/usr/local/bin/claude')).toBe('claude-sonnet-4-6'); + expect(resolveCliModel('/usr/local/bin/claude', 'explicit-model')).toBe('explicit-model'); + expect(resolveCliModel('/opt/bin/codex')).toBeUndefined(); + expect(resolveCliModel('/project/bin/team-reviewer')).toBeUndefined(); + }); it('maps raw Claude to real auth, noninteractive, and model flag shapes', () => { const kind = cliAdapterKind('/usr/local/bin/claude'); diff --git a/packages/sdk/tests/flow-executor-chain.test.ts b/packages/sdk/tests/flow-executor-chain.test.ts index 81085c954..3d7bbcf4c 100644 --- a/packages/sdk/tests/flow-executor-chain.test.ts +++ b/packages/sdk/tests/flow-executor-chain.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { once } from 'node:events'; import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -66,6 +66,44 @@ describe('flow executor LLM and output-binding chain', () => { expect(failures).toEqual([]); }); + it('runs a dollar-budgeted authored Claude agent with the same default used by preflight', async () => { + const { fixture, client, agent, failures } = await setup(); + const calls = join(fixture.root, 'claude-calls.jsonl'); + const claude = join(fixture.root, 'claude'); + writeFileSync(join(fixture.root, 'flows.json'), JSON.stringify({ models: ['claude-sonnet-4-6'] })); + writeFileSync(claude, `#!/usr/bin/env node +import { appendFileSync } from 'node:fs'; +appendFileSync(${JSON.stringify(calls)}, JSON.stringify(process.argv.slice(2)) + '\\n'); +if (process.argv[2] === 'auth') process.exit(0); +if (process.argv.includes('Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.')) { + process.stdout.write('RELAYFLOWS_MODEL_READY\\n'); process.exit(0); +} +process.stdout.write(JSON.stringify({ type: 'result', result: 'default-model-agent-ok', + usage: { input_tokens: 2, output_tokens: 1 } }) + '\\n'); +`); + chmodSync(claude, 0o755); + const handle = flow('default-model-agent', { budget: '$8/run' }, async f => { + const result = await f.agent('implementer', { cli: claude, task: 'Implement.' }); + expect(result.summary).toBe('default-model-agent-ok'); + f.done('success'); + }); + + const result = await executeAuthoredFlow(handle, client, undefined, { + flowPath: fixture.flowPath, localAgentStream: agent.stream, + }); + + expect(result.completionReason).toBe('success'); + const invocations = readFileSync(calls, 'utf8').trim().split('\n').map(line => JSON.parse(line) as string[]); + const modelScoped = invocations.filter(args => args.includes('--model')); + expect(modelScoped).toHaveLength(2); + expect(modelScoped.every(args => args[args.indexOf('--model') + 1] === 'claude-sonnet-4-6')).toBe(true); + expect(modelScoped.at(-1)).toEqual([ + '-p', '--dangerously-skip-permissions', '--model', 'claude-sonnet-4-6', + '--output-format', 'json', expect.stringContaining('Implement.'), + ]); + expect(failures).toEqual([]); + }); + it.each(['not JSON', '{"message":7}'])('fails invalid LLM output before the next step: %s', async output => { const { fixture, client, agent, failures } = await setup(output); const marker = join(fixture.root, 'must-not-exist'); diff --git a/packages/sdk/tests/pty-sidechannel.test.ts b/packages/sdk/tests/pty-sidechannel.test.ts index 3091577b0..4d4dff816 100644 --- a/packages/sdk/tests/pty-sidechannel.test.ts +++ b/packages/sdk/tests/pty-sidechannel.test.ts @@ -47,7 +47,7 @@ setTimeout(() => process.exit(0), 450); client.emit('step.dispatch', { run_id: 'r', step_id: 's', step_type: 'agent', attempt: 1, idempotency_key: 'k', lease_id: 'lease', lease_deadline_ms: Date.now() + 60_000, - pins: { workspace: [], streams: [] }, spec: { cli, instruction: 'test' }, + pins: { workspace: [], streams: [] }, spec: { cli, model: 'pty-test-model', instruction: 'test' }, }); try { const args = await done; @@ -84,7 +84,7 @@ process.stdin.on('end', () => { clearTimeout(watchdog); process.stdout.write('eo let peer: Socket | undefined; let driven = false; try { - const result = await runAgentCli(cli, 'test', undefined, undefined, undefined, undefined, 'agent', { + const result = await runAgentCli(cli, 'test', undefined, 'pty-test-model', undefined, undefined, 'agent', { dataDir, runId: 'r', stepId: 's', onDrive: () => { driven = true; }, onReady(path) { if (mode === 'none') return; @@ -112,7 +112,7 @@ process.stdin.on('end', () => { let attempted = false; let rejectedBeforeExit = false; try { - const result = await runAgentCli(cli, 'test', undefined, undefined, undefined, undefined, 'agent', { + const result = await runAgentCli(cli, 'test', undefined, 'pty-test-model', undefined, undefined, 'agent', { dataDir, runId: 'r', stepId: 's', onDrive: () => { driven = true; }, onReady(path) { peer = connect(path, () => peer!.write('HELLO view\n')); @@ -154,7 +154,7 @@ setTimeout(() => { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 4000); try { - const result = await runAgentCli(cli, 'test', undefined, undefined, undefined, controller.signal, 'agent', { + const result = await runAgentCli(cli, 'test', undefined, 'pty-test-model', undefined, controller.signal, 'agent', { dataDir, runId: 'r', stepId: 's', onDrive() {}, onReady(path) { peer = connect(path, () => { peer!.write('HELLO drive\n'); peer!.write(payload); }); diff --git a/packages/sdk/tests/worker-cli.test.ts b/packages/sdk/tests/worker-cli.test.ts index 481aecb90..364166f5b 100644 --- a/packages/sdk/tests/worker-cli.test.ts +++ b/packages/sdk/tests/worker-cli.test.ts @@ -2,6 +2,7 @@ import { chmodSync, existsSync, mkdtempSync, + readFileSync, realpathSync, rmSync, symlinkSync, @@ -56,6 +57,28 @@ async function withEnvironment( } } +describe('registered CLI model defaults', () => { + it('passes the same priced Claude default to the real provider invocation', async () => { + const directory = makeDirectory(); + const calls = join(directory, 'calls.json'); + const claude = makeWrapper(directory, 'claude', ` +const fs = require('node:fs'); +fs.writeFileSync(${JSON.stringify(calls)}, JSON.stringify(process.argv.slice(2))); +process.stdout.write(JSON.stringify({ type: 'result', result: 'default-model-ok', + usage: { input_tokens: 2, output_tokens: 1 } }) + '\\n'); +`); + + const result = await runAgentCli(claude, 'do the task', undefined); + + expect(result).toMatchObject({ exit_code: 0, stdout_tail: 'default-model-ok', + tokens_input: 2, tokens_output: 1 }); + expect(JSON.parse(readFileSync(calls, 'utf8'))).toEqual([ + '-p', '--dangerously-skip-permissions', '--model', 'claude-sonnet-4-6', + '--output-format', 'json', 'do the task', + ]); + }); +}); + describe('custom wrapper execution identity', () => { it('passes an explicit safe environment at identification and execution', async () => { const directory = makeDirectory(); From 22c5a6de7a2dd583e02a5a0c4e2e5010bda6f0ba Mon Sep 17 00:00:00 2001 From: Miya Date: Tue, 15 Sep 2026 18:56:23 +0200 Subject: [PATCH 2/3] test: allow the Claude default in shared fixtures Session-Id: 01a09c40-ce3b-7f11-a7df-b6b7ccab6fd9 --- testdata/flows.json | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/flows.json b/testdata/flows.json index 3396475ca..4692d4bc5 100644 --- a/testdata/flows.json +++ b/testdata/flows.json @@ -3,6 +3,7 @@ "executors": ["agent-worker"], "models": [ "claude-haiku-4-5-20251001", + "claude-sonnet-4-6", "claude-sonnet-5", "deterministic-test-stub", "test-model-v1" From c77be1987d80d46b1d1dba3e3c6bb7ec9cd51f72 Mon Sep 17 00:00:00 2001 From: Miya Date: Tue, 15 Sep 2026 19:31:12 +0200 Subject: [PATCH 3/3] fix(sdk): default Claude steps to Opus 5 Session-Id: 01a09c40-ce3b-7f11-a7df-b6b7ccab6fd9 --- docs/SURFACE.md | 15 ++--- packages/schema/flows.schema.json | 2 +- packages/schema/tests/parity.test.ts | 4 +- packages/sdk/src/adapters/base.ts | 6 +- packages/sdk/src/adapters/claude.ts | 2 +- packages/sdk/src/budget-preflight.ts | 13 +++-- packages/sdk/src/cli-adapter.ts | 27 ++++++++- packages/sdk/src/cli/check.ts | 5 +- packages/sdk/src/model-pricing.ts | 1 + packages/sdk/src/preflight.ts | 30 ++++++---- packages/sdk/src/spec.ts | 9 +-- packages/sdk/tests/adapters/claude.test.ts | 2 +- packages/sdk/tests/budget-preflight.test.ts | 57 +++++++++++++++++-- packages/sdk/tests/cli-adapter.test.ts | 10 +++- .../sdk/tests/flow-executor-chain.test.ts | 6 +- packages/sdk/tests/model-pricing.test.ts | 6 ++ packages/sdk/tests/preflight.test.ts | 21 ++++--- packages/sdk/tests/verb-field-lint.test.ts | 4 +- packages/sdk/tests/worker-cli.test.ts | 2 +- testdata/flows.json | 1 - 20 files changed, 162 insertions(+), 61 deletions(-) diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 68aa8539e..96bcc881d 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -144,13 +144,14 @@ No process runs between events: the handler wakes, executes to its next await, p the existing per-step `cli` and `model` fields. The validated selector and map remain authoring metadata through `flows check`, so unused and step-shadowed declarations are linted too; both are removed at the kernel - boundary. Explicit step values win independently: - step `cli`/`model` → named declaration → the existing flow/project CLI - default. Model has no flow/project default. An inline step that selects no - named declaration keeps the existing optional-model behavior. The worker - explicitly removes ambient `RELAYFLOW_MODEL`; raw provider adapters use a - model flag, while at worker execution a custom wrapper receives the model - only inside its identified same-process session when the step declares one. + boundary. CLI and model resolve independently. CLI priority is step → named + declaration → flow → project config. Model priority is step → named + declaration → registered adapter default. Claude's adapter default is + `claude-opus-5`; Codex and custom wrappers have no default. A frozen dollar + budget refuses before execution when the selected model has no frozen price. + The worker explicitly removes ambient `RELAYFLOW_MODEL`; raw provider + adapters use a model flag, while a custom wrapper receives an explicitly + declared model only inside its identified same-process session. **Anonymous resolution law:** `f.agent\`task\`` with no name is the *default agent*, resolved (never guessed) in order: step options → flow header → project config (`flows.json`) → platform default. *The platform-default rung is declared but not yet implemented: no platform default is provisioned as of gate 1, so a flow that reaches this rung refuses with `cli_unresolved` rather than guessing. `flows check` never invents an implicit default.* `flows check` prints each resolved step CLI and its declaration source, validates it before submission, and refuses a missing or unauthenticated resolution before the checked flow is submitted, never at minute 27. Gate 1 does not make this guarantee for callers that bypass `flows check`: the journal client's direct `run.start` path does not invoke surface preflight. diff --git a/packages/schema/flows.schema.json b/packages/schema/flows.schema.json index 5da42e0b9..3cbc4c37a 100644 --- a/packages/schema/flows.schema.json +++ b/packages/schema/flows.schema.json @@ -1296,7 +1296,7 @@ }, "model": { "title": "model", - "description": "Model the declared CLI must use. Raw Claude/Codex adapters receive their\nreal model flag; an identified Relayflows wrapper receives it in its\nsame-process execution request. Declared here so the choice is journaled with the step\ninstead of being ambient host state.", + "description": "Model the declared CLI must use. A step declaration wins over its selected\nnamed agent and any registered adapter default. Raw Claude/Codex adapters\nreceive the effective model as a real flag; an identified Relayflows\nwrapper receives an explicitly declared model in its same-process request.\nThe effective choice is journaled instead of being ambient host state.", "type": "string", "minLength": 1, "pattern": "^\\S(?:[\\s\\S]*\\S)?$", diff --git a/packages/schema/tests/parity.test.ts b/packages/schema/tests/parity.test.ts index d50d9f61f..bb6a32766 100644 --- a/packages/schema/tests/parity.test.ts +++ b/packages/schema/tests/parity.test.ts @@ -19,7 +19,9 @@ mkdirSync(join(root, 'bin')); // every check must agree, including environment readiness (no ignored refusals). const wrapper = '#!/bin/sh\nif [ "$1" = "--relayflows-adapter-v1" ]; then echo relayflows-agent-cli-v1; fi\nexit 0\n'; for (const file of ['preflight/authenticated-cli', 'preflight/analyze-story-claude-cli', 'bin/claude']) writeFileSync(join(root, file), wrapper, { mode: 0o755 }); -writeFileSync(join(root, 'flows.json'), readFileSync(new URL('../../../testdata/flows.json', import.meta.url))); +const parityConfig = JSON.parse(readFileSync(new URL('../../../testdata/flows.json', import.meta.url), 'utf8')); +parityConfig.models.push('claude-opus-5'); +writeFileSync(join(root, 'flows.json'), JSON.stringify(parityConfig)); const oldPath = process.env.PATH; process.env.PATH = `${join(root, 'bin')}:${oldPath ?? ''}`; afterAll(() => { process.env.PATH = oldPath; rmSync(root, { recursive: true, force: true }); }); diff --git a/packages/sdk/src/adapters/base.ts b/packages/sdk/src/adapters/base.ts index da0d1da38..6df66ea06 100644 --- a/packages/sdk/src/adapters/base.ts +++ b/packages/sdk/src/adapters/base.ts @@ -32,7 +32,11 @@ export interface HeadlessAdapter { /** Identity of this adapter — matches CliAdapterKind for registry keys. */ readonly kind: string; - /** Stable model used when an author omits one. */ + /** + * Stable model used only when neither the step nor its selected named agent + * declares one. Explicit authoring always wins. A default used with frozen + * dollar budgets must also have an entry in MODEL_PRICING. + */ readonly defaultModel?: string; /** Shape-check invocation before classifying an auth failure. */ diff --git a/packages/sdk/src/adapters/claude.ts b/packages/sdk/src/adapters/claude.ts index 187d4c9c6..e27d906e1 100644 --- a/packages/sdk/src/adapters/claude.ts +++ b/packages/sdk/src/adapters/claude.ts @@ -10,7 +10,7 @@ const MODEL_PROBE_PROMPT = 'Reply with exactly RELAYFLOWS_MODEL_READY and nothin * pre-#141 inline shape in `cli-adapter.ts` — only the packaging changed. */ export const claudeAdapter: HeadlessAdapter = { kind: 'claude', - defaultModel: 'claude-sonnet-4-6', + defaultModel: 'claude-opus-5', buildIdentification(): CliAdapterIdentification { return { invocation: { args: ['auth', 'status', '--help'], timeoutMs: 10_000 } }; diff --git a/packages/sdk/src/budget-preflight.ts b/packages/sdk/src/budget-preflight.ts index 319c77fc9..1c2172a1e 100644 --- a/packages/sdk/src/budget-preflight.ts +++ b/packages/sdk/src/budget-preflight.ts @@ -1,11 +1,12 @@ import type { CompiledFlowSpec } from './compile.js'; import type { PreflightRefusal } from './preflight.js'; import { MODEL_PRICING } from './model-pricing.js'; +import type { ResolvedCliModel } from './cli-adapter.js'; -/** Legacy explicit envelopes keep their worker-supplied pricing contract. */ +/** Frozen dollar budgets require an exact model with a frozen table price. */ export function budgetDiagnostics( flow: CompiledFlowSpec, - resolvedModels: ReadonlyMap = new Map(), + resolvedModels: ReadonlyMap = new Map(), ): PreflightRefusal[] { if (flow.budget?.pricing !== 'frozen') return []; const diagnostics: PreflightRefusal[] = []; @@ -16,15 +17,15 @@ export function budgetDiagnostics( ]; for (const step of flow.steps) { if (step.type === 'deterministic' || flow.budget.maxDollars === undefined) continue; - const model = resolvedModels.has(step.id) ? resolvedModels.get(step.id) - : step.model ?? (step.type === 'agent' && step.agent !== undefined + const resolved = resolvedModels.get(step.id); + const model = resolved?.model + ?? step.model ?? (step.type === 'agent' && step.agent !== undefined ? flow.agents?.[step.agent]?.model : undefined); if (model === undefined) diagnostics.push({ severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, message: `Step "${step.id}" needs a declared, priced model for its dollar budget.`, }); - else if (step.model === undefined - && (step.type !== 'agent' || step.agent === undefined || flow.agents?.[step.agent]?.model === undefined) + else if (resolved?.source === 'adapter' && !Object.hasOwn(MODEL_PRICING, model)) diagnostics.push({ severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, model, message: `Model "${model}" has no frozen price for budget accounting.`, diff --git a/packages/sdk/src/cli-adapter.ts b/packages/sdk/src/cli-adapter.ts index a7ac82e2b..8d3ace41d 100644 --- a/packages/sdk/src/cli-adapter.ts +++ b/packages/sdk/src/cli-adapter.ts @@ -28,10 +28,31 @@ export function cliAdapterKind(executable: string): CliAdapterKind { return resolveAdapterKind(executable); } -/** Resolve only defaults owned by registered raw-CLI adapters. */ +export type CliModelSource = 'step' | 'named' | 'adapter'; + +export interface ResolvedCliModel { + readonly model?: string; + readonly source?: CliModelSource; +} + +/** + * Resolve the model once, in authoring priority order: step, selected named + * agent, then the registered CLI adapter's default. Unregistered executables + * resolve through the wrapper adapter, whose absent default remains undefined. + */ +export function resolveCliModelSelection( + executable: string, + declarations: Readonly<{ step?: string; named?: string }> = {}, +): ResolvedCliModel { + if (declarations.step !== undefined) return { model: declarations.step, source: 'step' }; + if (declarations.named !== undefined) return { model: declarations.named, source: 'named' }; + const model = registeredAdapters()[resolveAdapterKind(executable)].defaultModel; + return model === undefined ? {} : { model, source: 'adapter' }; +} + +/** Resolve a runtime model, where any materialized value is step-owned. */ export function resolveCliModel(executable: string, model?: string): string | undefined { - if (model !== undefined) return model; - return registeredAdapters()[resolveAdapterKind(executable)].defaultModel; + return resolveCliModelSelection(executable, { step: model }).model; } /** Prove the adapter command shape before classifying an auth failure. */ diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index f9f56d81d..d6a2bf2c5 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -358,10 +358,9 @@ function bindResolvedCliPaths( const resolution = byStep.get(step.id); if (resolution === undefined) return step; const directory = resolution.source === 'project' ? configDirectory : flowDirectory; - const declaredModel = step.model ?? (step.type === 'agent' && step.agent !== undefined - ? flow.agents?.[step.agent]?.model : undefined); return { ...step, cli: canonicalCli(resolution.cli, directory), - ...(declaredModel === undefined && resolution.model !== undefined ? { model: resolution.model } : {}) }; + ...(resolution.modelSource === 'adapter' && resolution.model !== undefined + ? { model: resolution.model } : {}) }; }), }; } diff --git a/packages/sdk/src/model-pricing.ts b/packages/sdk/src/model-pricing.ts index 04672fe93..d41ecda79 100644 --- a/packages/sdk/src/model-pricing.ts +++ b/packages/sdk/src/model-pricing.ts @@ -2,6 +2,7 @@ export const MODEL_PRICING: Readonly>> = Object.freeze({ 'claude-sonnet-4-6': Object.freeze({ input: 3, output: 15 }), 'claude-opus-4-7': Object.freeze({ input: 15, output: 75 }), + 'claude-opus-5': Object.freeze({ input: 5, output: 25 }), 'codex-medium': Object.freeze({ input: 2, output: 8 }), 'codex-large': Object.freeze({ input: 5, output: 20 }), }); diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index f42a37acd..31107bd81 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -8,7 +8,7 @@ import type { TriggerSource } from '@relayflows/surface'; import { acceptsAnyOutput, inspectStepGate, type StepGateInspection } from './gate-contract.js'; import { compileSpec, CompileError } from './compile.js'; import { helperCall } from './yaml-helpers.js'; -import { resolveCliModel } from './cli-adapter.js'; +import { resolveCliModelSelection, type CliModelSource } from './cli-adapter.js'; import { isNamedGate, NAMED_GATE_FAILURE_KINDS, type NamedGateFailureKind } from './named-gates.js'; import { compileScopes, type ScopeInput, type MountRegistry } from './scope-compiler.js'; import { readMountRegistry } from './mount-registry.js'; @@ -23,8 +23,10 @@ export interface CliResolution { stepId: string; cli: string; source: CliResolutionSource; - /** Model the step or selected named agent declared, probed with the CLI. */ + /** Effective model probed with the CLI. */ model?: string; + /** Exact source selected by step > named agent > adapter default priority. */ + modelSource?: CliModelSource; } export interface CliProbeResult { @@ -239,11 +241,17 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul resolutionByStep.set(step.id, resolution); } } - diagnostics.push(...unknownModelDiagnostics(compiled, options, resolutionByStep)); diagnostics.push(...cliResolutionDiagnostics); + if (cliResolutionDiagnostics.length > 0) { + return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics }; + } + diagnostics.push(...unknownModelDiagnostics(compiled, options, resolutionByStep)); diagnostics.push(...budgetDiagnostics( compiled, - new Map(resolutions.map(resolution => [resolution.stepId, resolution.model])), + new Map(resolutions.map(resolution => [resolution.stepId, { + ...(resolution.model === undefined ? {} : { model: resolution.model }), + ...(resolution.modelSource === undefined ? {} : { source: resolution.modelSource }), + }])), )); if (diagnostics.length > 0) { return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics }; @@ -348,11 +356,11 @@ function unknownModelDiagnostics( for (const step of flow.steps) { if (step.type === 'deterministic') continue; - const named = step.type === 'agent' && step.agent !== undefined - ? flow.agents?.[step.agent] : undefined; - if (step.model === undefined && named !== undefined) continue; const resolution = resolutionByStep.get(step.id); - const model = step.model ?? resolution?.model; + // Selected named declarations were checked once above, including unused + // declarations. Other sources are step declarations or adapter defaults. + if (resolution?.modelSource === 'named') continue; + const model = resolution?.model ?? step.model; if (model === undefined || isKnownModel(model, options.models)) continue; if (!enforceRegistry) continue; diagnostics.push({ @@ -414,11 +422,11 @@ function resolveCli( const named = step.type === 'agent' && step.agent !== undefined ? flow.agents?.[step.agent] : undefined; - const declaredModel = step.model ?? named?.model; const resolved = (cli: string, source: CliResolutionSource): CliResolution => { - const effectiveModel = resolveCliModel(cli, declaredModel); + const effectiveModel = resolveCliModelSelection(cli, { step: step.model, named: named?.model }); return { stepId: step.id, cli, source, - ...(effectiveModel === undefined ? {} : { model: effectiveModel }) }; + ...(effectiveModel.model === undefined ? {} : { model: effectiveModel.model }), + ...(effectiveModel.source === undefined ? {} : { modelSource: effectiveModel.source }) }; }; if (step.cli !== undefined) return resolved(step.cli, 'step'); if (named !== undefined) return resolved(named.cli, 'named'); diff --git a/packages/sdk/src/spec.ts b/packages/sdk/src/spec.ts index 838e21bee..1cc48533a 100644 --- a/packages/sdk/src/spec.ts +++ b/packages/sdk/src/spec.ts @@ -238,10 +238,11 @@ export interface AgentStepSpec extends BaseStepSpec { /** Inert preflight declaration; overrides the flow/project CLI default. */ cli?: string; /** - * Model the declared CLI must use. Raw Claude/Codex adapters receive their - * real model flag; an identified Relayflows wrapper receives it in its - * same-process execution request. Declared here so the choice is journaled with the step - * instead of being ambient host state. + * Model the declared CLI must use. A step declaration wins over its selected + * named agent and any registered adapter default. Raw Claude/Codex adapters + * receive the effective model as a real flag; an identified Relayflows + * wrapper receives an explicitly declared model in its same-process request. + * The effective choice is journaled instead of being ambient host state. */ model?: string; surfaces?: AgentSurfaces; diff --git a/packages/sdk/tests/adapters/claude.test.ts b/packages/sdk/tests/adapters/claude.test.ts index c2b2afb6c..c1e252ff2 100644 --- a/packages/sdk/tests/adapters/claude.test.ts +++ b/packages/sdk/tests/adapters/claude.test.ts @@ -4,7 +4,7 @@ import { claudeAdapter } from '../../src/adapters/claude.js'; describe('claudeAdapter — HeadlessAdapter contract', () => { it('identifies itself as kind "claude"', () => { expect(claudeAdapter.kind).toBe('claude'); - expect(claudeAdapter.defaultModel).toBe('claude-sonnet-4-6'); + expect(claudeAdapter.defaultModel).toBe('claude-opus-5'); }); it('buildIdentification uses the auth-status help shape', () => { diff --git a/packages/sdk/tests/budget-preflight.test.ts b/packages/sdk/tests/budget-preflight.test.ts index 0a43b6e6c..b2649022f 100644 --- a/packages/sdk/tests/budget-preflight.test.ts +++ b/packages/sdk/tests/budget-preflight.test.ts @@ -3,8 +3,9 @@ import { preflight } from '../src/preflight.js'; import { compileSpec, CompileError, toKernelSpec, kernelToAuthoring } from '../src/compile.js'; import { flow } from '@relayflows/surface'; import { getFlowDefinition } from '@relayflows/surface/runtime'; +import { budgetDiagnostics } from '../src/budget-preflight.js'; -const options = () => ({models: ['claude-sonnet-4-6', 'unknown'], probes: { +const options = () => ({models: ['claude-opus-5', 'claude-opus-4-7', 'claude-sonnet-4-6', 'unknown'], probes: { cli: vi.fn(() => ({ exists: true, authenticated: true, modelAvailable: true })), executor: () => true, command: () => true, }}); @@ -48,16 +49,62 @@ describe('budget preflight', () => { const result = preflight({...input, steps:[step]}, o); expect(result.ok).toBe(true); expect(result.resolutions).toContainEqual(expect.objectContaining({ - stepId: 'ask', cli: 'claude', model: 'claude-sonnet-4-6', + stepId: 'ask', cli: 'claude', model: 'claude-opus-5', modelSource: 'adapter', })); - expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-sonnet-4-6'); + expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-opus-5'); }); - it('still requires a model for an unknown/custom CLI under a dollar budget', () => { + it.each(['codex', 'team-wrapper'])('still requires a model for registered/custom CLI %s with no default under a dollar budget', cli => { const input = spec('$20/run'); const { model, ...step } = input.steps[0]!; - expect(preflight({...input, steps:[{...step, cli:'team-wrapper'}]}, options()).diagnostics) + expect(preflight({...input, steps:[{...step, cli}]}, options()).diagnostics) .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price', stepId: 'ask'})])); }); + it('refuses an unpriced adapter default under a frozen dollar budget', () => { + const input = compileSpec(spec('$20/run')); + const { model, ...step } = input.steps[0]!; + expect(budgetDiagnostics({...input, steps:[step]}, new Map([ + ['ask', { model: 'future-default', source: 'adapter' as const }], + ]))).toEqual([expect.objectContaining({ + kind: 'budget_missing_price', stepId: 'ask', model: 'future-default', + })]); + }); + it('resolves and probes the adapter default without requiring price for a token-only budget', () => { + const input = spec({ tokens: 100 }); + const { model, ...step } = input.steps[0]!; + const o = options(); + const result = preflight({...input, steps:[step]}, o); + expect(result.ok).toBe(true); + expect(result.resolutions).toContainEqual(expect.objectContaining({ + stepId: 'ask', model: 'claude-opus-5', modelSource: 'adapter', + })); + expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-opus-5'); + }); + it('uses a selected named-agent model ahead of the Claude adapter default', () => { + const o = options(); + const result = preflight({ + version: '0.1.0', + budget: '$20/run', + agents: { reviewer: { cli: 'claude', model: 'claude-opus-4-7' } }, + steps: [{ id: 'ask', type: 'agent', agent: 'reviewer', instruction: 'Review.' }], + }, o); + expect(result.ok).toBe(true); + expect(result.resolutions).toEqual([expect.objectContaining({ + stepId: 'ask', cli: 'claude', model: 'claude-opus-4-7', modelSource: 'named', + })]); + expect(o.probes.cli).toHaveBeenCalledWith('claude', 'named', 'claude-opus-4-7'); + }); + it('enforces the model allowlist on an implicit Claude default before probing', () => { + const input = spec('$20/run'); + const { model, ...step } = input.steps[0]!; + const o = options(); + const result = preflight({...input, steps:[step]}, { + ...o, models: ['claude-sonnet-4-6'], modelRegistryPath: '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/project/flows.json', + }); + expect(result.diagnostics).toEqual([expect.objectContaining({ + kind: 'model_unknown', stepId: 'ask', model: 'claude-opus-5', + })]); + expect(o.probes.cli).not.toHaveBeenCalled(); + }); it.each(['$1/week', '-$1/run', '$1.0000001/run', {tokens: -1}, {wallclock: 'soon'}, {dollars: Infinity}, {typo: 2}])('refuses malformed budget %j', budget => { expect(preflight(spec(budget), options()).ok).toBe(false); }); diff --git a/packages/sdk/tests/cli-adapter.test.ts b/packages/sdk/tests/cli-adapter.test.ts index 672e4fb34..0fd07ac73 100644 --- a/packages/sdk/tests/cli-adapter.test.ts +++ b/packages/sdk/tests/cli-adapter.test.ts @@ -6,15 +6,23 @@ import { cliAdapterKind, modelReadinessProbe, resolveCliModel, + resolveCliModelSelection, WRAPPER_IDENTIFY_ARG, } from '../src/cli-adapter.js'; describe('typed CLI adapters', () => { it('resolves only Claude to its stable default and preserves explicit models', () => { - expect(resolveCliModel('/usr/local/bin/claude')).toBe('claude-sonnet-4-6'); + expect(resolveCliModel('/usr/local/bin/claude')).toBe('claude-opus-5'); expect(resolveCliModel('/usr/local/bin/claude', 'explicit-model')).toBe('explicit-model'); expect(resolveCliModel('/opt/bin/codex')).toBeUndefined(); expect(resolveCliModel('/project/bin/team-reviewer')).toBeUndefined(); + expect(resolveCliModelSelection('claude', { step: 'step-model', named: 'named-model' })) + .toEqual({ model: 'step-model', source: 'step' }); + expect(resolveCliModelSelection('claude', { named: 'named-model' })) + .toEqual({ model: 'named-model', source: 'named' }); + expect(resolveCliModelSelection('claude')).toEqual({ model: 'claude-opus-5', source: 'adapter' }); + expect(resolveCliModelSelection('codex')).toEqual({}); + expect(resolveCliModelSelection('team-reviewer')).toEqual({}); }); it('maps raw Claude to real auth, noninteractive, and model flag shapes', () => { const kind = cliAdapterKind('/usr/local/bin/claude'); diff --git a/packages/sdk/tests/flow-executor-chain.test.ts b/packages/sdk/tests/flow-executor-chain.test.ts index 3d7bbcf4c..fd171549b 100644 --- a/packages/sdk/tests/flow-executor-chain.test.ts +++ b/packages/sdk/tests/flow-executor-chain.test.ts @@ -70,7 +70,7 @@ describe('flow executor LLM and output-binding chain', () => { const { fixture, client, agent, failures } = await setup(); const calls = join(fixture.root, 'claude-calls.jsonl'); const claude = join(fixture.root, 'claude'); - writeFileSync(join(fixture.root, 'flows.json'), JSON.stringify({ models: ['claude-sonnet-4-6'] })); + writeFileSync(join(fixture.root, 'flows.json'), JSON.stringify({ models: ['claude-opus-5'] })); writeFileSync(claude, `#!/usr/bin/env node import { appendFileSync } from 'node:fs'; appendFileSync(${JSON.stringify(calls)}, JSON.stringify(process.argv.slice(2)) + '\\n'); @@ -96,9 +96,9 @@ process.stdout.write(JSON.stringify({ type: 'result', result: 'default-model-age const invocations = readFileSync(calls, 'utf8').trim().split('\n').map(line => JSON.parse(line) as string[]); const modelScoped = invocations.filter(args => args.includes('--model')); expect(modelScoped).toHaveLength(2); - expect(modelScoped.every(args => args[args.indexOf('--model') + 1] === 'claude-sonnet-4-6')).toBe(true); + expect(modelScoped.every(args => args[args.indexOf('--model') + 1] === 'claude-opus-5')).toBe(true); expect(modelScoped.at(-1)).toEqual([ - '-p', '--dangerously-skip-permissions', '--model', 'claude-sonnet-4-6', + '-p', '--dangerously-skip-permissions', '--model', 'claude-opus-5', '--output-format', 'json', expect.stringContaining('Implement.'), ]); expect(failures).toEqual([]); diff --git a/packages/sdk/tests/model-pricing.test.ts b/packages/sdk/tests/model-pricing.test.ts index bc0c0182b..42f9184e3 100644 --- a/packages/sdk/tests/model-pricing.test.ts +++ b/packages/sdk/tests/model-pricing.test.ts @@ -8,6 +8,12 @@ describe('pricedUsage', () => { expect(usage).toEqual({ tokens_in: 1_000_000, tokens_out: 500_000, dollars: '10.500000' }); }); + it('uses the frozen Claude Opus 5 standard rate', () => { + expect(pricedUsage('claude-opus-5', 1_000_000, 500_000)).toEqual({ + tokens_in: 1_000_000, tokens_out: 500_000, dollars: '17.500000', + }); + }); + it('returns undefined for an unpriced model rather than throwing after decode', () => { // Regression for the Cursor Bugbot HIGH finding "Unlisted models fail // after usage decode": the runtime previously threw here after the CLI diff --git a/packages/sdk/tests/preflight.test.ts b/packages/sdk/tests/preflight.test.ts index 12fc70f7e..cd0f64dbf 100644 --- a/packages/sdk/tests/preflight.test.ts +++ b/packages/sdk/tests/preflight.test.ts @@ -37,7 +37,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { const calls: Array<[string, string, string | undefined]> = []; const authored: FlowSpec = { version: '0.1.0', - agents: { reviewer: { cli: 'wrapper', model: 'allowed-model' } }, + agents: { reviewer: { cli: 'claude', model: 'allowed-model' } }, steps: [{ id: 'review', type: 'agent', agent: 'reviewer', instruction: 'Review.' }], }; @@ -54,11 +54,12 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(result.ok).toBe(true); expect(result.resolutions).toEqual([{ stepId: 'review', - cli: 'wrapper', + cli: 'claude', source: 'named', model: 'allowed-model', + modelSource: 'named', }]); - expect(calls).toEqual([['wrapper', 'named', 'allowed-model']]); + expect(calls).toEqual([['claude', 'named', 'allowed-model']]); }); it('refuses malformed raw input before any public preflight probe', () => { @@ -144,6 +145,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { const unresolved: FlowSpec['steps'][number] = { id: 'unresolved', type: 'agent', + model: 'must-not-check', instruction: 'No CLI is declared.', }; const resolvable: FlowSpec['steps'][number] = { @@ -154,11 +156,14 @@ describe('preflight: CLI resolution and refusal predicates', () => { }; const result = preflight({ version: '0.1.0', + budget: '$1/run', steps: order === 'unresolved first' ? [unresolved, resolvable, { id: 'command', type: 'deterministic', command: 'printf ready' }] : [{ id: 'command', type: 'deterministic', command: 'printf ready' }, resolvable, unresolved], triggers: [{ id: 'trigger', executor: 'must-not-probe' }], }, { + models: ['allowed-model'], + modelRegistryPath: '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/project/flows.json', probes: { cli: () => { calls.push('cli'); return { exists: true, authenticated: true }; }, command: () => { calls.push('command'); return true; }, @@ -492,19 +497,15 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(JSON.stringify(scenarios)).not.toContain('raw secret'); }); - it('reports every model and CLI static refusal on the same step', () => { + it('reports CLI resolution before evaluating model governance on the same step', () => { const result = preflight( flow({ id: 'a', type: 'agent', instruction: 'i', model: 'typo-model' }), { models: ['known-model'], modelRegistryPath: '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/project/flows.json', probes: probes() }, ); - expect(result.diagnostics.map((diagnostic) => diagnostic.kind)).toEqual([ - 'model_unknown', - 'cli_unresolved', - ]); + expect(result.diagnostics.map((diagnostic) => diagnostic.kind)).toEqual(['cli_unresolved']); expect(result.diagnostics[0]).toMatchObject({ stepId: 'a', - model: 'typo-model', }); }); @@ -645,6 +646,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { cli: 'claude', source: 'named', model: 'claude-sonnet-5', + modelSource: 'named', }]); // The probe still ran: model authority does not skip auth verification. expect(probeCalls).toBe(1); @@ -675,6 +677,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { cli: 'claude', source: 'step', model: 'claude-sonnet-5', + modelSource: 'step', }]); expect(probeCalls).toBe(1); }); diff --git a/packages/sdk/tests/verb-field-lint.test.ts b/packages/sdk/tests/verb-field-lint.test.ts index 50c16cf2b..b4255713a 100644 --- a/packages/sdk/tests/verb-field-lint.test.ts +++ b/packages/sdk/tests/verb-field-lint.test.ts @@ -423,8 +423,8 @@ describe('closed per-verb step fields', () => { }); expect(result.ok).toBe(true); expect(result.resolutions).toEqual([ - { stepId: 'answer', cli: 'llm-cli', source: 'step', model: 'project-model' }, - { stepId: 'act', cli: 'agent-cli', source: 'step', model: 'project-model' }, + { stepId: 'answer', cli: 'llm-cli', source: 'step', model: 'project-model', modelSource: 'step' }, + { stepId: 'act', cli: 'agent-cli', source: 'step', model: 'project-model', modelSource: 'step' }, ]); }); diff --git a/packages/sdk/tests/worker-cli.test.ts b/packages/sdk/tests/worker-cli.test.ts index 364166f5b..5819e8ec4 100644 --- a/packages/sdk/tests/worker-cli.test.ts +++ b/packages/sdk/tests/worker-cli.test.ts @@ -73,7 +73,7 @@ process.stdout.write(JSON.stringify({ type: 'result', result: 'default-model-ok' expect(result).toMatchObject({ exit_code: 0, stdout_tail: 'default-model-ok', tokens_input: 2, tokens_output: 1 }); expect(JSON.parse(readFileSync(calls, 'utf8'))).toEqual([ - '-p', '--dangerously-skip-permissions', '--model', 'claude-sonnet-4-6', + '-p', '--dangerously-skip-permissions', '--model', 'claude-opus-5', '--output-format', 'json', 'do the task', ]); }); diff --git a/testdata/flows.json b/testdata/flows.json index 4692d4bc5..3396475ca 100644 --- a/testdata/flows.json +++ b/testdata/flows.json @@ -3,7 +3,6 @@ "executors": ["agent-worker"], "models": [ "claude-haiku-4-5-20251001", - "claude-sonnet-4-6", "claude-sonnet-5", "deterministic-test-stub", "test-model-v1"