Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions docs/BUDGET.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@ Limits are non-negative; dollars support up to six decimal places.

A day is a UTC calendar day within a run. Completed spend resets for admission
at the next UTC day; unrelated runs do not share a global account. Header
syntax errors refuse as `budget_syntax_invalid`. Declared models without a
frozen price refuse as `budget_missing_price`; dollar budgets also require a
model on each worker step. Existing project model allowlist checks still apply.
syntax errors refuse as `budget_syntax_invalid`. Pricing is light enforcement
and never refuses a run: under a dollar budget, an LLM/agent step with no
model, or a model without a frozen price, warns as `budget_unmetered`, runs,
and contributes no dollars. Codex selects its own model, so a Codex step
without a declared model is expected to be unmetered. Priced steps still
accrue dollars and a crossed limit still stops the run as described below.
Existing project model allowlist checks still apply.

Every newly written `step.completed` includes:

Expand Down
6 changes: 4 additions & 2 deletions docs/SURFACE.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,10 @@ No process runs between events: the handler wakes, executes to its next await, p
boundary. CLI and model resolve independently. CLI priority is step → named
declaration → flow → project config. Model priority is step → named
declaration → registered adapter default. Claude's adapter default is
`claude-opus-5`; Codex and custom wrappers have no default. A frozen dollar
budget refuses before execution when the selected model has no frozen price.
`claude-opus-5`; Codex and custom wrappers have no default. Under a frozen
dollar budget, a step whose model has no frozen price (or no model, as with
Codex choosing its own) warns `budget_unmetered` and runs without accruing
dollars; pricing never refuses.
The worker explicitly removes ambient `RELAYFLOW_MODEL`; raw provider
adapters use a model flag, while a custom wrapper receives an explicitly
declared model only inside its identified same-process session.
Expand Down
4 changes: 2 additions & 2 deletions packages/sdk/src/adapters/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ export interface HeadlessAdapter {

/**
* Stable model used only when neither the step nor its selected named agent
* declares one. Explicit authoring always wins. A default used with frozen
* dollar budgets must also have an entry in MODEL_PRICING.
* declares one. Explicit authoring always wins. A default without an entry
* in MODEL_PRICING runs unmetered under a dollar budget (a warning).
*/
readonly defaultModel?: string;

Expand Down
1 change: 0 additions & 1 deletion packages/sdk/src/authored-flow-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ export type AuthoredFlowExecutionErrorCode =
| 'helper_slack.credential_missing'
| 'helper_slack.mount_required'
| 'budget_syntax_invalid'
| 'budget_missing_price'
| 'agent_cli_unresolved'
| 'agent_parked'
| 'llm_cli_unresolved'
Expand Down
2 changes: 1 addition & 1 deletion packages/sdk/src/authored-worker-step.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export function authoredWorkerRunner(
diagnostic.severity === 'refusal',
);
throw new AuthoredFlowExecutionError(
refusal?.kind === 'budget_missing_price' || refusal?.kind === 'budget_syntax_invalid' ? refusal.kind
refusal?.kind === 'budget_syntax_invalid' ? refusal.kind
: step.type === 'llm' ? 'llm_cli_unresolved' : 'agent_cli_unresolved',
refusal?.message
?? `flow "${definition.name}" step "${id}": no CLI could be resolved for f.${step.type} `
Expand Down
69 changes: 37 additions & 32 deletions packages/sdk/src/budget-preflight.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,47 @@
import type { CompiledFlowSpec } from './compile.js';
import type { PreflightRefusal } from './preflight.js';
import { MODEL_PRICING } from './model-pricing.js';
import type { ResolvedCliModel } from './cli-adapter.js';
import type { PreflightWarning } from './preflight.js';
import { hasPricing } from './model-pricing.js';
import { resolveAdapterKind } from './adapters/index.js';

/** Frozen dollar budgets require an exact model with a frozen table price. */
export interface BudgetStepResolution {
readonly cli?: string;
readonly model?: string;
}

/**
* Dollar budgets are light enforcement: pricing never refuses a run. A step
* whose model has no frozen price journals zero dollars (see `workerSpend`),
* so it cannot trip `maxDollars`, while its reported tokens still count toward
* any token budget; priced steps still accrue and a crossed limit still stops
* the run in the kernel. This warning names each dollar-unmetered step so the
* gap is reported, not silent.
*
* Codex selects its own model when none is declared, so a Codex step is
* expected to be unmetered and says so rather than asking for a fake price.
*/
export function budgetDiagnostics(
flow: CompiledFlowSpec,
resolvedModels: ReadonlyMap<string, ResolvedCliModel> = new Map(),
): PreflightRefusal[] {
if (flow.budget?.pricing !== 'frozen') return [];
const diagnostics: PreflightRefusal[] = [];
const declared = [
...Object.entries(flow.agents ?? {}).map(([agent, d]) => ({ agent, model: d.model })),
...flow.steps.flatMap(s => s.type !== 'deterministic' && s.model !== undefined
? [{ stepId: s.id, model: s.model }] : []),
];
resolved: ReadonlyMap<string, BudgetStepResolution> = new Map(),
): PreflightWarning[] {
if (flow.budget?.pricing !== 'frozen' || flow.budget.maxDollars === undefined) return [];
const warnings: PreflightWarning[] = [];
for (const step of flow.steps) {
if (step.type === 'deterministic' || flow.budget.maxDollars === undefined) continue;
const resolved = resolvedModels.get(step.id);
const model = resolved?.model
if (step.type === 'deterministic') continue;
const resolution = resolved.get(step.id);
const model = resolution?.model
?? step.model ?? (step.type === 'agent' && step.agent !== undefined
? flow.agents?.[step.agent]?.model : undefined);
if (model === undefined) diagnostics.push({
severity: 'refusal', kind: 'budget_missing_price', stepId: step.id,
message: `Step "${step.id}" needs a declared, priced model for its dollar budget.`,
});
else if (resolved?.source === 'adapter'
&& !Object.hasOwn(MODEL_PRICING, model)) diagnostics.push({
severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, model,
message: `Model "${model}" has no frozen price for budget accounting.`,
});
}
for (const declaration of declared) {
if (Object.hasOwn(MODEL_PRICING, declaration.model)) continue;
diagnostics.push({
severity: 'refusal', kind: 'budget_missing_price', ...declaration,
message: `Model "${declaration.model}" has no frozen price for budget accounting.`,
if (hasPricing(model)) continue;
const cli = resolution?.cli ?? step.cli;
const reason = model !== undefined
? `model "${model}" has no frozen price`
: cli !== undefined && resolveAdapterKind(cli) === 'codex'
? 'Codex selects its own model'
: 'no model is declared';
warnings.push({
severity: 'warning', kind: 'budget_unmetered', stepId: step.id,
message: `Step "${step.id}" is unmetered (${reason}); it does not count toward the dollar budget.`,
Comment thread
cursor[bot] marked this conversation as resolved.
});
}
return diagnostics;
return warnings;
}
2 changes: 0 additions & 2 deletions packages/sdk/src/cli/direct-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ export async function runDirectFlow(
&& (error.code === 'helper_slack.credential_missing'
|| error.code === 'helper_slack.mount_required'
|| error.code === 'budget_syntax_invalid'
|| error.code === 'budget_missing_price'
|| error.code === 'unsupported_promise_lifecycle'
|| error.code === 'unsupported_header'
|| error.code === 'agent_cli_unresolved'
Expand All @@ -167,7 +166,6 @@ export async function runDirectFlow(
error.code === 'helper_slack.credential_missing'
|| error.code === 'helper_slack.mount_required'
|| error.code === 'budget_syntax_invalid'
|| error.code === 'budget_missing_price'
) ? error.code : 'invalid_spec',
message: error.message,
}, path)),
Expand Down
6 changes: 5 additions & 1 deletion packages/sdk/src/failure-kinds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ const PREFLIGHT_ENVIRONMENT_FAILURE_KINDS = [
'mcp_undeclared_server',
'mcp_unreachable',
'budget_syntax_invalid',
'budget_missing_price',
'cli_missing',
'cli_unauthenticated',
'cli_unresolved',
Expand Down Expand Up @@ -61,12 +60,17 @@ export const CHECK_FAILURE_KINDS = [
* `vacuous_gate` is the same principle applied to a declared gate that judges
* nothing: `schema: {}` and `schema: true` are legal and accepted, but a gate
* accepting every output must not be reported as if it constrained one.
*
* `budget_unmetered` names an LLM/agent step under a dollar budget whose model
* has no frozen price (including Codex, which selects its own model). Pricing
* is light enforcement: the step runs and simply contributes no dollars.
*/
export const PREFLIGHT_WARNING_KINDS = [
'unprovable_effects',
'command_unresolved',
'command_unprovable',
'vacuous_gate',
'budget_unmetered',
] as const;

/**
Expand Down
7 changes: 3 additions & 4 deletions packages/sdk/src/model-pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@ export function hasPricing(model: string | undefined): boolean {
*
* Returns `undefined` for unpriced models — callers should omit `usage`
* from the journal payload rather than sending nulls that break the kernel
* wire schema. The refusal for a declared dollar budget against an unpriced
* model is `budgetDiagnostics` at preflight (before any CLI dispatches).
* Throwing here after usage decode would waste the CLI invocation that
* preflight was meant to prevent.
* wire schema. An unpriced step is unmetered, not refused: it contributes no
* dollars to a budget, and `budgetDiagnostics` warns about it at preflight.
* Codex model ids need no entry here; Codex selects its own model.
*/
export function pricedUsage(model: string | undefined, input = 0, output = 0):
| { tokens_in: number; tokens_out: number; dollars: string }
Expand Down
4 changes: 2 additions & 2 deletions packages/sdk/src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,11 +249,11 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul
diagnostics.push(...budgetDiagnostics(
compiled,
new Map(resolutions.map(resolution => [resolution.stepId, {
cli: resolution.cli,
...(resolution.model === undefined ? {} : { model: resolution.model }),
...(resolution.modelSource === undefined ? {} : { source: resolution.modelSource }),
}])),
));
if (diagnostics.length > 0) {
if (diagnostics.some((diagnostic) => diagnostic.severity === 'refusal')) {
return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics };
}

Expand Down
19 changes: 15 additions & 4 deletions packages/sdk/src/worker-spend.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,28 @@
import { pricedUsage } from './model-pricing.js';
import type { WorkerCliResult } from './worker-cli.js';

/** Zero-dollar usage for an unpriced step, only when the CLI reported tokens. */
function unmeteredUsage(input: number | undefined, output: number | undefined) {
if (input === undefined || output === undefined) return undefined;
return { tokens_in: input, tokens_out: output, dollars: '0.000000' };
}

/**
* Attach token/dollar usage to a worker's CLI result. Invalid token counts
* (non-integer or negative) are the one remaining failure mode — those are
* journaled as `worker_error` with the usage projected from clamped counts.
*
* Unpriced models are NOT a failure here: `pricedUsage` returns
* `dollars: null` and preflight (see `budgetDiagnostics`) has already refused
* declared dollar budgets against unpriced models before the CLI dispatched.
* Unpriced models are NOT a failure here: the step journals zero dollars, so it
* never trips a dollar budget, but its reported tokens still count toward any
* token budget. Preflight (see `budgetDiagnostics`) warns that such a step is
* unmetered for dollars.
*/
export function workerSpend(result: WorkerCliResult, model?: string) {
try { return { result, usage: pricedUsage(model, result.tokens_input, result.tokens_output) }; }
try {
const usage = pricedUsage(model, result.tokens_input, result.tokens_output)
?? unmeteredUsage(result.tokens_input, result.tokens_output);
return { result, usage };
}
catch (error) {
return {
result: { ...result, exit_code: null, stderr_tail: error instanceof Error ? error.message : 'Invalid model usage' },
Expand Down
62 changes: 51 additions & 11 deletions packages/sdk/tests/budget-preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,41 @@ describe('budget preflight', () => {
expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_syntax_invalid']);
expect(o.probes.cli).not.toHaveBeenCalled();
});
it('refuses an unpriced declared model before probing', () => {
it('warns, never refuses, on an unpriced declared model and still probes', () => {
const o = options();
const result = preflight(spec('$20/run', 'unknown'), o);
expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_missing_price']);
expect(o.probes.cli).not.toHaveBeenCalled();
expect(result.ok).toBe(true);
expect(result.diagnostics).toEqual([expect.objectContaining({
severity: 'warning', kind: 'budget_unmetered', stepId: 'ask',
message: expect.stringContaining('model "unknown" has no frozen price'),
})]);
expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'unknown');
});
it('retains frozen pricing when checking a compiled artifact', () => {
expect(preflight(kernelToAuthoring(toKernelSpec(compileSpec(spec('$20/run', 'unknown')))), options()).diagnostics)
.toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})]));
const result = preflight(kernelToAuthoring(toKernelSpec(compileSpec(spec('$20/run', 'unknown')))), options());
expect(result.ok).toBe(true);
expect(result.diagnostics).toEqual(expect.arrayContaining([
expect.objectContaining({severity: 'warning', kind: 'budget_unmetered'}),
]));
});
it('emits no budget warning for priced steps or for token-only budgets', () => {
expect(preflight(spec('$20/run'), options()).diagnostics).toEqual([]);
expect(preflight(spec({ tokens: 100 }, 'unknown'), options()).diagnostics).toEqual([]);
});
it('lets a Codex step without a model run unmetered beside a priced Claude step (burn#539)', () => {
const o = options();
const result = preflight({ version: '0.1.0', budget: '$8/run', steps: [
{ id: 'planner', type: 'agent', cli: 'claude', instruction: 'Plan.' },
{ id: 'plan-reviewer', type: 'agent', cli: 'codex', instruction: 'Review.', dependsOn: ['planner'] },
] }, o);
expect(result.ok).toBe(true);
expect(result.diagnostics.filter(d => d.severity === 'refusal')).toEqual([]);
expect(result.diagnostics).toEqual([expect.objectContaining({
severity: 'warning', kind: 'budget_unmetered', stepId: 'plan-reviewer',
message: expect.stringContaining('Codex selects its own model'),
})]);
expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-opus-5');
expect(o.probes.cli).toHaveBeenCalledWith('codex', 'step', undefined);
});
it('prices and probes the Claude default when a dollar-budgeted step omits model', () => {
const input = spec('$20/run');
Expand All @@ -53,19 +79,33 @@ describe('budget preflight', () => {
}));
expect(o.probes.cli).toHaveBeenCalledWith('claude', 'step', 'claude-opus-5');
});
it.each(['codex', 'team-wrapper'])('still requires a model for registered/custom CLI %s with no default under a dollar budget', cli => {
it.each([
['codex', 'Codex selects its own model'],
['team-wrapper', 'no model is declared'],
])('runs registered/custom CLI %s with no default model unmetered under a dollar budget', (cli, reason) => {
const input = spec('$20/run');
const { model, ...step } = input.steps[0]!;
expect(preflight({...input, steps:[{...step, cli}]}, options()).diagnostics)
.toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price', stepId: 'ask'})]));
const result = preflight({...input, steps:[{...step, cli}]}, options());
expect(result.ok).toBe(true);
expect(result.diagnostics).toEqual([expect.objectContaining({
severity: 'warning', kind: 'budget_unmetered', stepId: 'ask', message: expect.stringContaining(reason),
})]);
});
it('does not require Codex model ids to be priced', () => {
const input = compileSpec(spec('$20/run'));
const { model, ...step } = input.steps[0]!;
expect(budgetDiagnostics({...input, steps:[{...step, cli: 'codex'}]}, new Map([
['ask', { cli: 'codex', model: 'gpt-5.2-codex' }],
]))).toEqual([expect.objectContaining({ severity: 'warning', kind: 'budget_unmetered', stepId: 'ask' })]);
});
it('refuses an unpriced adapter default under a frozen dollar budget', () => {
it('warns on an unpriced adapter default under a frozen dollar budget', () => {
const input = compileSpec(spec('$20/run'));
const { model, ...step } = input.steps[0]!;
expect(budgetDiagnostics({...input, steps:[step]}, new Map([
['ask', { model: 'future-default', source: 'adapter' as const }],
['ask', { cli: 'claude', model: 'future-default' }],
]))).toEqual([expect.objectContaining({
kind: 'budget_missing_price', stepId: 'ask', model: 'future-default',
severity: 'warning', kind: 'budget_unmetered', stepId: 'ask',
message: expect.stringContaining('"future-default"'),
})]);
});
it('resolves and probes the adapter default without requiring price for a token-only budget', () => {
Expand Down
23 changes: 16 additions & 7 deletions packages/sdk/tests/model-pricing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,22 @@ describe('workerSpend', () => {
expect(spent.result.exit_code).toBe(0);
});

it('leaves usage undefined for an unpriced model without failing the step', () => {
// The step still succeeded — an unpriced model is a preflight concern
// when a dollar budget is declared, not a runtime failure per se.
const spent = workerSpend(priced, 'unlisted-model');
expect(spent.usage).toBeUndefined();
expect(spent.result.exit_code).toBe(0);
expect(spent.result.stderr_tail).toBe('');
it('keeps an unpriced model step metered for tokens but not dollars, without failing it', () => {
// The step still succeeded — an unpriced model is a preflight warning
// when a dollar budget is declared, not a runtime failure. Its tokens must
// still reach the kernel so token budgets see the step.
for (const model of ['unlisted-model', undefined]) {
const spent = workerSpend(priced, model);
expect(spent.usage).toEqual({ tokens_in: 100, tokens_out: 50, dollars: '0.000000' });
expect(spent.result.exit_code).toBe(0);
expect(spent.result.stderr_tail).toBe('');
}
});

it('attaches no usage for an unpriced model whose CLI reported no tokens', () => {
const unreported = { exit_code: 0, stdout_tail: '', stderr_tail: '' };
expect(workerSpend(unreported, 'unlisted-model').usage).toBeUndefined();
expect(workerSpend(unreported).usage).toBeUndefined();
});

it('journals invalid token counts as worker_error, projecting clamped counts', () => {
Expand Down
5 changes: 5 additions & 0 deletions packages/sdk/tests/preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,11 @@ describe('preflight: CLI resolution and refusal predicates', () => {
} as never),
{ probes: probes() },
),
// Pricing is light enforcement: an unmetered step under a dollar budget warns.
preflight(
{ ...flow({ id: 'a', type: 'llm', cli: 'codex', prompt: 'x' } as never), budget: '$1/run' } as never,
{ probes: probes() },
),
];
const warningKinds = scenarios.flatMap((result) => result.diagnostics)
.filter((diagnostic) => diagnostic.severity === 'warning')
Expand Down
Loading