From 81c49df32aebcbb1b21fc3f87eeae55cb2ed6ff1 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 16:14:31 +0200 Subject: [PATCH 01/33] feat(sdk): compile typed outputs to json_schema Add generic JSON output schemas to llm and agent StepSpec authoring. Compile output declarations into the existing kernel verification.json_schema primitive, reject ambiguous or malformed declarations before submission, and keep the kernel vocabulary unchanged. Pin both sides of persistence: valid agent JSON is journaled as the parsed value, while a schema mismatch journals null plus a failed json_schema verdict. Migrate the hn-monitor authoring YAML to the new sugar without changing its canonical kernel step. Refs #132 Session-Id: 01a06263-743a-7370-bf22-58d2512c5eee --- sdk/src/compile.ts | 22 ++++- sdk/src/index.ts | 2 + sdk/src/output-schema.ts | 35 +++++++ sdk/src/spec.ts | 17 +++- sdk/src/validate.ts | 7 +- sdk/tests/live-kernel.test.ts | 36 +++++-- sdk/tests/typed-output.test.ts | 170 +++++++++++++++++++++++++++++++++ testdata/hn-monitor.flow.yaml | 33 ++++--- 8 files changed, 294 insertions(+), 28 deletions(-) create mode 100644 sdk/src/output-schema.ts create mode 100644 sdk/tests/typed-output.test.ts diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index aeecde366..86d2d5d90 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -84,11 +84,12 @@ export function compileSpec(spec: unknown): FlowSpec { function compileStep(step: StepSpec): StepSpec { const maxIterations = step.maxIterations ?? 1; + const verification = typedOutputVerification(step); const base = { id: step.id, type: step.type, ...(step.dependsOn !== undefined ? { dependsOn: step.dependsOn } : {}), - ...(step.verification !== undefined ? { verification: step.verification } : {}), + ...(verification !== undefined ? { verification } : {}), maxIterations, ...(step.timeoutMs !== undefined ? { timeoutMs: step.timeoutMs } : {}), }; @@ -130,6 +131,13 @@ function compileStep(step: StepSpec): StepSpec { } } +function typedOutputVerification(step: StepSpec): StepSpec['verification'] { + if (step.type !== 'deterministic' && step.output !== undefined) { + return { type: 'json_schema', schema: step.output }; + } + return step.verification; +} + // Kernel defaults, materialized at compile time so the emitted spec is // byte-identical to the kernel's own serialization of it (spec.rs defaults). const KERNEL_RETRY_DEFAULTS = { @@ -389,6 +397,18 @@ function requireNoTimeout(step: StepSpec): void { } function toKernelVerification(step: StepSpec): KernelVerificationSpec { + const output = step.type === 'deterministic' ? undefined : step.output; + if (output !== undefined) { + if (step.verification !== undefined) { + throw new CompileError([ + `step "${step.id}": output already declares json_schema verification; remove verification`, + ]); + } + if (!isObject(output)) { + throw new CompileError([`step "${step.id}".output: expected a JSON Schema object`]); + } + return { json_schema: output }; + } const gate = step.verification; // No gate / explicit exit_code both compile to {}: exit_code == 0 is the // kernel's implicit gate for deterministic steps (kernel DESIGN.md §4). diff --git a/sdk/src/index.ts b/sdk/src/index.ts index f6ba94455..01d96e379 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -11,6 +11,7 @@ export type { ExitCodeGate, FlowSpec, JsonSchemaGate, + JsonOutputSchema, KernelAgentStep, KernelAgentSurfaces, KernelBudgetSpec, @@ -25,6 +26,7 @@ export type { KernelVerificationSpec, LlmStepSpec, OutputContainsGate, + OutputFromSchema, PermissionsSpec, RecoveryMode, StreamSurface, diff --git a/sdk/src/output-schema.ts b/sdk/src/output-schema.ts new file mode 100644 index 000000000..8500f68d4 --- /dev/null +++ b/sdk/src/output-schema.ts @@ -0,0 +1,35 @@ +/** Type-only marker carried by authoring schemas and erased at runtime. */ +declare const outputSchemaType: unique symbol; + +/** + * JSON Schema with a TypeScript-only output type. The symbol property is + * optional and never exists in emitted specs, so ordinary JSON Schema objects + * remain the authoring value while TypeScript gates can recover `TOutput`. + */ +export interface JsonOutputSchema extends Record { + readonly [outputSchemaType]?: TOutput; +} + +/** Recover the parsed value type carried by a {@link JsonOutputSchema}. */ +export type OutputFromSchema = + TSchema extends JsonOutputSchema ? TOutput : never; + +/** Validate authoring sugar before it can be compiled or submitted. */ +export function validateOutputDeclaration( + step: { output?: unknown; verification?: unknown }, + at: string, +): string[] { + if (step.output === undefined) return []; + const errors: string[] = []; + if (!isObject(step.output)) { + errors.push(`${at}.output: expected a JSON Schema object`); + } + if (step.verification !== undefined) { + errors.push(`${at}: output already declares json_schema verification; remove verification`); + } + return errors; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index 613293efe..2ce67fb86 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -9,6 +9,9 @@ // Zero-agent flows are legal: a spec with only `deterministic` (and/or `llm`) // steps is valid. Nothing here requires an `agent` step. +import type { JsonOutputSchema } from './output-schema.js'; +export type { JsonOutputSchema, OutputFromSchema } from './output-schema.js'; + /** The three rungs of the ladder (RFC §1; AGENTS.md rule 7). */ export type StepType = 'deterministic' | 'llm' | 'agent'; @@ -119,12 +122,17 @@ export interface DeterministicStepSpec extends BaseStepSpec { * never calls a model: it dispatches to an attached SDK worker (§5) which * returns `{output, usage}`; the kernel then runs the verification gate. */ -export interface LlmStepSpec extends BaseStepSpec { +export interface LlmStepSpec extends BaseStepSpec { type: 'llm'; prompt: string; model?: string; /** Inert preflight declaration; overrides the flow/project CLI default. */ cli?: string; + /** + * Typed-output authoring sugar. Compiles to the existing `json_schema` + * verification primitive and is removed before the kernel boundary. + */ + output?: JsonOutputSchema; } /** @@ -133,7 +141,7 @@ export interface LlmStepSpec extends BaseStepSpec { * every writeback is a journaled `effect.recorded` deduped by * `(step_id, idempotency_key, surface_path)`. */ -export interface AgentStepSpec extends BaseStepSpec { +export interface AgentStepSpec extends BaseStepSpec { type: 'agent'; instruction: string; /** Inert preflight declaration; overrides the flow/project CLI default. */ @@ -149,6 +157,11 @@ export interface AgentStepSpec extends BaseStepSpec { surfaces?: AgentSurfaces; recoveryMode?: RecoveryMode; permissions?: PermissionsSpec; + /** + * Typed-output authoring sugar. A successful CLI JSON object is the parsed + * value; the kernel persists it only after `json_schema` verification. + */ + output?: JsonOutputSchema; } export type StepSpec = DeterministicStepSpec | LlmStepSpec | AgentStepSpec; diff --git a/sdk/src/validate.ts b/sdk/src/validate.ts index dbe0ff9fb..a0bd73405 100644 --- a/sdk/src/validate.ts +++ b/sdk/src/validate.ts @@ -17,6 +17,7 @@ import type { VerificationSpec, } from './spec.js'; import { SPEC_SCHEMA_VERSION } from './spec.js'; +import { validateOutputDeclaration } from './output-schema.js'; export interface ValidationResult { ok: boolean; @@ -46,8 +47,8 @@ const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars'] as const; const STEP_COMMON_KEYS = ['id', 'type', 'dependsOn', 'verification', 'maxIterations', 'timeoutMs'] as const; const STEP_TYPE_KEYS: Record = { deterministic: ['command'], - llm: ['prompt', 'model', 'cli'], - agent: ['instruction', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], + llm: ['prompt', 'model', 'cli', 'output'], + agent: ['instruction', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions', 'output'], }; const VERIFICATION_KEYS: Record = { exit_code: ['type', 'expect'], @@ -248,8 +249,10 @@ class Validator { this.validateDeterministic(st as unknown as DeterministicStepSpec, at); } else if (type === 'llm') { this.validateLlm(st as unknown as LlmStepSpec, at); + for (const error of validateOutputDeclaration(st, at)) this.fail(error); } else { this.validateAgent(st as unknown as AgentStepSpec, at); + for (const error of validateOutputDeclaration(st, at)) this.fail(error); } } diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index 5ae7a9b01..7f8cf1845 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -370,8 +370,8 @@ steps: await worker.attach(); // Load hn-monitor's canonical spec, then patch the analyze-story - // step to declare a real CLI. Everything else — triggers, dedupe, - // wake context — comes straight from the existing gate-2 spec. + // step to declare a real CLI. The canonical fixture is separately pinned + // to the authoring YAML, whose `output:` compiles to this json_schema. const spec = JSON.parse( readFileSync(join(TESTDATA, 'hn-monitor.spec.canonical.json'), 'utf8'), ) as { steps: { id: string; cli?: string }[] }; @@ -400,6 +400,18 @@ steps: state: 'done', }); const finalEntries = (await client.journalRead(runId)).entries; + const stepCompleted = finalEntries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'analyze-story', + ) as { payload: { output: unknown; verification: unknown } } | undefined; + expect(stepCompleted?.payload).toMatchObject({ + output: { + story_title: 'stub', + relevance_score: 5, + reasoning: 'stub agent runtime — deterministic output for gate-2 clause-2 demo', + }, + verification: { gate: 'json_schema', verdict: 'pass' }, + }); const runCompleted = finalEntries.find( (entry) => (entry as { entry_type: string }).entry_type === 'run.completed', ) as { payload: { completionReason: string } } | undefined; @@ -464,8 +476,15 @@ steps: // a "hasn't completed yet" absence. const deadline = Date.now() + 10_000; let runCompleted: { payload: { completionReason: string } } | undefined; + let stepCompleted: { + payload: { completionReason: string; output: unknown; verification: unknown }; + } | undefined; while (Date.now() < deadline) { const entries = (await client.journalRead(runId)).entries; + stepCompleted = entries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'analyze-story', + ) as typeof stepCompleted; runCompleted = entries.find( (entry) => (entry as { entry_type: string }).entry_type === 'run.completed', ) as { payload: { completionReason: string } } | undefined; @@ -476,10 +495,15 @@ steps: runCompleted, 'run.completed entry never arrived within 10s — test cannot assert schema-live under a hung run', ).toBeDefined(); - // step_failed is the outer reason (a step failed → run failed); - // the inner step.completed record carries verification_failed - // for schema-rejected outputs. Pinning the outer reason avoids - // depending on retry/backoff behavior for this test. + // step_failed is the outer reason (a step failed → run failed). With one + // allowed semantic execution, the step records retries_exhausted while + // its verification record names the json_schema rejection. The rejected + // parsed value is nulled before the completion is persisted. + expect(stepCompleted?.payload).toMatchObject({ + completionReason: 'retries_exhausted', + output: null, + verification: { gate: 'json_schema', verdict: 'fail' }, + }); expect(runCompleted!.payload.completionReason).toBe('step_failed'); await worker.close(); diff --git a/sdk/tests/typed-output.test.ts b/sdk/tests/typed-output.test.ts new file mode 100644 index 000000000..3b11872b6 --- /dev/null +++ b/sdk/tests/typed-output.test.ts @@ -0,0 +1,170 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, expectTypeOf, it } from 'vitest'; +import { + compileSpec, + compileYaml, + compileYamlToCanonicalJson, + CompileError, + toKernelSpec, +} from '../src/compile.js'; +import type { + AgentStepSpec, + JsonOutputSchema, + LlmStepSpec, + OutputFromSchema, +} from '../src/spec.js'; + +interface Extraction { + actionable: boolean; + request: string; +} + +const TESTDATA = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'testdata'); + +const extractionSchema: JsonOutputSchema = { + type: 'object', + additionalProperties: false, + required: ['actionable', 'request'], + properties: { + actionable: { type: 'boolean' }, + request: { type: 'string' }, + }, +}; + +describe('typed llm and agent outputs', () => { + it('carries a schema output type for TypeScript gates', () => { + expectTypeOf>().toEqualTypeOf(); + + const llm: LlmStepSpec = { + id: 'extract', + type: 'llm', + prompt: 'Extract the request.', + output: extractionSchema, + }; + const agent: AgentStepSpec = { + id: 'research', + type: 'agent', + instruction: 'Research the request.', + output: extractionSchema, + }; + + expectTypeOf(llm.output).toEqualTypeOf | undefined>(); + expectTypeOf(agent.output).toEqualTypeOf | undefined>(); + }); + + it.each(['llm', 'agent'] as const)( + 'compiles %s output sugar to the existing json_schema primitive', + (type) => { + const source = { + version: '0.1.0', + steps: [{ + id: 'typed', + type, + ...(type === 'llm' ? { prompt: 'Return JSON.' } : { instruction: 'Return JSON.' }), + output: extractionSchema, + }], + }; + + const compiled = compileSpec(source); + expect(compiled.steps[0]).not.toHaveProperty('output'); + expect(compiled.steps[0]?.verification).toEqual({ + type: 'json_schema', + schema: extractionSchema, + }); + expect(toKernelSpec(compiled).steps[0]?.verification).toEqual({ + json_schema: extractionSchema, + }); + }, + ); + + it('accepts the output declaration in YAML', () => { + const compiled = compileYaml(` +version: '0.1.0' +steps: + - id: extract + type: llm + prompt: Return JSON. + output: + type: object + required: [answer] + properties: + answer: { type: number } +`); + + expect(compiled.steps[0]?.verification).toEqual({ + type: 'json_schema', + schema: { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + }, + }); + }); + + it('keeps the canonical hn-monitor kernel step unchanged', () => { + const yaml = readFileSync(join(TESTDATA, 'hn-monitor.flow.yaml'), 'utf8'); + const compiled = JSON.parse(compileYamlToCanonicalJson(yaml)) as { steps: unknown[] }; + const canonical = JSON.parse( + readFileSync(join(TESTDATA, 'hn-monitor.spec.canonical.json'), 'utf8'), + ) as { steps: unknown[] }; + + // Trigger key normalization is a separate pre-existing surface gap; this + // assertion is deliberately about the step boundary changed in this PR. + expect(compiled.steps).toEqual(canonical.steps); + }); + + it.each([ + { type: 'output_contains', value: 'done' }, + { type: 'json_schema', schema: extractionSchema }, + ])('fails closed when output conflicts with verification ($type)', (verification) => { + expect(() => compileSpec({ + version: '0.1.0', + steps: [{ + id: 'ambiguous', + type: 'agent', + instruction: 'Return JSON.', + output: extractionSchema, + verification, + }], + })).toThrowError(CompileError); + + try { + compileSpec({ + version: '0.1.0', + steps: [{ + id: 'ambiguous', + type: 'agent', + instruction: 'Return JSON.', + output: extractionSchema, + verification, + }], + }); + } catch (error) { + expect((error as CompileError).errors).toContain( + 'spec.steps[0]: output already declares json_schema verification; remove verification', + ); + } + }); + + it.each([null, [], 'not-a-schema'])( + 'fails closed on a non-object output schema (%j)', + (output) => { + const result = (() => { + try { + compileSpec({ + version: '0.1.0', + steps: [{ id: 'bad', type: 'llm', prompt: 'Return JSON.', output }], + }); + return null; + } catch (error) { + return error as CompileError; + } + })(); + + expect(result).toBeInstanceOf(CompileError); + expect(result?.errors).toContain('spec.steps[0].output: expected a JSON Schema object'); + }, + ); +}); diff --git a/testdata/hn-monitor.flow.yaml b/testdata/hn-monitor.flow.yaml index ec7d9003d..37e73cf21 100644 --- a/testdata/hn-monitor.flow.yaml +++ b/testdata/hn-monitor.flow.yaml @@ -24,20 +24,19 @@ steps: is relevant to AI agents/automation. Output a JSON summary with: story title, relevance score (1-10), and reasoning. recoveryMode: reset - verification: - type: json_schema - schema: - type: object - required: - - story_title - - relevance_score - - reasoning - properties: - story_title: - type: string - relevance_score: - type: integer - minimum: 1 - maximum: 10 - reasoning: - type: string + # Surface sugar: compiles to the kernel's existing json_schema gate. + output: + type: object + required: + - story_title + - relevance_score + - reasoning + properties: + story_title: + type: string + relevance_score: + type: integer + minimum: 1 + maximum: 10 + reasoning: + type: string From 5ab0fee366362de33ea1bd7d88a96daef1cad18a Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 17:10:52 +0200 Subject: [PATCH 02/33] fix(sdk): enforce structured output contracts Session-Id: 01a060b1-3228-74b1-806b-f4a3393d6b37 --- .github/workflows/cloud-runtime-artifact.yml | 15 +++- docs/SURFACE.md | 25 ++++++ sdk/package.json | 3 +- sdk/src/compile.ts | 13 ++- sdk/src/index.ts | 1 - sdk/src/output-schema.ts | 17 +--- sdk/src/spec.ts | 14 ++-- sdk/tests/typed-output.test.ts | 85 ++++++++++++++------ sdk/tsconfig.tests.json | 10 +++ 9 files changed, 127 insertions(+), 56 deletions(-) create mode 100644 sdk/tsconfig.tests.json diff --git a/.github/workflows/cloud-runtime-artifact.yml b/.github/workflows/cloud-runtime-artifact.yml index c81cf655c..eb213bb3d 100644 --- a/.github/workflows/cloud-runtime-artifact.yml +++ b/.github/workflows/cloud-runtime-artifact.yml @@ -42,9 +42,22 @@ jobs: working-directory: kernel run: cargo build --locked --release -p relayflowd + - name: Install SDK dependencies + run: npm ci --prefix sdk + + - name: Test SDK and type-level authoring contracts + working-directory: sdk + run: | + npm run build + npm run typecheck:tests + ./node_modules/.bin/vitest run \ + tests/typed-output.test.ts \ + tests/validate.test.ts \ + tests/spec-parity.test.ts \ + tests/deterministic-llm.test.ts + - name: Build standalone flows CLI run: | - npm ci --prefix sdk mkdir -p dist/cloud-artifact-input bun build sdk/src/cli-executable.ts \ --compile \ diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 4ea7f3131..a14ca5b59 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -91,6 +91,31 @@ No process runs between events: the handler wakes, executes to its next await, p **Project-config discovery:** starting in the flow file's directory, `flows check` walks parent directories through the filesystem root and selects the first readable `flows.json`. That nearest file is the whole project config; it is not merged with outer files. Its schema is `{ "cli"?: , "executors"?: [] }`; unknown keys fail closed as `config_invalid`. A nearer config therefore defines a self-contained nested project boundary and prevents accidental inheritance of outer credentials or executors. The selected path is printed with project-level resolutions and named in an unresolved-CLI refusal; if it declares no `cli`, outer configs remain shadowed. At gate 1, a trigger executor is considered registered only when its name is present in this author-written `executors` array; `flows check` does not yet contact a registry, broker, or RelayCron, and absence is `no_executor`. 7. **Two dialects, one journal.** Declarative YAML — data, fully preflightable, sage's compile target, gate 9's self-authoring output. Imperative TS — journal-memoized function, maximum ergonomics. YAML is canonical; TS is the power tool. TS preflights its declared surface (agents, helpers, tools, identity), not arbitrary control flow — declared honestly per covenant 2. +### Structured output declarations + +Declarative `llm` and `agent` steps may declare an `output` JSON Schema. This +is authoring sugar for the existing kernel `json_schema` verification gate; the +compiler removes `output` before the journal boundary and emits the schema as +`verification.json_schema`. Authors must choose either `output` or an explicit +`verification` block. Declaring both is ambiguous and fails closed. + +```yaml +- id: extract + type: llm + prompt: Return the actionable request as JSON. + output: + type: object + required: [actionable, request] + properties: + actionable: { type: boolean } + request: { type: string } +``` + +The declaration does not add a kernel primitive and does not yet infer a +TypeScript result type from arbitrary JSON Schema. Typed parsed values belong +to the imperative `f.llm` / `f.agent` surface once that surface has a real +consumer; the spec SDK does not publish an unchecked phantom type in advance. + The authoring surface deliberately narrows `steps: []`: `flows check` refuses it as `invalid_spec`, while the kernel accepts it. This is a chosen authoring-time narrowing, not a kernel guarantee. diff --git a/sdk/package.json b/sdk/package.json index 4532bf635..05763cb7a 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -23,8 +23,9 @@ "demo:hn": "npm run build && node dist/demo-hn-monitor.js", "prepare": "npm run build", "typecheck": "tsc --noEmit", + "typecheck:tests": "tsc -p tsconfig.tests.json", "test:prep": "( cd ../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../testdata/preflight ] || find ../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + )", - "test": "npm run test:prep && npm run build && vitest run", + "test": "npm run test:prep && npm run build && npm run typecheck:tests && vitest run", "test:watch": "vitest" }, "license": "UNLICENSED", diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index 86d2d5d90..281a2b041 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -30,6 +30,7 @@ import type { } from './spec.js'; import { SPEC_SCHEMA_VERSION } from './spec.js'; import { canonicalize, specHash } from './canonical.js'; +import { validateOutputDeclaration } from './output-schema.js'; import { validateSpec, type ValidationResult } from './validate.js'; export class CompileError extends Error { @@ -133,6 +134,8 @@ function compileStep(step: StepSpec): StepSpec { function typedOutputVerification(step: StepSpec): StepSpec['verification'] { if (step.type !== 'deterministic' && step.output !== undefined) { + const errors = validateOutputDeclaration(step, `step "${step.id}"`); + if (errors.length > 0) throw new CompileError(errors); return { type: 'json_schema', schema: step.output }; } return step.verification; @@ -399,14 +402,8 @@ function requireNoTimeout(step: StepSpec): void { function toKernelVerification(step: StepSpec): KernelVerificationSpec { const output = step.type === 'deterministic' ? undefined : step.output; if (output !== undefined) { - if (step.verification !== undefined) { - throw new CompileError([ - `step "${step.id}": output already declares json_schema verification; remove verification`, - ]); - } - if (!isObject(output)) { - throw new CompileError([`step "${step.id}".output: expected a JSON Schema object`]); - } + const errors = validateOutputDeclaration(step, `step "${step.id}"`); + if (errors.length > 0) throw new CompileError(errors); return { json_schema: output }; } const gate = step.verification; diff --git a/sdk/src/index.ts b/sdk/src/index.ts index 01d96e379..4c4a42fd9 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -26,7 +26,6 @@ export type { KernelVerificationSpec, LlmStepSpec, OutputContainsGate, - OutputFromSchema, PermissionsSpec, RecoveryMode, StreamSurface, diff --git a/sdk/src/output-schema.ts b/sdk/src/output-schema.ts index 8500f68d4..bf448b13a 100644 --- a/sdk/src/output-schema.ts +++ b/sdk/src/output-schema.ts @@ -1,18 +1,5 @@ -/** Type-only marker carried by authoring schemas and erased at runtime. */ -declare const outputSchemaType: unique symbol; - -/** - * JSON Schema with a TypeScript-only output type. The symbol property is - * optional and never exists in emitted specs, so ordinary JSON Schema objects - * remain the authoring value while TypeScript gates can recover `TOutput`. - */ -export interface JsonOutputSchema extends Record { - readonly [outputSchemaType]?: TOutput; -} - -/** Recover the parsed value type carried by a {@link JsonOutputSchema}. */ -export type OutputFromSchema = - TSchema extends JsonOutputSchema ? TOutput : never; +/** JSON Schema accepted by the `output` authoring declaration. */ +export type JsonOutputSchema = Record; /** Validate authoring sugar before it can be compiled or submitted. */ export function validateOutputDeclaration( diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index 2ce67fb86..ce0fd9942 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -10,7 +10,7 @@ // steps is valid. Nothing here requires an `agent` step. import type { JsonOutputSchema } from './output-schema.js'; -export type { JsonOutputSchema, OutputFromSchema } from './output-schema.js'; +export type { JsonOutputSchema } from './output-schema.js'; /** The three rungs of the ladder (RFC §1; AGENTS.md rule 7). */ export type StepType = 'deterministic' | 'llm' | 'agent'; @@ -122,17 +122,17 @@ export interface DeterministicStepSpec extends BaseStepSpec { * never calls a model: it dispatches to an attached SDK worker (§5) which * returns `{output, usage}`; the kernel then runs the verification gate. */ -export interface LlmStepSpec extends BaseStepSpec { +export interface LlmStepSpec extends BaseStepSpec { type: 'llm'; prompt: string; model?: string; /** Inert preflight declaration; overrides the flow/project CLI default. */ cli?: string; /** - * Typed-output authoring sugar. Compiles to the existing `json_schema` + * Structured-output authoring sugar. Compiles to the existing `json_schema` * verification primitive and is removed before the kernel boundary. */ - output?: JsonOutputSchema; + output?: JsonOutputSchema; } /** @@ -141,7 +141,7 @@ export interface LlmStepSpec extends BaseStepSpec { * every writeback is a journaled `effect.recorded` deduped by * `(step_id, idempotency_key, surface_path)`. */ -export interface AgentStepSpec extends BaseStepSpec { +export interface AgentStepSpec extends BaseStepSpec { type: 'agent'; instruction: string; /** Inert preflight declaration; overrides the flow/project CLI default. */ @@ -158,10 +158,10 @@ export interface AgentStepSpec extends BaseStepSpec { recoveryMode?: RecoveryMode; permissions?: PermissionsSpec; /** - * Typed-output authoring sugar. A successful CLI JSON object is the parsed + * Structured-output authoring sugar. A successful CLI JSON object is the parsed * value; the kernel persists it only after `json_schema` verification. */ - output?: JsonOutputSchema; + output?: JsonOutputSchema; } export type StepSpec = DeterministicStepSpec | LlmStepSpec | AgentStepSpec; diff --git a/sdk/tests/typed-output.test.ts b/sdk/tests/typed-output.test.ts index 3b11872b6..91c73fd0b 100644 --- a/sdk/tests/typed-output.test.ts +++ b/sdk/tests/typed-output.test.ts @@ -11,19 +11,14 @@ import { } from '../src/compile.js'; import type { AgentStepSpec, + DeterministicStepSpec, JsonOutputSchema, LlmStepSpec, - OutputFromSchema, } from '../src/spec.js'; -interface Extraction { - actionable: boolean; - request: string; -} - const TESTDATA = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'testdata'); -const extractionSchema: JsonOutputSchema = { +const extractionSchema: JsonOutputSchema = { type: 'object', additionalProperties: false, required: ['actionable', 'request'], @@ -34,24 +29,18 @@ const extractionSchema: JsonOutputSchema = { }; describe('typed llm and agent outputs', () => { - it('carries a schema output type for TypeScript gates', () => { - expectTypeOf>().toEqualTypeOf(); - - const llm: LlmStepSpec = { - id: 'extract', - type: 'llm', - prompt: 'Extract the request.', - output: extractionSchema, - }; - const agent: AgentStepSpec = { - id: 'research', - type: 'agent', - instruction: 'Research the request.', + it('typechecks output declarations on llm and agent steps only', () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + + const deterministic: DeterministicStepSpec = { + id: 'build', + type: 'deterministic', + command: 'true', + // @ts-expect-error structured output sugar is llm/agent-only. output: extractionSchema, }; - - expectTypeOf(llm.output).toEqualTypeOf | undefined>(); - expectTypeOf(agent.output).toEqualTypeOf | undefined>(); + void deterministic; }); it.each(['llm', 'agent'] as const)( @@ -167,4 +156,54 @@ steps: expect(result?.errors).toContain('spec.steps[0].output: expected a JSON Schema object'); }, ); + + it.each(['llm', 'agent'] as const)( + 'compiles a raw %s spec directly through the public kernel boundary', + (type) => { + const step: LlmStepSpec | AgentStepSpec = type === 'llm' + ? { + id: 'direct', + type: 'llm', + prompt: 'Return JSON.', + output: extractionSchema, + } + : { + id: 'direct', + type: 'agent', + instruction: 'Return JSON.', + output: extractionSchema, + }; + const kernel = toKernelSpec({ + version: '0.1.0', + steps: [step], + }); + + expect(kernel.steps[0]?.verification).toEqual({ json_schema: extractionSchema }); + }, + ); + + it('fails closed on a conflicting raw spec at the public kernel boundary', () => { + expect(() => toKernelSpec({ + version: '0.1.0', + steps: [{ + id: 'direct-conflict', + type: 'agent', + instruction: 'Return JSON.', + output: extractionSchema, + verification: { type: 'output_contains', value: 'done' }, + }], + })).toThrowError(/output already declares json_schema verification/); + }); + + it('fails closed on a malformed raw spec at the public kernel boundary', () => { + expect(() => toKernelSpec({ + version: '0.1.0', + steps: [{ + id: 'direct-malformed', + type: 'llm', + prompt: 'Return JSON.', + output: null, + } as unknown as Parameters[0]['steps'][number]], + })).toThrowError(/expected a JSON Schema object/); + }); }); diff --git a/sdk/tsconfig.tests.json b/sdk/tsconfig.tests.json new file mode 100644 index 000000000..2af505b54 --- /dev/null +++ b/sdk/tsconfig.tests.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": ".", + "types": ["node", "vitest"] + }, + "include": ["src/**/*.ts", "tests/typed-output.test.ts"], + "exclude": ["node_modules", "dist"] +} From 24c43d6b293b507795ae823f3f5bc3458ac8b3b5 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 17:48:43 +0200 Subject: [PATCH 03/33] ops(review): persist PR #133 swarm transcripts --- ops/reviews/20260902-1620-pr133-history.md | 444 ++++++++++++++++++ ops/reviews/20260902-1620-pr133-structure.md | 439 +++++++++++++++++ .../20260902-1625-pr133-maintainability.md | 412 ++++++++++++++++ 3 files changed, 1295 insertions(+) create mode 100644 ops/reviews/20260902-1620-pr133-history.md create mode 100644 ops/reviews/20260902-1620-pr133-structure.md create mode 100644 ops/reviews/20260902-1625-pr133-maintainability.md diff --git a/ops/reviews/20260902-1620-pr133-history.md b/ops/reviews/20260902-1620-pr133-history.md new file mode 100644 index 000000000..5a2be39d2 --- /dev/null +++ b/ops/reviews/20260902-1620-pr133-history.md @@ -0,0 +1,444 @@ +# PR #133 fresh exact-head history / RFC review + +- Review lens: RFC and history fit, backwards compatibility, issue #132 acceptance scope, regression risk, evidence reproducibility +- Repository: `AgentWorkforce/flows` +- Base: `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- Reviewed head: `5ab0fee366362de33ea1bd7d88a96daef1cad18a` +- Date: 2026-09-02 +- Verdict: PASS + +## Findings + +No blocking findings. + +The change fits RFC-0001 decision 13: `output` is an authoring-surface field which is erased before the journal boundary and lowered to the existing kernel `json_schema` verification primitive. No kernel files, step verbs, or journal vocabulary change. The implementation fails closed on malformed or ambiguous declarations, including direct callers of the public `toKernelSpec` boundary. + +The repair commit removes the speculative `OutputFromSchema` / generic result-type API introduced at the first commit, and the test-only TypeScript project makes the negative `@ts-expect-error` assertion part of ordinary `npm test` and exact-head artifact CI. This matches the repository rule against speculative abstraction and provides a real type gate. + +Backwards-compatibility scope is preserved: + +- Existing explicit `verification: { type: json_schema, ... }` authoring still compiles. +- A direct comparison below shows the legacy declaration and new `output` sugar produce byte-identical kernel specs. +- The previous-generation `workflows/` tree is unchanged at base versus head. This review therefore establishes non-interference and continued acceptance of the legacy declaration; it does not claim to have executed every previous-generation v1 workflow through an external v1 runner. + +PR and issue scope are honest. Issue #132 remains open. PR #133 says `Refs #132`, not `Closes #132`, and its body explicitly limits the change to slice 1. The issue's full done condition still requires imperative typed `f.llm` / `f.agent` parsed values, a parsed-value gate in the research flow, the published surface package, direct-run input, parallel dispatch, and other listed slices. This PR adds declarative YAML/object `StepSpec.output` schema sugar only and documents that limitation. + +## Constitution read + +I read both required files completely before assessing code: + +```text +$ wc -l AGENTS.md docs/RFC-0001-everything-is-a-relayflow.md + 75 AGENTS.md + 244 docs/RFC-0001-everything-is-a-relayflow.md + 319 total + +$ shasum -a 256 AGENTS.md docs/RFC-0001-everything-is-a-relayflow.md +3e542e190bd4375a105612cf119b18e39b7732be181126c5bcf75c37010b65b8 AGENTS.md +cf8c0f41b12dc37699aa348d10c1907dae8b7e8550a34ad976aac1349639c7b7 docs/RFC-0001-everything-is-a-relayflow.md +``` + +The complete files were read with `sed -n '1,160p' AGENTS.md` and the three +non-overlapping RFC commands `sed -n '1,90p'`, `sed -n '91,180p'`, and +`sed -n '181,280p'`. The hashes above pin the exact contents read without +substituting a prose summary for file output. + +Relevant RFC fit observed from that literal read: + +- RFC §1: `llm` is distinct from `agent`, has value output, and verification is the rail. +- Covenant 2: malformed/ambiguous declarations must fail closed before a run. +- Decision 13: surface vocabulary may be open, but it must compile to the closed kernel vocabulary. +- The open versioning question requires compatibility rather than implicit deprecation. + +## Exact checkout and change inventory + +```text +$ pwd +/Users/khaliqgant/AgentWorkforce/flows-132-typed-output-wt +$ git rev-parse HEAD +5ab0fee366362de33ea1bd7d88a96daef1cad18a +$ git rev-parse a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git branch --show-current +feat/v2-typed-outputs +$ git log --oneline --decorate --no-merges a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a +5ab0fee (HEAD -> feat/v2-typed-outputs, origin/feat/v2-typed-outputs) fix(sdk): enforce structured output contracts +81c49df feat(sdk): compile typed outputs to json_schema +``` + +```text +$ git diff --stat a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a + .github/workflows/cloud-runtime-artifact.yml | 15 +- + docs/SURFACE.md | 25 ++++ + sdk/package.json | 3 +- + sdk/src/compile.ts | 19 ++- + sdk/src/index.ts | 1 + + sdk/src/output-schema.ts | 22 +++ + sdk/src/spec.ts | 13 ++ + sdk/src/validate.ts | 7 +- + sdk/tests/live-kernel.test.ts | 36 ++++- + sdk/tests/typed-output.test.ts | 209 +++++++++++++++++++++++++++ + sdk/tsconfig.tests.json | 10 ++ + testdata/hn-monitor.flow.yaml | 33 ++--- + 12 files changed, 365 insertions(+), 28 deletions(-) + +$ git diff --name-status a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a +M .github/workflows/cloud-runtime-artifact.yml +M docs/SURFACE.md +M sdk/package.json +M sdk/src/compile.ts +M sdk/src/index.ts +A sdk/src/output-schema.ts +M sdk/src/spec.ts +M sdk/src/validate.ts +M sdk/tests/live-kernel.test.ts +A sdk/tests/typed-output.test.ts +A sdk/tsconfig.tests.json +M testdata/hn-monitor.flow.yaml +``` + +I inspected the complete patch, including both commits' net diff and then the +repair-only diff. The exact patch is reproducibly pinned by its literal hash, +line count, and per-file numstat: + +```text +$ git diff --find-renames --find-copies --no-ext-diff a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a | shasum -a 256 +cb334c3a6d3ae3dc9ecc007d06a599920e8111b8eec6d66038ee5881ddb685f2 - + +$ git diff --find-renames --find-copies --no-ext-diff a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a | wc -l + 588 + +$ git diff --numstat a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a +14 1 .github/workflows/cloud-runtime-artifact.yml +25 0 docs/SURFACE.md +2 1 sdk/package.json +18 1 sdk/src/compile.ts +1 0 sdk/src/index.ts +22 0 sdk/src/output-schema.ts +13 0 sdk/src/spec.ts +5 2 sdk/src/validate.ts +30 6 sdk/tests/live-kernel.test.ts +209 0 sdk/tests/typed-output.test.ts +10 0 sdk/tsconfig.tests.json +16 17 testdata/hn-monitor.flow.yaml + +$ git show --stat --oneline 81c49df32aebcbb1b21fc3f87eeae55cb2ed6ff1 +81c49df feat(sdk): compile typed outputs to json_schema + sdk/src/compile.ts | 22 +++++- + sdk/src/index.ts | 2 + + sdk/src/output-schema.ts | 35 +++++++++ + sdk/src/spec.ts | 17 ++++- + sdk/src/validate.ts | 7 +- + sdk/tests/live-kernel.test.ts | 36 +++++++-- + sdk/tests/typed-output.test.ts | 170 +++++++++++++++++++++++++++++++++++++++++ + testdata/hn-monitor.flow.yaml | 33 ++++---- + 8 files changed, 294 insertions(+), 28 deletions(-) + +$ git show --stat --oneline 5ab0fee366362de33ea1bd7d88a96daef1cad18a +5ab0fee fix(sdk): enforce structured output contracts + .github/workflows/cloud-runtime-artifact.yml | 15 ++++- + docs/SURFACE.md | 25 ++++++++ + sdk/package.json | 3 +- + sdk/src/compile.ts | 13 ++--- + sdk/src/index.ts | 1 - + sdk/src/output-schema.ts | 17 +----- + sdk/src/spec.ts | 14 ++--- + sdk/tests/typed-output.test.ts | 85 ++++++++++++++++++++-------- + sdk/tsconfig.tests.json | 10 ++++ + 9 files changed, 127 insertions(+), 56 deletions(-) +``` + +The repair-only patch was also inspected in full. It removes `OutputFromSchema` and generic `TOutput`, adds the dedicated test tsconfig and CI invocation, centralizes validation in `validateOutputDeclaration`, adds internal-normalization validation, and pins raw `toKernelSpec` success/conflict/malformed arms. + +## Issue #132 and PR claim check + +```text +$ gh issue view 132 --repo AgentWorkforce/flows --json number,title,state,url +{"number":132,"state":"OPEN","title":"v2 authoring ergonomics: close the gaps found in the research-flow / v1 / Smithers comparison","url":"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/AgentWorkforce/flows/issues/132"} + +$ gh pr view 133 --repo AgentWorkforce/flows --json number,title,state,baseRefName,headRefName,headRefOid,url +{"baseRefName":"main","headRefName":"feat/v2-typed-outputs","headRefOid":"5ab0fee366362de33ea1bd7d88a96daef1cad18a","number":133,"state":"OPEN","title":"feat(sdk): compile structured outputs to json_schema","url":"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/AgentWorkforce/flows/pull/133"} +``` + +The literal issue comments establish the migration and slice boundaries: + +```text +$ gh issue view 132 --repo AgentWorkforce/flows --comments +author: kjgbot +association: member +edited: false +status: none +-- +Implementation ownership started on 2026-09-02. + +- Active first slice: typed `llm` / `agent` outputs on branch `feat/v2-typed-outputs`, owned by Agent Relay worker `flows-132-typed-output`, based on merged main `a0d42ff`. This must compile to the existing `json_schema` primitive; no kernel vocabulary expansion. +- Next slices, each as a separately reviewed PR: publish `@relayflows/surface`; direct-run input; journaled `agents[].model` surface plus strict typo/model linting; gate-3 parallel dispatch; settle replayable data gates versus code gates; verb-specific unknown-field refusal. +- Migration policy: v1 remains the default and supported runtime while these v2 authoring gaps close. Stored version remains authoritative for resume/schedules. Deprecation is a later explicit decision, not implied by this issue. + +The Cloud runtime artifact foundation merged separately in #131. A real Cloud v2 proof can proceed on the deterministic surface while this issue gates broader author migration and the research-flow completion criteria. +``` + +The current PR body was read in full. Its relevant scope language is literal: + +```text +## Scope boundary + +This is issue #132 slice 1 only. It does not publish +`@relayflows/surface`, implement imperative typed return values, or close #132. +v1 explicit `verification:` remains supported. +``` + +That description matches the patch: no `research/` implementation or local shim is changed, no imperative `f.llm` / `f.agent` surface is published here, and issue #132 remains open. + +## Backwards compatibility evidence + +The previous-generation v1 workflow tree is byte-identical between base and head: + +```text +$ if git diff --quiet a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..5ab0fee366362de33ea1bd7d88a96daef1cad18a -- workflows; then echo V1_WORKFLOW_TREE_UNCHANGED=true; else echo V1_WORKFLOW_TREE_UNCHANGED=false; fi +V1_WORKFLOW_TREE_UNCHANGED=true + +$ for f in workflows/review-swarm.yaml workflows/watchdog.yaml workflows/bootstrap-gate1.yaml workflows/drive.yaml; do printf '%s ' "$f"; git show a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2:"$f" | shasum -a 256 | awk '{print $1}'; done +workflows/review-swarm.yaml 6343d8c9f1505a1cbd3660c507730aa371a3d72b6a4103fff815741f43377ca6 +workflows/watchdog.yaml e04a14e20dd1c3562e66901994dbf4533224581bc45262114c0c9c7aa72aa589 +workflows/bootstrap-gate1.yaml f7f74dd595f1a68c3d819fe7e01f2a4a07bb3d0bcbd314777f30c9d07a02035e +workflows/drive.yaml b691305d132c2b2bacb5b4e44a4e6efa7b7872e9f52d4201bda1f00e09ae58c4 + +$ for f in workflows/review-swarm.yaml workflows/watchdog.yaml workflows/bootstrap-gate1.yaml workflows/drive.yaml; do printf '%s ' "$f"; git show 5ab0fee366362de33ea1bd7d88a96daef1cad18a:"$f" | shasum -a 256 | awk '{print $1}'; done +workflows/review-swarm.yaml 6343d8c9f1505a1cbd3660c507730aa371a3d72b6a4103fff815741f43377ca6 +workflows/watchdog.yaml e04a14e20dd1c3562e66901994dbf4533224581bc45262114c0c9c7aa72aa589 +workflows/bootstrap-gate1.yaml f7f74dd595f1a68c3d819fe7e01f2a4a07bb3d0bcbd314777f30c9d07a02035e +workflows/drive.yaml b691305d132c2b2bacb5b4e44a4e6efa7b7872e9f52d4201bda1f00e09ae58c4 +``` + +Legacy explicit verification and the new sugar lower identically: + +```text +$ PATH=/Users/khaliqgant/.local/share/mise/installs/node/22.23.2/bin:/usr/bin:/bin node --input-type=module -e '