From 40668967e449f8d00eaa4d9175ab76d0f629a88c Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 15:01:29 +0200 Subject: [PATCH 01/16] docs: checkpoint WS-13 scope and verification baseline Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/evidence/ws13/README.md diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md new file mode 100644 index 00000000..1d61da1b --- /dev/null +++ b/docs/evidence/ws13/README.md @@ -0,0 +1,17 @@ +# WS-13 local development evidence + +Base: `origin/main` at `f0a3b3b` (2.0.8), isolated branch +`feat/flows-local-dev-ux`. Implementation and verification are in progress. + +Both PR #243 and #244 diffs were inspected before SDK edits. #243 changes +README.md, authored-flow-error.ts, authored-flow-executor.ts, +authored-flow-loader.ts, cli/direct-run.ts, authored-flow tests, live-kernel +tests, and the surface manifest. #244 changes local-drive scripts, their tests, +BACKLOG, drive-local.yaml, and evidence; its current diff has no SDK source edit. +The fetched base already contains the authored-agent implementation. + +The fetched tree contains four examples, including social-post-pipeline. +The existing gallery explicitly says three typecheck but do not run. No +all-green gallery or clean-machine timing claim has been established. + +Veto tools were not exposed in this session. No merge is authorized. From 4b1f0da44b5e76c76e526577f3d992c80070acb4 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 15:07:23 +0200 Subject: [PATCH 02/16] feat: add SDK scaffolding, local agent opt-in and progress rendering Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- packages/create-flow/bin/create-flow.js | 34 +++++++++ packages/create-flow/package.json | 16 +++++ packages/sdk/src/authored-flow-executor.ts | 16 ++++- packages/sdk/src/cli.ts | 32 +++++++-- packages/sdk/src/cli/direct-run.ts | 10 ++- packages/sdk/src/cli/run.ts | 3 + packages/sdk/src/create-flow.ts | 82 ++++++++++++++++++++++ packages/sdk/src/index.ts | 4 ++ packages/sdk/src/local-agent.ts | 29 ++++++++ packages/sdk/src/progress.ts | 42 +++++++++++ packages/sdk/tests/local-dev-ux.test.ts | 72 +++++++++++++++++++ 11 files changed, 331 insertions(+), 9 deletions(-) create mode 100755 packages/create-flow/bin/create-flow.js create mode 100644 packages/create-flow/package.json create mode 100644 packages/sdk/src/create-flow.ts create mode 100644 packages/sdk/src/local-agent.ts create mode 100644 packages/sdk/src/progress.ts create mode 100644 packages/sdk/tests/local-dev-ux.test.ts diff --git a/packages/create-flow/bin/create-flow.js b/packages/create-flow/bin/create-flow.js new file mode 100755 index 00000000..849387c5 --- /dev/null +++ b/packages/create-flow/bin/create-flow.js @@ -0,0 +1,34 @@ +#!/usr/bin/env node +import { createFlow } from '@relayflows/sdk'; + +const usage = 'Usage: create-flow [--name ] [--template agent|deterministic] [--cli ] [--no-install]'; +const args = process.argv.slice(2); +try { + if (args.length === 1 && ['--help', '-h'].includes(args[0])) { + console.log(usage); + } else { + let target; + const options = {}; + const seen = new Set(); + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--no-install') { + if (seen.has(arg)) throw new Error(usage); + seen.add(arg); + options.install = false; + } else if (arg === '--name' || arg === '--cli' || arg === '--template') { + if (seen.has(arg) || !args[i + 1] || args[i + 1].startsWith('-')) throw new Error(usage); + seen.add(arg); + options[arg.slice(2)] = args[++i]; + } else if (arg.startsWith('-') || target !== undefined) { + throw new Error(usage); + } else target = arg; + } + if (!target) throw new Error(usage); + const result = await createFlow(target, options); + console.log(`Created ${result.directory}\nNext: cd ${JSON.stringify(result.directory)}${result.installed ? '' : ' && npm install'} && npm start`); + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/packages/create-flow/package.json b/packages/create-flow/package.json new file mode 100644 index 00000000..6aadb036 --- /dev/null +++ b/packages/create-flow/package.json @@ -0,0 +1,16 @@ +{ + "name": "create-flow", + "version": "2.0.8", + "description": "Create a Relayflows project using the SDK scaffolder.", + "type": "module", + "license": "Apache-2.0", + "bin": { "create-flow": "./bin/create-flow.js" }, + "files": ["bin"], + "engines": { "node": ">=22.18.0" }, + "dependencies": { "@relayflows/sdk": "2.0.8" }, + "repository": { + "type": "git", + "url": "git+https://github.com/AgentWorkforce/flows.git", + "directory": "packages/create-flow" + } +} diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 4b7dd53f..b761be63 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -11,6 +11,7 @@ import { } from '@relayflows/surface'; import type { FlowHandle } from '@relayflows/surface/runtime'; import { join } from 'node:path'; +import { observeStep, type ProgressEvent } from './progress.js'; import { compileSpec, toKernelSpec } from './compile.js'; import { getAuthoredFlowDefinition } from './authored-flow.js'; import type { GetFlowDefinition } from './authored-flow-loader.js'; @@ -117,6 +118,8 @@ export interface ExecuteAuthoredFlowOptions { /** Passed straight through to classifyOutcome (cli/run.ts) for f.agent's wait. */ readonly signal?: RunLifecycleOptions['signal']; readonly onWait?: RunLifecycleOptions['onWait']; + readonly onProgress?: (event: ProgressEvent) => void; + readonly localAgentStream?: string; } export async function executeAuthoredFlow( @@ -126,6 +129,8 @@ export async function executeAuthoredFlow( options: ExecuteAuthoredFlowOptions = {}, ): Promise { const getDefinition = options.getDefinition ?? getAuthoredFlowDefinition; + const localAgentStream = options.localAgentStream; + const onProgress = options.onProgress; const flowPath = options.flowPath ?? join(process.cwd(), 'flow.ts'); // Named separately from `options` because `lowerAgent` below has its own, // differently-typed `options: AgentOptions` parameter that shadows this one. @@ -172,6 +177,10 @@ export async function executeAuthoredFlow( id: string, options: AgentOptions, ): Promise => { + if (options.workspace !== undefined && localAgentStream !== undefined) { + throw new AuthoredFlowExecutionError('unsupported_workspace_permission', + 'The local agent worker accepts stream-only steps. Remove workspace or attach a worker that holds its revision pins.'); + } if (options.workspace !== undefined && WORKSPACE_PERMISSION_ANNOTATION.test(options.workspace)) { throw new AuthoredFlowExecutionError( 'unsupported_workspace_permission', @@ -191,6 +200,9 @@ export async function executeAuthoredFlow( id, type: 'agent', instruction: options.task, + ...(localAgentStream === undefined ? {} : { + surfaces: { streams: [{ stream: localAgentStream }] }, + }), ...(options.workspace === undefined ? {} : { surfaces: { workspace: [{ surface: options.workspace }] }, }), @@ -260,7 +272,7 @@ export async function executeAuthoredFlow( id, 'run', () => assertOperationAllowed('run', definition.name, requestedCompletion), - () => lowerDeterministic(id, command), + () => observeStep(id, 'deterministic', () => lowerDeterministic(id, command), options.onProgress), lifecycle, )); }, @@ -282,7 +294,7 @@ export async function executeAuthoredFlow( id, 'agent', () => assertOperationAllowed('agent', definition.name, requestedCompletion), - () => lowerAgent(id, options), + () => observeStep(id, 'agent', () => lowerAgent(id, options), onProgress), lifecycle, )); }, diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index c816374b..35feebf1 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { renderProgress, type ProgressEvent } from './progress.js'; import { realpathSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; import { @@ -31,7 +32,7 @@ type CliExitCode = 0 | 1 | 2 | 3; type ParsedArgs = | { command: 'cloud-run'; value: string; json: boolean; wait: boolean } | { command: 'check'; json: boolean; value: string } - | { command: 'run'; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; value: string } + | { command: 'run'; localAgent: boolean; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; value: string } | { command: 'resume'; dataDir: string; json: boolean; spawn: boolean; value: string } | { command: 'hn-monitor'; sub: 'start'; dataDir: string; specPath: string; pollIntervalMs: number | undefined } | { command: 'tick'; sub: 'start'; dataDir: string; specPath: string; scheduleId: string; @@ -44,7 +45,7 @@ const USAGE = [ 'flows check [--json] ', 'flows run [--json] [--no-spawn] [--data-dir ] ', 'flows run --cloud [--json] [--wait] ', - 'flows run [--json] [--no-spawn] [--data-dir ] --input ', + 'flows run [--json] [--no-spawn] [--data-dir ] [--local-agent] --input ', 'flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] ', 'flows resume [--json] [--no-spawn] [--data-dir ] ', 'flows hn-monitor start [--data-dir ] [--poll-interval-ms ] ', @@ -135,8 +136,19 @@ export async function runCli( // single `connect()` seam immediately before journal-client.ts is used -- // not here. Hoisting it above the dispatch would start a daemon as a side // effect of an invocation that is about to be refused for bad input. + const startedSteps = new Map(); + const showProgress = (event: ProgressEvent): void => { + if (event.type === 'step.started') startedSteps.set(event.stepId, performance.now()); + if (!parsed.json) for (const line of renderProgress([event])) io.stderr(line); + }; const lifecycle = { - onWait: (progress: RunProgress) => emitWait(progress, io), + localAgent: parsed.command === 'run' && parsed.localAgent, + onProgress: showProgress, + onWait: (progress: RunProgress) => { + emitWait(progress, io); + showProgress({ type: 'step.running', stepId: progress.stepId, stepType: progress.stepType, + elapsedMs: performance.now() - (startedSteps.get(progress.stepId) ?? performance.now()) }); + }, daemon: { spawn: parsed.spawn && spawnAllowedByEnv() }, }; const execution = parsed.command === 'run' @@ -167,6 +179,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let json = false; let cloud = false; let wait = false; + let localAgent = false; let dataDir = DEFAULT_DATA_DIR; let sawDataDir = false; let spawn = true; @@ -181,6 +194,11 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { else wait = true; continue; } + if (argument === '--local-agent') { + if (command !== 'run' || localAgent) return undefined; + localAgent = true; + continue; + } if (argument === '--json') { if (json) return undefined; json = true; @@ -215,16 +233,20 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (positionals.length !== 1) return undefined; if (cloud) { - if (sawInput || sawDataDir || !spawn) return undefined; + // `--cloud` submits the spec to Cloud, so every flag that only describes a + // local run -- an inline input, a data dir, a suppressed daemon, a local + // agent -- describes nothing there and is refused rather than ignored. + if (sawInput || sawDataDir || !spawn || localAgent) return undefined; return { command: 'cloud-run', value: positionals[0]!, json, wait }; } if (wait) return undefined; + if (localAgent && !isAuthoredFlowPath(positionals[0]!)) return undefined; if (command === 'run' && input !== undefined && !isAuthoredFlowPath(positionals[0]!)) return undefined; return command === 'check' ? { command, json, value: positionals[0]! } : command === 'run' - ? { command, dataDir, input, json, spawn, value: positionals[0]! } + ? { command, localAgent, dataDir, input, json, spawn, value: positionals[0]! } : { command, dataDir, json, spawn, value: positionals[0]! }; } diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index a8bfc1bb..8348875b 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -1,3 +1,4 @@ +import { attachLocalAgent } from '../local-agent.js'; import { AuthoredFlowExecutionError, executeAuthoredFlow, @@ -43,11 +44,15 @@ export async function runDirectFlow( const connected = await connect(client, 'run', dataDir, base, options); if (connected !== undefined) return connected; + let localAgent: Awaited> | undefined; try { const { handle, getDefinition } = await loadAuthoredFlow(path); + if (options.localAgent) localAgent = await attachLocalAgent(client); const result = await executeAuthoredFlow(handle, client, input, { getDefinition, flowPath: path, + onProgress: options.onProgress, + localAgentStream: localAgent?.stream, ...(options.signal !== undefined ? { signal: options.signal } : {}), ...(options.onWait !== undefined ? { onWait: options.onWait } : {}), }); @@ -69,7 +74,8 @@ export async function runDirectFlow( completedSteps: result.journalSteps.length, }, }; - } catch (error) { + } catch (caught) { + const error = localAgent?.failure ?? caught; // `agent_cli_unresolved` and `unsupported_workspace_permission` are // preflight-shaped refusals, not protocol failures — `flows check` // returns exit 2 for the equivalent declarative-spec failures, and this @@ -116,6 +122,6 @@ export async function runDirectFlow( const runId = error instanceof AuthoredFlowExecutionError ? error.runId : undefined; return protocolFailure('run', base, socketPath, error, runId); } finally { - client.close(); + try { await localAgent?.close(); } finally { client.close(); } } } diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index f899c9f9..e76bb871 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -1,4 +1,5 @@ import { join, resolve } from 'node:path'; +import type { ProgressEvent } from '../progress.js'; import { toKernelSpec } from '../compile.js'; import { ensureDaemon, type EnsureDaemonOptions } from '../daemon-lifecycle.js'; import { daemonRefusal } from './daemon-refusal.js'; @@ -59,6 +60,8 @@ export interface RunProgress { } export interface RunLifecycleOptions { + onProgress?: (event: ProgressEvent) => void; + localAgent?: boolean; signal?: AbortSignal; onWait?: (progress: RunProgress) => void; /** diff --git a/packages/sdk/src/create-flow.ts b/packages/sdk/src/create-flow.ts new file mode 100644 index 00000000..85a5adaf --- /dev/null +++ b/packages/sdk/src/create-flow.ts @@ -0,0 +1,82 @@ +import { spawn } from 'node:child_process'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; + +export interface CreateFlowOptions { + name?: string; + template?: 'agent' | 'deterministic'; + /** CLI used by future f.agent steps. It must already be authenticated. */ + cli?: string; + /** Install project dependencies (default true). */ + install?: boolean; +} + +export interface CreatedFlow { + directory: string; + flowPath: string; + configPath: string; + installed: boolean; +} + +/** Scaffold a new project. Existing directories are never overwritten. */ +export async function createFlow(target: string, opts: CreateFlowOptions = {}): Promise { + if (!target.trim()) throw new Error('Choose a new directory, for example: create-flow my-flow'); + const directory = resolve(target); + const name = opts.name ?? basename(directory); + if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(name)) { + throw new Error('Flow name must be 1–64 lowercase letters, numbers or hyphens, starting with a letter or number.'); + } + const template = opts.template ?? 'agent'; + if (template !== 'agent' && template !== 'deterministic') throw new Error('Template must be agent or deterministic.'); + const cli = opts.cli ?? 'claude'; + if (!cli.trim() || /[\x00-\x1f\x7f]/.test(cli)) throw new Error('CLI must be a nonempty command or path.'); + const { version } = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as { version: string }; + await mkdir(dirname(directory), { recursive: true }); + try { + await mkdir(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error(`Directory already exists: ${directory}. Choose a new directory; no files were changed.`); + } + throw error; + } + const flowFile = `${name}.flow.ts`; + const agentStep = template === 'agent' + ? ` const answer = await f.agent('greeter', { task: 'Reply with one short hello sentence. Do not use tools or modify files.' });\n console.log(answer.summary);\n` + : ''; + const runFlags = template === 'agent' ? ' --local-agent' : ''; + const files: Record = { + [flowFile]: `import { flow } from '@relayflows/surface';\n\nexport default flow(${JSON.stringify(name)}, async (f) => {\n const greeting = await f.run('echo "Hello from Relayflows"');\n console.log(greeting.trim());\n${agentStep} f.done('success');\n});\n`, + 'flows.json': JSON.stringify({ cli }, null, 2) + '\n', + 'package.json': JSON.stringify({ + name, private: true, type: 'module', + scripts: { start: `flows run ${flowFile}${runFlags} --input '{}'` }, + engines: { node: '>=22.18.0' }, + dependencies: { relayflows: version, '@relayflows/surface': version }, + }, null, 2) + '\n', + '.gitignore': 'node_modules/\n.relayflowd/\n', + 'README.md': `# ${name}\n\nRun \`npm start\` (or \`npx flows run ${flowFile}${runFlags} --input '{}'\`).\n\n${template === 'agent' ? `Requires ${JSON.stringify(cli)} installed and authenticated. The local agent worker runs\non this machine with the CLI's existing access; it provides no workspace isolation.\n` : 'This starter needs no model credentials.\n'}\n\`flows.json\` selects the CLI for agent steps. Steps execute through the local\njournal; authored TypeScript bodies are not yet durably resumable as a whole.\n`, + }; + for (const [file, contents] of Object.entries(files)) { + await writeFile(join(directory, file), contents, { flag: 'wx' }); + } + if (opts.install !== false) { + try { + await installDependencies(directory); + } catch (error) { + throw new Error(`Created ${directory}, but dependency installation failed. Run npm install there to retry.`, { cause: error }); + } + } + return { directory, flowPath: join(directory, flowFile), configPath: join(directory, 'flows.json'), installed: opts.install !== false }; +} + +function installDependencies(cwd: string): Promise { + return new Promise((resolveInstall, reject) => { + const child = spawn('npm', ['install', '--no-audit', '--no-fund'], { cwd, stdio: 'inherit' }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (code === 0) resolveInstall(); + else reject(new Error(`npm install exited ${signal ?? code}`)); + }); + }); +} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index e87982c7..08ddceaf 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -205,3 +205,7 @@ export { type TickPayload, type TickSchedule, } from './tick-source.js'; + +export { createFlow, type CreateFlowOptions, type CreatedFlow } from './create-flow.js'; + +export { renderProgress, type ProgressEvent } from './progress.js'; diff --git a/packages/sdk/src/local-agent.ts b/packages/sdk/src/local-agent.ts new file mode 100644 index 00000000..fb68ef2e --- /dev/null +++ b/packages/sdk/src/local-agent.ts @@ -0,0 +1,29 @@ +import { randomUUID } from 'node:crypto'; +import type { JournalClient } from './journal-client.js'; +import { AgentWorker } from './worker.js'; + +/** A local worker for stream-only steps; no workspace recovery is claimed. */ +export async function attachLocalAgent(client: JournalClient): Promise<{ + stream: string; + readonly failure: unknown; + close(): Promise; +}> { + // A fresh, unconsumed stream has offset zero. The executor declares exactly + // this stream on its agent steps. No worktree revision is invented. + const stream = `local-agent-${randomUUID()}`; + const worker = new AgentWorker(client, { + workerId: stream, + capacity: 1, + pins: { workspace: [], streams: [{ stream, read_offset: 0 }] }, + }); + let failure: unknown; + worker.on('error', error => { failure = error; client.close(); }); + await worker.attach(); + return { + stream, + get failure() { return failure; }, + async close() { + await worker.close(); + }, + }; +} diff --git a/packages/sdk/src/progress.ts b/packages/sdk/src/progress.ts new file mode 100644 index 00000000..27ce47c2 --- /dev/null +++ b/packages/sdk/src/progress.ts @@ -0,0 +1,42 @@ +import type { CompletionReason } from './protocol.js'; +import type { StepType } from './spec.js'; + +export interface ProgressEvent { + type: 'step.started' | 'step.running' | 'step.completed' | 'step.failed'; + stepId: string; + stepType: StepType; + elapsedMs: number; + completionReason?: CompletionReason; +} + +/** Pure terminal rendering: caller owns the event source, clock, and output. */ +export function renderProgress(events: Iterable): string[] { + return Array.from(events, event => { + const icon = { 'step.started': '○', 'step.running': '↻', 'step.completed': '✓', 'step.failed': '✗' }[event.type]; + const state = event.type.slice('step.'.length); + const agent = event.stepType === 'agent' ? ` [agent: ${state === 'started' ? 'preparing' : state}]` : ''; + const reason = event.completionReason ? ` completionReason: ${event.completionReason}` : ''; + // Agent-authored names cannot inject terminal control sequences. + const name = event.stepId.replace(/[\x00-\x1f\x7f-\x9f]/g, '?'); + return `${icon} ${name} (${event.stepType})${agent} ${(Math.max(0, event.elapsedMs) / 1000).toFixed(2)}s${reason}`; + }); +} + +/** Observe the existing executor; success is emitted only after its journal read. */ +export async function observeStep( + stepId: string, + stepType: StepType, + execute: () => Promise, + emit?: (event: ProgressEvent) => void, +): Promise { + const started = performance.now(); + emit?.({ type: 'step.started', stepId, stepType, elapsedMs: 0 }); + try { + const result = await execute(); + emit?.({ type: 'step.completed', stepId, stepType, elapsedMs: performance.now() - started, completionReason: 'success' }); + return result; + } catch (error) { + emit?.({ type: 'step.failed', stepId, stepType, elapsedMs: performance.now() - started }); + throw error; + } +} diff --git a/packages/sdk/tests/local-dev-ux.test.ts b/packages/sdk/tests/local-dev-ux.test.ts new file mode 100644 index 00000000..1c35beff --- /dev/null +++ b/packages/sdk/tests/local-dev-ux.test.ts @@ -0,0 +1,72 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createFlow } from '../src/create-flow.js'; +import { observeStep, renderProgress, type ProgressEvent } from '../src/progress.js'; + +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function target() { + const root = await mkdtemp(join(tmpdir(), 'flows-scaffold-')); + roots.push(root); + return join(root, 'hello'); +} + +describe('SDK project scaffolder', () => { + it('emits an agent starter, local-worker command and the chosen CLI', async () => { + const directory = await target(); + const result = await createFlow(directory, { install: false, cli: 'codex' }); + expect(result.installed).toBe(false); + expect(JSON.parse(await readFile(result.configPath, 'utf8'))).toEqual({ cli: 'codex' }); + expect(await readFile(result.flowPath, 'utf8')).toContain("await f.agent('greeter'"); + const manifest = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); + expect(manifest.scripts.start).toBe("flows run hello.flow.ts --local-agent --input '{}'"); + expect(manifest.dependencies.relayflows).toBe(manifest.dependencies['@relayflows/surface']); + }); + it('offers a credential-free deterministic starter', async () => { + const result = await createFlow(await target(), { install: false, template: 'deterministic' }); + expect(await readFile(result.flowPath, 'utf8')).not.toContain('f.agent'); + expect(await readFile(join(result.directory, 'package.json'), 'utf8')).not.toContain('--local-agent'); + }); + it('refuses an existing project without changing any files', async () => { + const directory = await target(); + await createFlow(directory, { install: false }); + const sentinel = join(directory, 'flows.json'); + await writeFile(sentinel, 'keep me'); + await expect(createFlow(directory, { install: false })).rejects.toThrow('already exists'); + expect(await readFile(sentinel, 'utf8')).toBe('keep me'); + }); + it('validates names and template before writing', async () => { + const directory = await target(); + await expect(createFlow(directory, { name: '../escape', install: false })).rejects.toThrow('Flow name'); + await expect(createFlow(directory, { template: 'unknown' as 'agent', install: false })).rejects.toThrow('Template'); + expect(await readdir(roots.at(-1)!)).toEqual([]); + }); +}); + +describe('progress is an observation of execution', () => { + it('does not report completion before the journal operation resolves', async () => { + const events: ProgressEvent[] = []; + let finish!: () => void; + const journalWrite = new Promise(resolve => { finish = resolve; }); + const observed = observeStep('greet', 'agent', () => journalWrite, event => events.push(event)); + expect(events.map(event => event.type)).toEqual(['step.started']); + finish(); + await observed; + expect(events.map(event => event.type)).toEqual(['step.started', 'step.completed']); + expect(renderProgress(events).join('\n')).toContain('[agent: completed]'); + }); + it('propagates a journal failure without inventing a successful completion', async () => { + const events: ProgressEvent[] = []; + const failure = new Error('journal_write_failed'); + await expect(observeStep('write', 'deterministic', async () => { throw failure; }, event => events.push(event))) + .rejects.toBe(failure); + expect(events.map(event => event.type)).toEqual(['step.started', 'step.failed']); + expect(events.at(-1)?.completionReason).toBeUndefined(); + }); + it('renders time and strips terminal controls from step names', () => { + expect(renderProgress([{ type: 'step.running', stepId: '\x1b[2Jagent', stepType: 'agent', elapsedMs: 1234 }])) + .toEqual(['↻ ?[2Jagent (agent) [agent: running] 1.23s']); + }); +}); From 5824c9251f1fdc8a86f31da559b1b21c024b9d45 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 15:17:58 +0200 Subject: [PATCH 03/16] fix: keep npm flows on the SDK and record gallery refusals Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/agent-probe-timeout.cast | 10 +++ docs/evidence/ws13/agent-probe-timeout.txt | 14 ++++ docs/evidence/ws13/agent-run.cast | 3 + docs/evidence/ws13/agent-run.txt | 9 +++ .../ws13/gallery-dependency-upgrade-bot.txt | 6 ++ .../ws13/gallery-pr-review-pipeline.txt | 6 ++ docs/evidence/ws13/gallery-research.txt | 5 ++ .../ws13/gallery-social-post-pipeline.txt | 6 ++ docs/evidence/ws13/record.py | 61 ++++++++++++++++ docs/evidence/ws13/stage-registry.mjs | 35 ++++++++++ examples/research/package.json | 2 +- examples/research/tsconfig.json | 2 +- packages/relayflows/bin/flows.js | 23 +------ packages/runtime-darwin-arm64/README.md | 25 +++---- packages/runtime-darwin-arm64/package.json | 5 +- packages/runtime-linux-x64/README.md | 21 +++--- packages/runtime-linux-x64/package.json | 5 +- packages/sdk/tests/local-agent-live.test.ts | 69 +++++++++++++++++++ 18 files changed, 250 insertions(+), 57 deletions(-) create mode 100644 docs/evidence/ws13/agent-probe-timeout.cast create mode 100644 docs/evidence/ws13/agent-probe-timeout.txt create mode 100644 docs/evidence/ws13/agent-run.cast create mode 100644 docs/evidence/ws13/agent-run.txt create mode 100644 docs/evidence/ws13/gallery-dependency-upgrade-bot.txt create mode 100644 docs/evidence/ws13/gallery-pr-review-pipeline.txt create mode 100644 docs/evidence/ws13/gallery-research.txt create mode 100644 docs/evidence/ws13/gallery-social-post-pipeline.txt create mode 100644 docs/evidence/ws13/record.py create mode 100644 docs/evidence/ws13/stage-registry.mjs create mode 100644 packages/sdk/tests/local-agent-live.test.ts diff --git a/docs/evidence/ws13/agent-probe-timeout.cast b/docs/evidence/ws13/agent-probe-timeout.cast new file mode 100644 index 00000000..731db67f --- /dev/null +++ b/docs/evidence/ws13/agent-probe-timeout.cast @@ -0,0 +1,10 @@ +{"version": 2, "width": 120, "height": 30, "timestamp": 1788873089, "title": "Relayflows local development", "command": "npx --no-install flows run hello.flow.ts --local-agent --input '{}'", "env": {"TERM": "xterm-256color"}} +[5.045106, "o", "npm notice run npx\r\n"] +[5.045358, "o", "npm notice run 'flows' run hello.flow.ts --local-agent --input {}\r\n"] +[7.742345, "o", "\u25cb run-1 (deterministic) 0.00s\r\n"] +[7.953248, "o", "\u2713 run-1 (deterministic) 0.21s completionReason: success\r\n"] +[7.955388, "o", "Hello from Relayflows\r\n"] +[7.955632, "o", "\u25cb agent-2 (agent) [agent: preparing] 0.00s\r\n"] +[18.136722, "o", "\u2717 agent-2 (agent) [agent: failed] 10.18s\r\n"] +[18.16956, "o", "REFUSED [invalid_spec] agent_cli_unresolved: Could not verify CLI \"claude\" for step \"agent-2\": the probe timed out after 10000ms.\r\n"] +[18.173621, "o", "\r\nEXIT_CODE=2\r\nELAPSED_SECONDS=18.174\r\nTIMED_OUT=False\r\n"] diff --git a/docs/evidence/ws13/agent-probe-timeout.txt b/docs/evidence/ws13/agent-probe-timeout.txt new file mode 100644 index 00000000..32d31d34 --- /dev/null +++ b/docs/evidence/ws13/agent-probe-timeout.txt @@ -0,0 +1,14 @@ +$ cd /tmp/ws13-consumer/hello +$ npx --no-install flows run hello.flow.ts --local-agent --input '{}' +npm notice run npx +npm notice run 'flows' run hello.flow.ts --local-agent --input {} +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.21s completionReason: success +Hello from Relayflows +○ agent-2 (agent) [agent: preparing] 0.00s +✗ agent-2 (agent) [agent: failed] 10.18s +REFUSED [invalid_spec] agent_cli_unresolved: Could not verify CLI "claude" for step "agent-2": the probe timed out after 10000ms. + +EXIT_CODE=2 +ELAPSED_SECONDS=18.174 +TIMED_OUT=False diff --git a/docs/evidence/ws13/agent-run.cast b/docs/evidence/ws13/agent-run.cast new file mode 100644 index 00000000..5a9c0d73 --- /dev/null +++ b/docs/evidence/ws13/agent-run.cast @@ -0,0 +1,3 @@ +{"version": 2, "width": 120, "height": 30, "timestamp": 1788873468, "title": "Relayflows local development", "command": "npx --no-install flows run hello.flow.ts --local-agent --input '{}'", "env": {"TERM": "xterm-256color"}} +[0.052747, "o", "dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib\r\n Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node\r\n Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file)\r\n"] +[0.052983, "o", "\r\nEXIT_CODE=-6\r\nELAPSED_SECONDS=0.053\r\nTIMED_OUT=False\r\n"] diff --git a/docs/evidence/ws13/agent-run.txt b/docs/evidence/ws13/agent-run.txt new file mode 100644 index 00000000..20428ae6 --- /dev/null +++ b/docs/evidence/ws13/agent-run.txt @@ -0,0 +1,9 @@ +$ cd /tmp/ws13-consumer/hello +$ npx --no-install flows run hello.flow.ts --local-agent --input '{}' +dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib + Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node + Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file) + +EXIT_CODE=-6 +ELAPSED_SECONDS=0.053 +TIMED_OUT=False diff --git a/docs/evidence/ws13/gallery-dependency-upgrade-bot.txt b/docs/evidence/ws13/gallery-dependency-upgrade-bot.txt new file mode 100644 index 00000000..cfe5c14a --- /dev/null +++ b/docs/evidence/ws13/gallery-dependency-upgrade-bot.txt @@ -0,0 +1,6 @@ +$ cd /tmp/ws13-gallery +$ node /tmp/ws13-consumer/hello/node_modules/relayflows/bin/flows.js run examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts --local-agent --input '{}' --data-dir /tmp/ws13-gallery-depen +REFUSED [invalid_spec] unsupported_header: flow "dependency-upgrade-bot" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=6.596 diff --git a/docs/evidence/ws13/gallery-pr-review-pipeline.txt b/docs/evidence/ws13/gallery-pr-review-pipeline.txt new file mode 100644 index 00000000..82d23d7d --- /dev/null +++ b/docs/evidence/ws13/gallery-pr-review-pipeline.txt @@ -0,0 +1,6 @@ +$ cd /tmp/ws13-gallery +$ node /tmp/ws13-consumer/hello/node_modules/relayflows/bin/flows.js run examples/pr-review-pipeline/pr-review-pipeline.flow.ts --local-agent --input '{"diffRange": "origin/main...HEAD"}' --data-dir /tmp/ws13-gallery-pr-re +REFUSED [invalid_spec] unsupported_header: flow "pr-review-pipeline" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=4.990 diff --git a/docs/evidence/ws13/gallery-research.txt b/docs/evidence/ws13/gallery-research.txt new file mode 100644 index 00000000..08a8b222 --- /dev/null +++ b/docs/evidence/ws13/gallery-research.txt @@ -0,0 +1,5 @@ +$ cd /tmp/ws13-gallery +$ node --experimental-strip-types examples/research/shims/run.ts --slug ws13-verification --question 'Compare durable step journals with deterministic replay. Keep every report under 200 words.' --timeout-minutes 1 +None +EXIT_CODE=124 +ELAPSED_SECONDS=150.067 diff --git a/docs/evidence/ws13/gallery-social-post-pipeline.txt b/docs/evidence/ws13/gallery-social-post-pipeline.txt new file mode 100644 index 00000000..48daddc3 --- /dev/null +++ b/docs/evidence/ws13/gallery-social-post-pipeline.txt @@ -0,0 +1,6 @@ +$ cd /tmp/ws13-gallery +$ node /tmp/ws13-consumer/hello/node_modules/relayflows/bin/flows.js run examples/social-post-pipeline/social-post-pipeline.flow.ts --local-agent --input '{"brand": "Relayflows", "topic": "durable steps", "approver": "local-reviewer"}' --data-dir /tmp/ws13-gallery-socia +REFUSED [invalid_spec] unsupported_header: flow "social-post-pipeline" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=6.698 diff --git a/docs/evidence/ws13/record.py b/docs/evidence/ws13/record.py new file mode 100644 index 00000000..71851759 --- /dev/null +++ b/docs/evidence/ws13/record.py @@ -0,0 +1,61 @@ +"""Capture a real terminal session as asciicast v2 plus a text transcript. +Usage: python3 record.py OUTPUT_PREFIX CWD COMMAND [ARG ...] +""" +import codecs +import json +import os +import pty +import select +import shlex +import signal +import sys +import time +from pathlib import Path + +prefix, cwd, *argv = sys.argv[1:] +started = time.monotonic() +header = {'version': 2, 'width': 120, 'height': 30, 'timestamp': int(time.time()), + 'title': 'Relayflows local development', 'command': shlex.join(argv), + 'env': {'TERM': 'xterm-256color'}} +with open(prefix + '.cast', 'w') as cast, open(prefix + '.txt', 'w') as transcript: + cast.write(json.dumps(header) + '\n') + transcript.write(f'$ cd {shlex.quote(cwd)}\n$ {shlex.join(argv)}\n') + transcript.flush() + pid, fd = pty.fork() + if pid == 0: + os.chdir(cwd) + os.execvpe(argv[0], argv, os.environ) + decoder = codecs.getincrementaldecoder('utf-8')('replace') + timed_out = False + while True: + if time.monotonic() - started > 180: + timed_out = True + os.killpg(pid, signal.SIGTERM) + ready, _, _ = select.select([fd], [], [], 1) + if ready: + try: + data = os.read(fd, 65536) + except OSError: + break + if not data: + break + text = decoder.decode(data) + cast.write(json.dumps([round(time.monotonic() - started, 6), 'o', text]) + '\n') + cast.flush() + transcript.write(text) + transcript.flush() + if timed_out: + time.sleep(0.2) + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + break + _, status = os.waitpid(pid, 0) + code = os.waitstatus_to_exitcode(status) + elapsed = time.monotonic() - started + ending = f'\nEXIT_CODE={code}\nELAPSED_SECONDS={elapsed:.3f}\nTIMED_OUT={timed_out}\n' + transcript.write(ending) + cast.write(json.dumps([round(elapsed, 6), 'o', ending.replace('\n', '\r\n')]) + '\n') + print(ending) + sys.exit(code if code >= 0 else 128 - code) diff --git a/docs/evidence/ws13/stage-registry.mjs b/docs/evidence/ws13/stage-registry.mjs new file mode 100644 index 00000000..0fb634a5 --- /dev/null +++ b/docs/evidence/ws13/stage-registry.mjs @@ -0,0 +1,35 @@ +// Serve packed candidate packages locally; redirect other dependencies to npm. +// Usage: node stage-registry.mjs /absolute/artifact-directory [port] +import { createServer } from 'node:http'; +import { createHash } from 'node:crypto'; +import { readFileSync, readdirSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { execFileSync } from 'node:child_process'; + +const directory = resolve(process.argv[2]); +const packages = new Map(); +const tarballs = new Map(); +for (const file of readdirSync(directory).filter(file => file.endsWith('.tgz'))) { + const path = join(directory, file); + const manifest = JSON.parse(execFileSync('tar', ['-xOf', path, 'package/package.json'], { encoding: 'utf8' })); + const data = readFileSync(path); + packages.set(manifest.name, { manifest, file, integrity: `sha512-${createHash('sha512').update(data).digest('base64')}` }); + tarballs.set(`/tarballs/${file}`, data); +} +createServer((req, res) => { + const url = new URL(req.url, `http://${req.headers.host}`); + const tarball = tarballs.get(url.pathname); + if (tarball) { res.end(tarball); return; } + const name = decodeURIComponent(url.pathname.slice(1)); + const candidate = packages.get(name); + if (!candidate) { + res.writeHead(302, { location: `https://registry.npmjs.org${req.url}` }); + res.end(); + return; + } + const { manifest, file, integrity } = candidate; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ name, 'dist-tags': { latest: manifest.version }, versions: { + [manifest.version]: { ...manifest, dist: { tarball: `${url.origin}/tarballs/${file}`, integrity } }, + } })); +}).listen(Number(process.argv[3] ?? 48731), '0.0.0.0', () => console.log('Candidate registry ready')); diff --git a/examples/research/package.json b/examples/research/package.json index dc8f0846..8ca98b3f 100644 --- a/examples/research/package.json +++ b/examples/research/package.json @@ -5,7 +5,7 @@ "description": "The research example: a v2 relayflow with shims. `npm test` runs its node:test suites; `npm run typecheck` uses the sdk's TypeScript. Marks the directory as ESM so node --experimental-strip-types runs it without reparsing.", "scripts": { "test": "node --experimental-strip-types --test tests/*.test.ts", - "typecheck": "cd ../../sdk && ./node_modules/.bin/tsc -p ../examples/research/tsconfig.json", + "typecheck": "../../packages/sdk/node_modules/.bin/tsc -p tsconfig.json", "check": "npm test && npm run typecheck" } } diff --git a/examples/research/tsconfig.json b/examples/research/tsconfig.json index cd3fa677..836608f5 100644 --- a/examples/research/tsconfig.json +++ b/examples/research/tsconfig.json @@ -1,5 +1,5 @@ { - "//": "OPT-IN typecheck for the research flow. Not part of `npm test`. Run: cd sdk && ./node_modules/.bin/tsc -p ../examples/research/tsconfig.json (not npx: without node_modules, npx fetches an unrelated tsc and passes vacuously)", + "//": "OPT-IN typecheck for the research flow. Not part of `npm test`. Run: npm --prefix examples/research run typecheck (not npx: without node_modules, npx fetches an unrelated tsc and passes vacuously)", "compilerOptions": { "target": "ES2022", "module": "ESNext", diff --git a/packages/relayflows/bin/flows.js b/packages/relayflows/bin/flows.js index 585c4bf1..e8d375ad 100755 --- a/packages/relayflows/bin/flows.js +++ b/packages/relayflows/bin/flows.js @@ -1,23 +1,4 @@ #!/usr/bin/env node -import { spawnSync } from 'node:child_process'; -import { existsSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { runCli } from '@relayflows/sdk'; -// Not `import '@relayflows/sdk/dist/cli.js'`: that subpath isn't in the -// SDK's package "exports", so specifier resolution would refuse it. Locating -// the installed dependency's real CLI file directly sidesteps that — this -// package's only job is finding it and forwarding argv/stdio. -const packageRoot = join(dirname(fileURLToPath(import.meta.url)), '..'); -const sdkCli = join(packageRoot, 'node_modules', '@relayflows', 'sdk', 'dist', 'cli.js'); - -if (!existsSync(sdkCli)) { - process.stderr.write( - `relayflows: could not find @relayflows/sdk at ${sdkCli}\n` + - 'Reinstall with `npm install -g relayflows`.\n', - ); - process.exit(1); -} - -const result = spawnSync(process.execPath, [sdkCli, ...process.argv.slice(2)], { stdio: 'inherit' }); -process.exit(result.status ?? 1); +process.exitCode = await runCli(process.argv.slice(2)); diff --git a/packages/runtime-darwin-arm64/README.md b/packages/runtime-darwin-arm64/README.md index 60dafcd9..65e5f409 100644 --- a/packages/runtime-darwin-arm64/README.md +++ b/packages/runtime-darwin-arm64/README.md @@ -1,20 +1,13 @@ # @relayflows/runtime-darwin-arm64 -Prebuilt Relayflow v2 runtime for `darwin-arm64` (Apple Silicon): +Prebuilt `relayflowd` kernel daemon for `darwin-arm64`. Install `relayflows` to get +the `flows` command backed by `@relayflows/sdk`; this package exports only +`relayflowd`, so it cannot replace the SDK CLI during npm's bin linking. -- `bin/relayflowd` — the kernel daemon (Rust, `cargo build --release -p relayflowd`, target `aarch64-apple-darwin`) -- `bin/flows` — the standalone CLI (`bun build --target=bun-darwin-arm64`) +The daemon is built from the release commit and published with npm provenance. +The package declares `os`/`cpu` and is an optional dependency of `relayflows`. +Unsupported platforms need a locally built daemon via `RELAYFLOWD_BIN`. -Built natively on a `macos-14` GitHub Actions runner from the same commit and -tag as every other release package, and published with npm provenance. - -This package is platform-specific by design. It declares `os`/`cpu`, so npm -refuses to install it anywhere else rather than yielding a binary that cannot -run. `@relayflows/sdk`'s `relayflowd-path.ts` resolves it as an optional -dependency of the `relayflows` CLI package — installing `relayflows` on an -Apple Silicon Mac pulls this in automatically; every other platform's npm -skips it. - -Intel Macs (`darwin-x64`) are not covered by this package and fall through to -`relayflowd-path.ts`'s later resolution steps (a source checkout or `PATH`) -until a `@relayflows/runtime-darwin-x64` package exists. +The tarball also retains the legacy `bin/flows` executable required by the +existing release gate. It is not registered as an npm command. Removing it +from the archive requires a separate change by the release-gate owner. diff --git a/packages/runtime-darwin-arm64/package.json b/packages/runtime-darwin-arm64/package.json index 31cd18e8..58d27081 100644 --- a/packages/runtime-darwin-arm64/package.json +++ b/packages/runtime-darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@relayflows/runtime-darwin-arm64", "version": "2.0.8", - "description": "Relayflow v2 runtime for darwin-arm64: the relayflowd kernel and the flows CLI, as prebuilt binaries", + "description": "Relayflow kernel for darwin-arm64: the prebuilt relayflowd daemon", "license": "Apache-2.0", "repository": { "type": "git", @@ -18,8 +18,7 @@ "bin/" ], "bin": { - "relayflowd": "./bin/relayflowd", - "flows": "./bin/flows" + "relayflowd": "./bin/relayflowd" }, "engines": { "node": ">=20" diff --git a/packages/runtime-linux-x64/README.md b/packages/runtime-linux-x64/README.md index ed4ba056..69845dc5 100644 --- a/packages/runtime-linux-x64/README.md +++ b/packages/runtime-linux-x64/README.md @@ -1,16 +1,13 @@ # @relayflows/runtime-linux-x64 -Prebuilt Relayflow v2 runtime for `linux-x64`: +Prebuilt `relayflowd` kernel daemon for `linux-x64`. Install `relayflows` to get +the `flows` command backed by `@relayflows/sdk`; this package exports only +`relayflowd`, so it cannot replace the SDK CLI during npm's bin linking. -- `bin/relayflowd` — the kernel daemon (Rust, `cargo build --release -p relayflowd`) -- `bin/flows` — the standalone CLI (`bun build --target=bun-linux-x64`) +The daemon is built from the release commit and published with npm provenance. +The package declares `os`/`cpu` and is an optional dependency of `relayflows`. +Unsupported platforms need a locally built daemon via `RELAYFLOWD_BIN`. -Both are the exact binaries the repository's cloud runtime artifact ships, built -from the same commit and published with npm provenance. - -This package is platform-specific by design. It declares `os`/`cpu`, so npm -refuses to install it anywhere else rather than yielding a binary that cannot -run. A cross-platform wrapper that selects among per-platform packages is the -natural next step once a second platform is built; today CI produces linux-x64 -only, and publishing a wrapper that can resolve exactly one platform would -promise a portability that does not exist. +The tarball also retains the legacy `bin/flows` executable required by the +existing release gate. It is not registered as an npm command. Removing it +from the archive requires a separate change by the release-gate owner. diff --git a/packages/runtime-linux-x64/package.json b/packages/runtime-linux-x64/package.json index 6cc28c43..710530c6 100644 --- a/packages/runtime-linux-x64/package.json +++ b/packages/runtime-linux-x64/package.json @@ -1,7 +1,7 @@ { "name": "@relayflows/runtime-linux-x64", "version": "2.0.8", - "description": "Relayflow v2 runtime for linux-x64: the relayflowd kernel and the flows CLI, as prebuilt binaries", + "description": "Relayflow kernel for linux-x64: the prebuilt relayflowd daemon", "license": "Apache-2.0", "repository": { "type": "git", @@ -18,8 +18,7 @@ "bin/" ], "bin": { - "relayflowd": "./bin/relayflowd", - "flows": "./bin/flows" + "relayflowd": "./bin/relayflowd" }, "engines": { "node": ">=20" diff --git a/packages/sdk/tests/local-agent-live.test.ts b/packages/sdk/tests/local-agent-live.test.ts new file mode 100644 index 00000000..7491a8bd --- /dev/null +++ b/packages/sdk/tests/local-agent-live.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +const roots: string[] = []; +const sdk = resolve('.'); +const cli = join(sdk, 'dist/cli.js'); +const wrapperHelper = resolve('../../testdata/preflight/wrapper-session.mjs'); +afterEach(() => { + for (const root of roots.splice(0)) { + const connection = join(root, 'data/connection.json'); + if (existsSync(connection)) { + const { pid } = JSON.parse(readFileSync(connection, 'utf8')); + if (typeof pid === 'number') { + try { process.kill(pid, 'SIGTERM'); } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } + } + } + rmSync(root, { recursive: true, force: true }); + } +}); + +function fixture(exitCode = 0, workspace?: string) { + const root = mkdtempSync(join(tmpdir(), 'flows-local-agent-')); + roots.push(root); + symlinkSync(join(sdk, 'node_modules'), join(root, 'node_modules')); + const marker = join(root, 'invoked'); + const wrapper = join(root, 'agent.mjs'); + writeFileSync(wrapper, `#!/usr/bin/env node\nimport { receiveWrapperRequest } from ${JSON.stringify(wrapperHelper)};\nimport { writeFileSync } from 'node:fs';\nif (process.argv[2] === 'auth') process.exit(0);\nconst request = await receiveWrapperRequest();\nif (request) { writeFileSync(${JSON.stringify(marker)}, request.instruction); console.log('local-agent-ok'); process.exit(${exitCode}); }\n`); + chmodSync(wrapper, 0o755); + writeFileSync(join(root, 'flows.json'), JSON.stringify({ cli: wrapper })); + writeFileSync(join(root, 'package.json'), '{"type":"module"}'); + writeFileSync(join(root, 'hello.flow.ts'), `import { flow } from '@relayflows/surface';\nexport default flow('hello', async f => { await f.agent('greeter', ${JSON.stringify({ task: 'hello', ...(workspace ? { workspace } : {}) })}); f.done('success'); });\n`); + return { root, marker, invoke: (...flags: string[]) => spawnSync(process.execPath, + [cli, 'run', 'hello.flow.ts', '--input', '{}', '--local-agent', '--data-dir', join(root, 'data'), ...flags], + { cwd: root, encoding: 'utf8', timeout: 30000 }) }; +} + +describe('built CLI local agent against a real daemon', () => { + it('dispatches through the wrapper and keeps --json stdout report-shaped', () => { + const f = fixture(); + const result = f.invoke('--json'); + expect(result.status, result.stderr + result.stdout).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, status: 'completed', completionReason: 'success' }); + expect(readFileSync(f.marker, 'utf8')).toBe('hello'); + expect(result.stderr).not.toContain('✓'); + }); + it('renders actual agent completion in text output', () => { + const result = fixture().invoke(); + expect(result.status, result.stderr + result.stdout).toBe(0); + expect(result.stderr).toContain('✓ agent-1 (agent) [agent: completed]'); + }); + it('returns a failed run when the agent process fails', () => { + const result = fixture(7).invoke(); + expect(result.status, result.stderr + result.stdout).toBe(1); + expect(result.stderr).toContain('✗ agent-1'); + expect(result.stderr).not.toContain('[agent: completed]'); + }); + it('refuses a workspace it cannot pin before invoking the agent', () => { + const f = fixture(0, 'repo'); + const result = f.invoke(); + expect(result.status, result.stderr + result.stdout).toBe(2); + expect(result.stderr).toContain('stream-only'); + expect(existsSync(f.marker)).toBe(false); + }); +}); From 761f46b7d6b97391d8bdca30819b017cc074cd4a Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 15:26:31 +0200 Subject: [PATCH 04/16] docs: show measured local UX and unsupported gallery features Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- README.md | 91 +++++++++---------- .../ws13/agent-host-node-failure.cast | 3 + .../evidence/ws13/agent-host-node-failure.txt | 9 ++ docs/evidence/ws13/agent-run.cast | 13 ++- docs/evidence/ws13/agent-run.txt | 17 +++- examples/README.md | 42 ++++++--- examples/dependency-upgrade-bot/README.md | 8 +- examples/pr-review-pipeline/README.md | 8 +- examples/social-post-pipeline/README.md | 8 +- packages/create-flow/bin/create-flow.js | 5 +- packages/relayflows/bin/flows.js | 2 +- packages/sdk/package.json | 12 +++ 12 files changed, 140 insertions(+), 78 deletions(-) create mode 100644 docs/evidence/ws13/agent-host-node-failure.cast create mode 100644 docs/evidence/ws13/agent-host-node-failure.txt diff --git a/README.md b/README.md index 1b31f12d..0a8846ca 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,65 @@ # relay(Flows) -**Step functions for coding agent workflows** - -Agent Relay is building infrastructure for autonomous agents. A relayflow is a readable step function -that runs on the relay and produces a verifiable artifact or result that can be paused -for human input and resumed from any step wherever needed. It is an agentic pipeline -that can load in any model + harness along with deterministic gates to generate -reliable results. +**Turn a coding-agent task into steps you can inspect and verify.** +A flow combines shell commands and coding agents with a journal that records +what each step did and why it completed. Start with a small local flow; add +verification as the task grows. ```ts -import { flow } from "@relayflows/surface"; - -export default flow("fix-failing-tests", async (f) => { - const result = await f - .run("npm test 2>&1; echo EXIT:$?") - .gate((out) => !out.includes("EXIT:0"), "tests are already green, nothing to fix"); - - const fix = await f - .agent("fixer", { - task: `The test suite is failing. Diagnose and fix it:\n${result}`, - workspace: "src/**: readwrite", - }) - .gate((r) => r.artifacts.length > 0, "the agent must actually change something"); - - f.done("success"); +import { flow } from '@relayflows/surface'; + +export default flow('hello', async (f) => { + const greeting = await f.run('echo "Hello from Relayflows"'); + console.log(greeting.trim()); + const answer = await f.agent('greeter', { + task: 'Reply with one short hello sentence. Do not use tools or modify files.', + }); + console.log(answer.summary); + f.done('success'); }); ``` -# Use Cases +The new scaffolder in this branch creates the flow, `flows.json`, and an npm +project, then installs its dependencies: -Flows can be run locally or in production on our hosted cloud. We're built entire -applications using flows that are stacked to run in a sequence with review gates that -can run autonomously over days and weeks. Every agent session is observable and replayable. +```sh +npx create-flow@latest my-flow +cd my-flow +npm start +``` -- Cloud pipeline to use agents to generate a social media post. The pipeline coordinates agents who do research, verify the post, check for authenticity, generate graphics, and gate on a human approval — [`examples/social-post-pipeline/`](examples/social-post-pipeline/) -- Pull request review pipeline with different agents looking at the pull request from different angles (security, optimization etc) and agents communicate when needed to reach consensus — [`examples/pr-review-pipeline/`](examples/pr-review-pipeline/) -- Dependency upgrade bot: deterministic check flags a dependency out of date which fires an agent who does the upgrade in a sandbox. This upgrade is gated on another agent verifying the entire application with computer use in another sandbox. If completely verified a pull request is opened up — [`examples/dependency-upgrade-bot/`](examples/dependency-upgrade-bot/) +**Release status:** `create-flow` is not published yet. The commands above are +the intended released entry point; use the [candidate artifact procedure](docs/evidence/ws13/README.md) +to try this branch. A clean-machine first-agent run under 60 seconds has not +been established. +The agent starter requires Node 22.18+ and an installed, authenticated Claude +CLI. Use `--cli codex` to select Codex, or `--template deterministic` for a +starter that needs no model credentials. The generated command is +`flows run my-flow.flow.ts --local-agent --input '{}'`. -# Get Started +`--local-agent` attaches the existing SDK agent worker to the local daemon. +It accepts stream-only agent steps and runs the chosen CLI with its existing +local access. Workspace revision pins and isolation require a worker that +provides those capabilities. Authored TypeScript bodies are not yet durably +resumable as a whole; each lowered step has its own journal run. -Install the CLI, then the authoring package in your own project: -```sh -npm install -g relayflows -mkdir my-flow && cd my-flow && npm install @relayflows/surface -``` +For SDK callers, the CLI is optional: -Write a flow — save this as `hello.flow.ts`: ```ts -import { flow } from "@relayflows/surface"; +import { createFlow, renderProgress } from '@relayflows/sdk'; -export default flow("hello", async (f) => { - await f.run('echo "hello from a relayflow"'); - f.done("success"); -}); +await createFlow('./my-flow', { cli: 'claude' }); +// renderProgress(events) returns terminal lines; callers own event delivery. ``` -Run it: -```sh -flows run hello.flow.ts --input '{}' -``` - -That's the whole loop — `flows run` spins up the local kernel itself on first use, no separate daemon step. You should see a completed run report. - -`f.run` and `f.agent` both actually dispatch today. `f.agent` runs a real coding-agent CLI the same way a declarative `type: agent` step does — it needs a `flows.json` in your project declaring which CLI to use (see `docs/SURFACE.md` §5 and `packages/sdk/src/cli/check.ts`'s `readProjectConfig`); without one, `flows run` refuses with a clear `agent_cli_unresolved` diagnostic rather than hanging. `f.llm`, `f.human`, `f.dispatch`, and `f.cloud` are still `docs/SURFACE.md`'s design surface, not yet runnable — see [`examples/`](examples/) for what the full shape looks like, and each example's own README for exactly what runs today versus what's still landing. +See the [example gallery and individual run results](examples/README.md). +The larger examples currently refuse or time out; their intended budgets, +artifact gates, workspace restrictions, and human approvals are preserved. Give your agent a skill to write a flow: + ```sh npx skills add https://github.com/agentworkforce/skills --skill writing-relayflows ``` diff --git a/docs/evidence/ws13/agent-host-node-failure.cast b/docs/evidence/ws13/agent-host-node-failure.cast new file mode 100644 index 00000000..5a9c0d73 --- /dev/null +++ b/docs/evidence/ws13/agent-host-node-failure.cast @@ -0,0 +1,3 @@ +{"version": 2, "width": 120, "height": 30, "timestamp": 1788873468, "title": "Relayflows local development", "command": "npx --no-install flows run hello.flow.ts --local-agent --input '{}'", "env": {"TERM": "xterm-256color"}} +[0.052747, "o", "dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib\r\n Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node\r\n Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file)\r\n"] +[0.052983, "o", "\r\nEXIT_CODE=-6\r\nELAPSED_SECONDS=0.053\r\nTIMED_OUT=False\r\n"] diff --git a/docs/evidence/ws13/agent-host-node-failure.txt b/docs/evidence/ws13/agent-host-node-failure.txt new file mode 100644 index 00000000..20428ae6 --- /dev/null +++ b/docs/evidence/ws13/agent-host-node-failure.txt @@ -0,0 +1,9 @@ +$ cd /tmp/ws13-consumer/hello +$ npx --no-install flows run hello.flow.ts --local-agent --input '{}' +dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib + Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node + Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file) + +EXIT_CODE=-6 +ELAPSED_SECONDS=0.053 +TIMED_OUT=False diff --git a/docs/evidence/ws13/agent-run.cast b/docs/evidence/ws13/agent-run.cast index 5a9c0d73..e423aaf8 100644 --- a/docs/evidence/ws13/agent-run.cast +++ b/docs/evidence/ws13/agent-run.cast @@ -1,3 +1,10 @@ -{"version": 2, "width": 120, "height": 30, "timestamp": 1788873468, "title": "Relayflows local development", "command": "npx --no-install flows run hello.flow.ts --local-agent --input '{}'", "env": {"TERM": "xterm-256color"}} -[0.052747, "o", "dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib\r\n Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node\r\n Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file)\r\n"] -[0.052983, "o", "\r\nEXIT_CODE=-6\r\nELAPSED_SECONDS=0.053\r\nTIMED_OUT=False\r\n"] +{"version": 2, "width": 120, "height": 30, "timestamp": 1788873624, "title": "Relayflows local development", "command": "npx --no-install flows run hello.flow.ts --local-agent --input '{}'", "env": {"TERM": "xterm-256color"}} +[101.678796, "o", "\u25cb run-1 (deterministic) 0.00s\r\n"] +[102.701875, "o", "\u2713 run-1 (deterministic) 1.02s completionReason: success\r\n"] +[102.705587, "o", "Hello from Relayflows\r\n"] +[102.706328, "o", "\u25cb agent-2 (agent) [agent: preparing] 0.00s\r\n"] +[115.218552, "o", "WAITING [worker_lease] Run \"01M20JY3TM2STS5QE7HTM5QKKW\" step \"agent-2\" (agent) is running under a worker lease until 1788873770124.\r\n\u21bb agent-2 (agent) [agent: running] 12.51s\r\n"] +[131.671616, "o", "\u2713 agent-2 (agent) [agent: completed] 28.95s completionReason: success\r\n"] +[131.702003, "o", "Hello! How can I help you today?\r\n\r\n"] +[131.974035, "o", "RUN 01M20JYM4T9FNGNFX4NQCK34JK completed (3 steps) completionReason: success\r\n"] +[132.636834, "o", "\r\nEXIT_CODE=0\r\nELAPSED_SECONDS=132.637\r\nTIMED_OUT=False\r\n"] diff --git a/docs/evidence/ws13/agent-run.txt b/docs/evidence/ws13/agent-run.txt index 20428ae6..13916f34 100644 --- a/docs/evidence/ws13/agent-run.txt +++ b/docs/evidence/ws13/agent-run.txt @@ -1,9 +1,16 @@ $ cd /tmp/ws13-consumer/hello $ npx --no-install flows run hello.flow.ts --local-agent --input '{}' -dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib - Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node - Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file) +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 1.02s completionReason: success +Hello from Relayflows +○ agent-2 (agent) [agent: preparing] 0.00s +WAITING [worker_lease] Run "01M20JY3TM2STS5QE7HTM5QKKW" step "agent-2" (agent) is running under a worker lease until 1788873770124. +↻ agent-2 (agent) [agent: running] 12.51s +✓ agent-2 (agent) [agent: completed] 28.95s completionReason: success +Hello! How can I help you today? + +RUN 01M20JYM4T9FNGNFX4NQCK34JK completed (3 steps) completionReason: success -EXIT_CODE=-6 -ELAPSED_SECONDS=0.053 +EXIT_CODE=0 +ELAPSED_SECONDS=132.637 TIMED_OUT=False diff --git a/examples/README.md b/examples/README.md index 271d986c..d1c06c37 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,18 +1,30 @@ -# examples — relayflows authored on the v2 surface +# Example gallery -Each example is a self-contained unit: the flow in the v2 dialect -(`docs/SURFACE.md`), pure helpers, `shims/` that execute it on today's runtime -with `REPLACE-WHEN: gate-N` headers, tests, and a README. Examples are the -consumers that tell gate-1 SDK work what `@relayflows/surface` must export. +These four examples describe larger flows. **None has a green end-to-end +result in the WS-13 verification run.** The table records time until refusal +or the verification timeout, not time to successful completion. -| Example | What it shows | -|---|---| -| [`research/`](research/) | Fan-out to three model lanes (Claude, Codex, Grok), two subagents each, one synthesis; postfix gates on the workspace; dynamic input. First real run: `research/runs/2026-09-02-agent-memory/`. | -| [`social-post-pipeline/`](social-post-pipeline/) | Research → draft → adversarial fact-check → graphic, gated on a human approval before anything publishes. Written directly against the real `@relayflows/surface` package. | -| [`pr-review-pipeline/`](pr-review-pipeline/) | Security/correctness/performance reviewer agents fan out in parallel, gated on writing their findings, then a consensus agent reconciles disagreement between them. | -| [`dependency-upgrade-bot/`](dependency-upgrade-bot/) | A deterministic check flags an outdated dependency; one agent upgrades it in a sandbox, a second, independent agent verifies the whole app with computer use in a separate sandbox before a PR opens. | +| Example | What it demonstrates | Observed result | Time | +|---|---|---|---:| +| [dependency-upgrade-bot](dependency-upgrade-bot/) | Upgrade → independent verification → PR | Refused: unsupported `budget` header, exit 2 | [6.596s](../docs/evidence/ws13/gallery-dependency-upgrade-bot.txt) | +| [pr-review-pipeline](pr-review-pipeline/) | Three review lenses → consensus | Refused: unsupported `budget` header, exit 2 | [4.990s](../docs/evidence/ws13/gallery-pr-review-pipeline.txt) | +| [social-post-pipeline](social-post-pipeline/) | Research → draft → fact-check → graphic → human approval | Refused: unsupported `budget` header, exit 2 | [6.698s](../docs/evidence/ws13/gallery-social-post-pipeline.txt) | +| [research](research/) | Claude/Codex/Grok fan-out → synthesis via existing shims | Verification timed out with no output captured | [150.067s](../docs/evidence/ws13/gallery-research.txt) | -The last three are written directly against the real `@relayflows/surface` -package (`npm --prefix packages/surface run typecheck:examples`) rather than -against local shims — they typecheck today but don't run yet; each one's -README says exactly what's real and what gate work it's waiting on. +The first three were invoked individually with the packed candidate CLI, +`--local-agent`, explicit inputs, and separate local daemon directories. +Research was invoked through its documented shim with a one-minute per-step +bound and a 150-second outer verification bound; the latter does not establish +whether preflight or execution was responsible. Exact commands and captured +output are linked in the table. These are runs on an existing development +host, not a clean machine. + +Removing the unsupported headers or weakening the examples' artifact gates +would change what they promise. Further runtime work is required before these +can be advertised as runnable. Social-post-pipeline additionally depends on +`f.human`; workspace permission annotations and postfix gates also remain +unsupported by the authored executor. + +The research example's `npm run typecheck` command now uses the actual +`packages/sdk` compiler path. Its shim tests and typecheck are separate from +an end-to-end run. diff --git a/examples/dependency-upgrade-bot/README.md b/examples/dependency-upgrade-bot/README.md index 60e51ae0..d7eb4f96 100644 --- a/examples/dependency-upgrade-bot/README.md +++ b/examples/dependency-upgrade-bot/README.md @@ -34,7 +34,13 @@ npm outdated (deterministic, gated) → upgrade (agent, sandbox A) → verify wi checks that a real PR URL came back — not just that the `gh` command exited 0. -## Status: typechecks, does not run yet +## Status: refused before execution + +WS-13 invoked this example with the packed CLI and `--local-agent`. It +refused the unsupported `budget` header before entering the body. See the +[gallery](../README.md) for the exact command, output, and elapsed time. +The remaining limitations below describe what still needs to land after that +first refusal is resolved. ```sh cd packages/surface && npm run typecheck:examples diff --git a/examples/pr-review-pipeline/README.md b/examples/pr-review-pipeline/README.md index 4860d485..02b7a098 100644 --- a/examples/pr-review-pipeline/README.md +++ b/examples/pr-review-pipeline/README.md @@ -32,7 +32,13 @@ mechanism — My Senior Dev's multi-agent PR review — in two layers: Unlike the other two examples in this directory, this one never calls `f.human` — nothing here needs it to make sense as a demonstration. -## Status: typechecks, does not run yet +## Status: refused before execution + +WS-13 invoked this example with the packed CLI and `--local-agent`. It +refused the unsupported `budget` header before entering the body. See the +[gallery](../README.md) for the exact command, output, and elapsed time. +The remaining limitations below describe what still needs to land after that +first refusal is resolved. ```sh cd packages/surface && npm run typecheck:examples diff --git a/examples/social-post-pipeline/README.md b/examples/social-post-pipeline/README.md index 04ff0fa9..0e36a1cb 100644 --- a/examples/social-post-pipeline/README.md +++ b/examples/social-post-pipeline/README.md @@ -32,7 +32,13 @@ part isn't the agents — it's what each `.gate()` checks: - The human gate (`f.human`) is the last word. Everything upstream can pass every gate and the flow still won't publish without a yes. -## Status: typechecks, does not run yet +## Status: refused before execution + +WS-13 invoked this example with the packed CLI and `--local-agent`. It +refused the unsupported `budget` header before entering the body. See the +[gallery](../README.md) for the exact command, output, and elapsed time. +The remaining limitations below describe what still needs to land after that +first refusal is resolved. ```sh cd packages/surface && npm run typecheck:examples diff --git a/packages/create-flow/bin/create-flow.js b/packages/create-flow/bin/create-flow.js index 849387c5..6d32ff2c 100755 --- a/packages/create-flow/bin/create-flow.js +++ b/packages/create-flow/bin/create-flow.js @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { createFlow } from '@relayflows/sdk'; +import { createFlow } from '@relayflows/sdk/create-flow'; +const shellQuote = value => "'" + value.replaceAll("'", "'\\''") + "'"; const usage = 'Usage: create-flow [--name ] [--template agent|deterministic] [--cli ] [--no-install]'; const args = process.argv.slice(2); try { @@ -26,7 +27,7 @@ try { } if (!target) throw new Error(usage); const result = await createFlow(target, options); - console.log(`Created ${result.directory}\nNext: cd ${JSON.stringify(result.directory)}${result.installed ? '' : ' && npm install'} && npm start`); + console.log(`Created ${result.directory}\nNext: cd ${shellQuote(result.directory)}${result.installed ? '' : ' && npm install'} && npm start`); } } catch (error) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/packages/relayflows/bin/flows.js b/packages/relayflows/bin/flows.js index e8d375ad..366e34e3 100755 --- a/packages/relayflows/bin/flows.js +++ b/packages/relayflows/bin/flows.js @@ -1,4 +1,4 @@ #!/usr/bin/env node -import { runCli } from '@relayflows/sdk'; +import { runCli } from '@relayflows/sdk/cli'; process.exitCode = await runCli(process.argv.slice(2)); diff --git a/packages/sdk/package.json b/packages/sdk/package.json index efc41391..2d596671 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -12,6 +12,18 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" + }, + "./create-flow": { + "types": "./dist/create-flow.d.ts", + "import": "./dist/create-flow.js" + }, + "./progress": { + "types": "./dist/progress.d.ts", + "import": "./dist/progress.js" + }, + "./cli": { + "types": "./dist/cli.d.ts", + "import": "./dist/cli.js" } }, "files": [ From e122d8d946a35d99157fea430cffe9169d5b0a9b Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 16:31:53 +0200 Subject: [PATCH 05/16] test: capture cold-clone timings and final local worker evidence Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- README.md | 10 +- docs/evidence/ws13/README.md | 127 ++++- .../evidence/ws13/agent-host-node-failure.txt | 6 +- docs/evidence/ws13/agent-probe-timeout.txt | 16 +- docs/evidence/ws13/agent-run.txt | 20 +- docs/evidence/ws13/artifact-check.txt | 14 + docs/evidence/ws13/artifacts.json | 22 + docs/evidence/ws13/cold-clone-direct.txt | 49 ++ docs/evidence/ws13/cold-clone-npx.txt | 29 + docs/evidence/ws13/cold-clone.sh | 13 + docs/evidence/ws13/cold-container.txt | 16 + docs/evidence/ws13/cold-start.sh | 11 + docs/evidence/ws13/cold-trixie.txt | 39 ++ docs/evidence/ws13/container-tests.txt | 522 ++++++++++++++++++ docs/evidence/ws13/focused-tests.txt | 18 + docs/evidence/ws13/launcher-before-fix.txt | 4 + .../ws13/local-agent-tests-30s-ceiling.txt | 40 ++ .../evidence/ws13/local-agent-tests-final.txt | 15 + .../ws13/local-agent-tests-first-attempt.txt | 79 +++ docs/evidence/ws13/local-agent-tests.txt | 15 + docs/evidence/ws13/record.py | 2 +- docs/evidence/ws13/research-typecheck.txt | 3 + docs/evidence/ws13/test-types-final.txt | 3 + docs/evidence/ws13/typechecks.txt | 3 + packages/sdk/tests/local-agent-live.test.ts | 15 +- 25 files changed, 1053 insertions(+), 38 deletions(-) create mode 100644 docs/evidence/ws13/artifact-check.txt create mode 100644 docs/evidence/ws13/artifacts.json create mode 100644 docs/evidence/ws13/cold-clone-direct.txt create mode 100644 docs/evidence/ws13/cold-clone-npx.txt create mode 100644 docs/evidence/ws13/cold-clone.sh create mode 100644 docs/evidence/ws13/cold-container.txt create mode 100644 docs/evidence/ws13/cold-start.sh create mode 100644 docs/evidence/ws13/cold-trixie.txt create mode 100644 docs/evidence/ws13/container-tests.txt create mode 100644 docs/evidence/ws13/focused-tests.txt create mode 100644 docs/evidence/ws13/launcher-before-fix.txt create mode 100644 docs/evidence/ws13/local-agent-tests-30s-ceiling.txt create mode 100644 docs/evidence/ws13/local-agent-tests-final.txt create mode 100644 docs/evidence/ws13/local-agent-tests-first-attempt.txt create mode 100644 docs/evidence/ws13/local-agent-tests.txt create mode 100644 docs/evidence/ws13/research-typecheck.txt create mode 100644 docs/evidence/ws13/test-types-final.txt create mode 100644 docs/evidence/ws13/typechecks.txt diff --git a/README.md b/README.md index 0a8846ca..f250eafc 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,9 @@ npm start **Release status:** `create-flow` is not published yet. The commands above are the intended released entry point; use the [candidate artifact procedure](docs/evidence/ws13/README.md) to try this branch. A clean-machine first-agent run under 60 seconds has not -been established. +been established. The [clone + deterministic starter measurement](docs/evidence/ws13/cold-clone-direct.txt) +completed in 49.975 seconds in a fresh Linux container with Node and Git +provisioned before the timer; it does not measure an agent invocation. The agent starter requires Node 22.18+ and an installed, authenticated Claude CLI. Use `--cli codex` to select Codex, or `--template deterministic` for a @@ -48,13 +50,17 @@ resumable as a whole; each lowered step has its own journal run. For SDK callers, the CLI is optional: ```ts -import { createFlow, renderProgress } from '@relayflows/sdk'; +import { createFlow } from '@relayflows/sdk/create-flow'; +import { renderProgress } from '@relayflows/sdk/progress'; await createFlow('./my-flow', { cli: 'claude' }); // renderProgress(events) returns terminal lines; callers own event delivery. ``` See the [example gallery and individual run results](examples/README.md). +[Watch the captured agent run](docs/evidence/ws13/agent-run.cast) +([text transcript](docs/evidence/ws13/agent-run.txt)). + The larger examples currently refuse or time out; their intended budgets, artifact gates, workspace restrictions, and human approvals are preserved. diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md index 1d61da1b..5d2050b2 100644 --- a/docs/evidence/ws13/README.md +++ b/docs/evidence/ws13/README.md @@ -1,17 +1,120 @@ # WS-13 local development evidence -Base: `origin/main` at `f0a3b3b` (2.0.8), isolated branch -`feat/flows-local-dev-ux`. Implementation and verification are in progress. +**Acceptance is incomplete.** This branch implements SDK scaffolding, terminal +progress, an opt-in local agent worker, and the npm launcher fix. It does not +establish a clean-machine first-agent run under 60 seconds or a green gallery. + +## Captured results + +| Check | Result | Evidence | +|---|---|---| +| SDK, API type tests, and test-source typechecks | Exit 0 | [Commands and output](typechecks.txt) | +| Clone + empty-cache scaffold + direct deterministic run on fresh Debian Trixie | Completed in 49.975s; Node/Git provisioning excluded; no agent | [Command and output](cold-clone-direct.txt) | +| Earlier clone-inclusive run using npx for the final invocation | Completed in 60.063s; misses the timing target | [Command and output](cold-clone-npx.txt) | +| Focused scaffolding/authored-flow/CLI tests | 121 passed | [Command and output](focused-tests.txt) | +| Built CLI + real daemon + scripted agent wrapper | 4 passed with isolated Node 22.22.2 | [Final command and output](local-agent-tests-final.txt) | +| First live-worker test attempt | 3 process timeouts, 1 passed | [Command and output](local-agent-tests-first-attempt.txt) | +| Real Claude invocation in the generated project | Completed, 132.637s for the command; existing authenticated macOS host | [Transcript](agent-run.txt), [asciicast v2 recording](agent-run.cast) | +| Earlier recording attempts | Auth probe timeout; then a broken host Node shared-library dependency | [Auth timeout](agent-probe-timeout.txt), [host failure](agent-host-node-failure.txt) | +| Empty-cache install + deterministic run in fresh Debian Trixie container | Completed in 43.374s; deterministic template, no source clone or agent | [Command and output](cold-trixie.txt) | +| Empty-cache install + deterministic run in fresh Debian Bookworm container | Refused: published Linux daemon requires GLIBC_2.39; 55.223s | [Command and output](cold-container.txt) | +| Linux container test runner | esbuild Go runtime crashed under amd64 emulation before collecting tests | [Command, script and full output](container-tests.txt) | +| Research typecheck after correcting its compiler path | No type errors reported | [Command and output](research-typecheck.txt) | + +All four existing gallery entries were invoked separately. See the +[gallery table](../../../examples/README.md) for individual timings and literal +commands. Three refuse the unsupported `budget` header. Research reached the +outer 150-second verification limit without captured output; this does not +identify whether its preflight or execution was responsible. No gallery flow +was weakened or represented as successful. + +The recording uses the initial packed implementation plus the npm bin fix. +Its agent step invokes the real installed Claude CLI. The host already had +Node, provider authentication, and dependencies; this is **not** a cold-machine +measurement. The recorded command does not include a clone or installation. +Text transcripts normalize terminal CRLF to LF; the `.cast` files retain the +captured terminal bytes and elapsed timestamps. + +The functional CLI fixture has a 90-second cleanup ceiling. Its original +30-second process limit terminated a request while the worker still held a +live lease ([captured failure](local-agent-tests-30s-ceiling.txt)); startup and +preflight happen before that lease begins. Kernel lease behavior and the +separate 60-second cold-start criterion were not changed. Test-source types +were [checked again](test-types-final.txt) after fixing fixture binary discovery +to ask the existing Cargo wrapper for this worktree's target directory. + +The local worker is stream-only. Each invocation declares a fresh stream at +offset zero and uses the existing `AgentWorker` and journal protocol. It +refuses workspace declarations rather than inventing revision pins. It is a +worker attached to the selected local daemon, not an OS sandbox. The executor +still lowers each authored step to a separate kernel run; whole-body durable +resume is not introduced by this change. + +## Candidate artifacts, not a published release + +`create-flow` is not published. The new CLI imports the SDK's lightweight +`/create-flow` export; `relayflows` imports `/cli`, so package lookup works with +both nested and hoisted npm dependencies. Runtime packages stop registering +their legacy bundled executable as the competing npm `flows` command. +[The original artifact failure](launcher-before-fix.txt) is retained. + +Build and pack from this branch with Node 22.18+: -Both PR #243 and #244 diffs were inspected before SDK edits. #243 changes -README.md, authored-flow-error.ts, authored-flow-executor.ts, -authored-flow-loader.ts, cli/direct-run.ts, authored-flow tests, live-kernel -tests, and the surface manifest. #244 changes local-drive scripts, their tests, -BACKLOG, drive-local.yaml, and evidence; its current diff has no SDK source edit. -The fetched base already contains the authored-agent implementation. +```sh +npm --prefix packages/sdk ci --ignore-scripts +npm --prefix packages/sdk run build +mkdir -p /tmp/ws13-artifacts +npm pack --ignore-scripts --pack-destination /tmp/ws13-artifacts ./packages/sdk +npm pack --ignore-scripts --pack-destination /tmp/ws13-artifacts ./packages/create-flow +npm pack --ignore-scripts --pack-destination /tmp/ws13-artifacts ./packages/relayflows +``` + +For full installation testing, stage each runtime package's `bin/` from the +published 2.0.8 package before packing the updated runtime manifest. This +session reused those published binaries; it did not rebuild or change Rust. +The runtime tarballs retain legacy `bin/flows` because the existing release +gate requires it, while their npm `bin` maps now export only `relayflowd`. +The Debian failure above belongs to that published binary's libc requirement. + +Serve all candidate tarballs locally: + +```sh +node docs/evidence/ws13/stage-registry.mjs /tmp/ws13-artifacts 48734 +``` + +In a separate terminal, point npm at that registry; dependencies outside this +branch redirect to the public npm registry: + +```sh +npm_config_registry=http://127.0.0.1:48734 npx --yes create-flow@latest /tmp/my-flow +cd /tmp/my-flow +npm start +``` + +The final served tarballs match the SHA-256 values in [artifacts.json](artifacts.json). +[Packed-file hash check](artifact-check.txt). Restart the registry after repacking; it freezes package metadata and tarball bytes +at startup. Earlier cold recordings used the SDK-root launcher; the final +clone-inclusive recording uses the lightweight SDK/cli launcher. + +`cold-start.sh` uses the same registry with the deterministic template and an +empty cache. `record.py` captures real process output as an asciicast and text +transcript. Neither script silently converts a refusal into a successful run. + +## Scope and release blockers + +Base: `origin/main` at `f0a3b3b` (2.0.8), isolated branch +`feat/flows-local-dev-ux`. Both #243 and #244 diffs were inspected before SDK +edits. #243 is now merged; #244 remains open. Overlap with #243 is README.md, +`packages/sdk/src/authored-flow-executor.ts`, and `packages/sdk/src/cli/direct-run.ts`. +There is no file overlap with #244's inspected diff. Existing executor error, +output, gate, and lifecycle behavior is reused; its pre-existing size was not +expanded into an unrelated refactor. -The fetched tree contains four examples, including social-post-pipeline. -The existing gallery explicitly says three typecheck but do not run. No -all-green gallery or clean-machine timing claim has been established. +The independent release-gate owner must add `create-flow` to package versioning +and publishing, and to `scripts/pack-release.mjs`, which currently refuses that +package name. That script also requires the legacy runtime executable. +Those gates were not edited. No package was published and no merge is allowed. -Veto tools were not exposed in this session. No merge is authorized. +Veto tools were not exposed in this session. Several status DMs encountered server/overload errors. A later status to +`session-thread-rollout` received queue receipt `223075216797716480`; reading +was not confirmed. A coordination reply to WS-14 timed out. diff --git a/docs/evidence/ws13/agent-host-node-failure.txt b/docs/evidence/ws13/agent-host-node-failure.txt index 20428ae6..8031a783 100644 --- a/docs/evidence/ws13/agent-host-node-failure.txt +++ b/docs/evidence/ws13/agent-host-node-failure.txt @@ -1,8 +1,8 @@ $ cd /tmp/ws13-consumer/hello $ npx --no-install flows run hello.flow.ts --local-agent --input '{}' -dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib - Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node - Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file) +dyld[9200]: Library not loaded: /opt/homebrew/opt/ada-url/lib/libada.3.dylib + Referenced from: <87FBC746-7D47-3FD8-B0A3-97018CBF954B> /opt/homebrew/Cellar/node/26.5.0/bin/node + Reason: tried: '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/opt/ada-url/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/System/Volumes/Preboot/Cryptexes/OS/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file), '/opt/homebrew/Cellar/ada-url/4.0.0/lib/libada.3.dylib' (no such file) EXIT_CODE=-6 ELAPSED_SECONDS=0.053 diff --git a/docs/evidence/ws13/agent-probe-timeout.txt b/docs/evidence/ws13/agent-probe-timeout.txt index 32d31d34..aef48cda 100644 --- a/docs/evidence/ws13/agent-probe-timeout.txt +++ b/docs/evidence/ws13/agent-probe-timeout.txt @@ -1,13 +1,13 @@ $ cd /tmp/ws13-consumer/hello $ npx --no-install flows run hello.flow.ts --local-agent --input '{}' -npm notice run npx -npm notice run 'flows' run hello.flow.ts --local-agent --input {} -○ run-1 (deterministic) 0.00s -✓ run-1 (deterministic) 0.21s completionReason: success -Hello from Relayflows -○ agent-2 (agent) [agent: preparing] 0.00s -✗ agent-2 (agent) [agent: failed] 10.18s -REFUSED [invalid_spec] agent_cli_unresolved: Could not verify CLI "claude" for step "agent-2": the probe timed out after 10000ms. +npm notice run npx +npm notice run 'flows' run hello.flow.ts --local-agent --input {} +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.21s completionReason: success +Hello from Relayflows +○ agent-2 (agent) [agent: preparing] 0.00s +✗ agent-2 (agent) [agent: failed] 10.18s +REFUSED [invalid_spec] agent_cli_unresolved: Could not verify CLI "claude" for step "agent-2": the probe timed out after 10000ms. EXIT_CODE=2 ELAPSED_SECONDS=18.174 diff --git a/docs/evidence/ws13/agent-run.txt b/docs/evidence/ws13/agent-run.txt index 13916f34..d80f9df0 100644 --- a/docs/evidence/ws13/agent-run.txt +++ b/docs/evidence/ws13/agent-run.txt @@ -1,15 +1,15 @@ $ cd /tmp/ws13-consumer/hello $ npx --no-install flows run hello.flow.ts --local-agent --input '{}' -○ run-1 (deterministic) 0.00s -✓ run-1 (deterministic) 1.02s completionReason: success -Hello from Relayflows -○ agent-2 (agent) [agent: preparing] 0.00s -WAITING [worker_lease] Run "01M20JY3TM2STS5QE7HTM5QKKW" step "agent-2" (agent) is running under a worker lease until 1788873770124. -↻ agent-2 (agent) [agent: running] 12.51s -✓ agent-2 (agent) [agent: completed] 28.95s completionReason: success -Hello! How can I help you today? - -RUN 01M20JYM4T9FNGNFX4NQCK34JK completed (3 steps) completionReason: success +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 1.02s completionReason: success +Hello from Relayflows +○ agent-2 (agent) [agent: preparing] 0.00s +WAITING [worker_lease] Run "01M20JY3TM2STS5QE7HTM5QKKW" step "agent-2" (agent) is running under a worker lease until 1788873770124. +↻ agent-2 (agent) [agent: running] 12.51s +✓ agent-2 (agent) [agent: completed] 28.95s completionReason: success +Hello! How can I help you today? + +RUN 01M20JYM4T9FNGNFX4NQCK34JK completed (3 steps) completionReason: success EXIT_CODE=0 ELAPSED_SECONDS=132.637 diff --git a/docs/evidence/ws13/artifact-check.txt b/docs/evidence/ws13/artifact-check.txt new file mode 100644 index 00000000..9722430c --- /dev/null +++ b/docs/evidence/ws13/artifact-check.txt @@ -0,0 +1,14 @@ +$ python3 - <<'PY' +from pathlib import Path +import hashlib, json +for item in json.loads(Path('docs/evidence/ws13/artifacts.json').read_text()): + p = Path('/tmp/ws13-artifacts') / item['file'] + actual = hashlib.sha256(p.read_bytes()).hexdigest() + assert actual == item['sha256'], p + print(f'{actual} {p}') +PY +bcf04be7d69457fb45c09385b7f4394bad79a1e3eaf40c4595f4c814433b9e47 /tmp/ws13-artifacts/create-flow-2.0.8.tgz +ae555a4aa2a65b15fe934286d158d2b5477ad7a1eba8fbf738889416475ec7c0 /tmp/ws13-artifacts/relayflows-2.0.8.tgz +6e2d749b641abd66812c5e2633f37937b941854533b65993cb571ddfbb1c0744 /tmp/ws13-artifacts/relayflows-runtime-darwin-arm64-2.0.8.tgz +fd5940895bed98279c1d8aa4510901c5466dada3d7492906cc67f47f3d0a7f34 /tmp/ws13-artifacts/relayflows-runtime-linux-x64-2.0.8.tgz +6c1986cb526f7e348190be83b4665dd2094e5434bd8429b2bfab7e0cf077b0f6 /tmp/ws13-artifacts/relayflows-sdk-2.0.8.tgz diff --git a/docs/evidence/ws13/artifacts.json b/docs/evidence/ws13/artifacts.json new file mode 100644 index 00000000..37a565c4 --- /dev/null +++ b/docs/evidence/ws13/artifacts.json @@ -0,0 +1,22 @@ +[ + { + "file": "create-flow-2.0.8.tgz", + "sha256": "bcf04be7d69457fb45c09385b7f4394bad79a1e3eaf40c4595f4c814433b9e47" + }, + { + "file": "relayflows-2.0.8.tgz", + "sha256": "ae555a4aa2a65b15fe934286d158d2b5477ad7a1eba8fbf738889416475ec7c0" + }, + { + "file": "relayflows-runtime-darwin-arm64-2.0.8.tgz", + "sha256": "6e2d749b641abd66812c5e2633f37937b941854533b65993cb571ddfbb1c0744" + }, + { + "file": "relayflows-runtime-linux-x64-2.0.8.tgz", + "sha256": "fd5940895bed98279c1d8aa4510901c5466dada3d7492906cc67f47f3d0a7f34" + }, + { + "file": "relayflows-sdk-2.0.8.tgz", + "sha256": "6c1986cb526f7e348190be83b4665dd2094e5434bd8429b2bfab7e0cf077b0f6" + } +] diff --git a/docs/evidence/ws13/cold-clone-direct.txt b/docs/evidence/ws13/cold-clone-direct.txt new file mode 100644 index 00000000..feac9a4d --- /dev/null +++ b/docs/evidence/ws13/cold-clone-direct.txt @@ -0,0 +1,49 @@ +Environment: fresh linux/amd64 node:22-trixie-slim container. Git and CA certificates provisioned before timer. Empty npm cache. Public origin/main clone; final candidate npm tarballs served by the local registry. Deterministic template, no provider credentials. +$ bash cold-clone.sh http://host.docker.internal:48734 +Unable to find image 'node:22-trixie-slim' locally +22-trixie-slim: Pulling from library/node +16938f2846b3: Pulling fs layer +6310eb16bf42: Pulling fs layer +0a2cf2a45a8d: Pulling fs layer +6be37a3ab578: Pulling fs layer +ef24898c32a4: Pulling fs layer +16938f2846b3: Download complete +ef24898c32a4: Download complete +6be37a3ab578: Download complete +6310eb16bf42: Download complete +0a2cf2a45a8d: Download complete +bc985e67f78c: Download complete +ef24898c32a4: Pull complete +6310eb16bf42: Pull complete +9f6222264bca: Download complete +0a2cf2a45a8d: Pull complete +16938f2846b3: Pull complete +6be37a3ab578: Pull complete +Digest: sha256:7b8a0c89c54499bee567618f96578e1a12a800f062fbdbfd1fb6a443fa6f6284 +Status: Downloaded newer image for node:22-trixie-slim +debconf: unable to initialize frontend: Dialog +debconf: (TERM is not set, so the dialog frontend is not usable.) +debconf: falling back to frontend: Readline +debconf: unable to initialize frontend: Readline +debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/x86_64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl-base /usr/lib/x86_64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8, line 37.) +debconf: falling back to frontend: Teletype +debconf: unable to initialize frontend: Teletype +debconf: (This frontend requires a controlling tty.) +debconf: falling back to frontend: Noninteractive +v22.23.2 +Cloning into '/tmp/flows'... + +added 14 packages in 18s +Created /tmp/hello +Next: cd '/tmp/hello' && npm start +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.41s completionReason: success +Hello from Relayflows +RUN 01M20NDS4GA7R63D78ZRD17XPF completed (2 steps) completionReason: success +EXIT_CODE=0 +ELAPSED_SECONDS=49.975 diff --git a/docs/evidence/ws13/cold-clone-npx.txt b/docs/evidence/ws13/cold-clone-npx.txt new file mode 100644 index 00000000..1d3a60a1 --- /dev/null +++ b/docs/evidence/ws13/cold-clone-npx.txt @@ -0,0 +1,29 @@ +Environment: fresh linux/amd64 node:22-trixie-slim container. Git and CA certificates provisioned before timer. Empty npm cache. Public origin/main clone; candidate npm registry. Deterministic template. This earlier run used the SDK-root launcher, before switching its import to SDK/cli. +$ bash cold-clone.sh http://host.docker.internal:48733 +Original final command: npx --no-install flows run hello.flow.ts --input '{}' +debconf: unable to initialize frontend: Dialog +debconf: (TERM is not set, so the dialog frontend is not usable.) +debconf: falling back to frontend: Readline +debconf: unable to initialize frontend: Readline +debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/x86_64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl-base /usr/lib/x86_64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8, line 37.) +debconf: falling back to frontend: Teletype +debconf: unable to initialize frontend: Teletype +debconf: (This frontend requires a controlling tty.) +debconf: falling back to frontend: Noninteractive +v22.23.2 +Cloning into '/tmp/flows'... + +added 14 packages in 20s +Created /tmp/hello +Next: cd '/tmp/hello' && npm start +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.37s completionReason: success +Hello from Relayflows +RUN 01M20MDSDAAGMGVY1GRPHJY0TF completed (2 steps) completionReason: success +EXIT_CODE=0 +ELAPSED_SECONDS=60.063 diff --git a/docs/evidence/ws13/cold-clone.sh b/docs/evidence/ws13/cold-clone.sh new file mode 100644 index 00000000..4b75ea3e --- /dev/null +++ b/docs/evidence/ws13/cold-clone.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eu +export npm_config_registry="$1" +export npm_config_cache=/tmp/ws13-empty-cache +export npm_config_audit=false +export npm_config_fund=false +node --version +git clone --depth 1 https://github.com/AgentWorkforce/flows.git /tmp/flows +cd /tmp/flows +npx --yes create-flow@latest /tmp/hello --template deterministic +cd /tmp/hello +export PATH="/tmp/hello/node_modules/.bin:$PATH" +flows run hello.flow.ts --input '{}' diff --git a/docs/evidence/ws13/cold-container.txt b/docs/evidence/ws13/cold-container.txt new file mode 100644 index 00000000..6b0ecd85 --- /dev/null +++ b/docs/evidence/ws13/cold-container.txt @@ -0,0 +1,16 @@ +Environment: fresh linux/amd64 node:22-bookworm container, empty npm cache, candidate registry on the host. No provider credentials. +$ bash cold-start.sh http://host.docker.internal:48733 +v22.23.2 + +added 14 packages in 19s +Created /tmp/hello +Next: cd '/tmp/hello' && npm start +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice +REFUSED [daemon_start_failed] relayflowd exited 1 during startup. Last output in "/tmp/hello/.relayflowd/relayflowd.log": +/tmp/hello/node_modules/@relayflows/runtime-linux-x64/bin/relayflowd: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.39' not found (required by /tmp/hello/node_modules/@relayflows/runtime-linux-x64/bin/relayflowd) +EXIT_CODE=2 +ELAPSED_SECONDS=55.223 diff --git a/docs/evidence/ws13/cold-start.sh b/docs/evidence/ws13/cold-start.sh new file mode 100644 index 00000000..89e38a45 --- /dev/null +++ b/docs/evidence/ws13/cold-start.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -eu +# Candidate registry argument supplies this unpublished branch's packed npm artifacts. +export npm_config_registry="$1" +export npm_config_cache=/tmp/ws13-empty-cache +export npm_config_audit=false +export npm_config_fund=false +node --version +npx --yes create-flow@latest /tmp/hello --template deterministic +cd /tmp/hello +npx --no-install flows run hello.flow.ts --input '{}' diff --git a/docs/evidence/ws13/cold-trixie.txt b/docs/evidence/ws13/cold-trixie.txt new file mode 100644 index 00000000..0d9ca906 --- /dev/null +++ b/docs/evidence/ws13/cold-trixie.txt @@ -0,0 +1,39 @@ +Environment: fresh linux/amd64 node:22-trixie-slim container, empty npm cache, candidate registry on the host. No provider credentials; no source clone included. +$ bash cold-start.sh http://host.docker.internal:48733 +Unable to find image 'node:22-trixie-slim' locally +22-trixie-slim: Pulling from library/node +16938f2846b3: Pulling fs layer +6310eb16bf42: Pulling fs layer +ef24898c32a4: Pulling fs layer +0a2cf2a45a8d: Pulling fs layer +6be37a3ab578: Pulling fs layer +16938f2846b3: Download complete +6be37a3ab578: Download complete +ef24898c32a4: Download complete +bc985e67f78c: Download complete +6310eb16bf42: Download complete +9f6222264bca: Download complete +0a2cf2a45a8d: Download complete +ef24898c32a4: Pull complete +6310eb16bf42: Pull complete +0a2cf2a45a8d: Pull complete +16938f2846b3: Pull complete +6be37a3ab578: Pull complete +Digest: sha256:7b8a0c89c54499bee567618f96578e1a12a800f062fbdbfd1fb6a443fa6f6284 +Status: Image is up to date for node:22-trixie-slim +v22.23.2 + +added 14 packages in 16s +Created /tmp/hello +Next: cd '/tmp/hello' && npm start +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.34s completionReason: success +Hello from Relayflows +RUN 01M20KZR4QNKTHWD8M65AD0PS3 completed (2 steps) completionReason: success +EXIT_CODE=0 +ELAPSED_SECONDS=43.374 diff --git a/docs/evidence/ws13/container-tests.txt b/docs/evidence/ws13/container-tests.txt new file mode 100644 index 00000000..705ee106 --- /dev/null +++ b/docs/evidence/ws13/container-tests.txt @@ -0,0 +1,522 @@ +$ docker run --rm --platform linux/amd64 -v /tmp/ws13-artifacts:/artifacts:ro -v /tmp/ws13-container-checks.sh:/verify.sh:ro node:22-bookworm bash /verify.sh + +Contents of /verify.sh: +set -eu +node --version +mkdir -p /work /runtime +tar -xf /artifacts/source.tar -C /work +tar -xzf /artifacts/relayflows-runtime-linux-x64-2.0.8.tgz -C /runtime package/bin/relayflowd +cd /work +npm --prefix packages/sdk ci --ignore-scripts --no-audit --no-fund +npm --prefix packages/sdk run build +cd packages/sdk +RELAYFLOWD_BIN=/runtime/package/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-dev-ux.test.ts tests/local-agent-live.test.ts tests/cli.test.ts tests/authored-flow.test.ts tests/authored-flow-lifecycle-executor.test.ts + +Captured output: +v22.23.2 + +added 59 packages in 21s +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + + + RUN v2.1.9 /work/packages/sdk + +runtime: lfstack.push invalid packing: node=0xffff59972a80 cnt=0x1 packed=0xffff59972a800001 -> node=0xffffffff59972a80 +fatal error: lfstack.push + +runtime stack: +runtime.throw({0x9c2507?, 0x520000c000380350?}) + runtime/panic.go:1047 +0x5d fp=0xffff815ffb28 sp=0xffff815ffaf8 pc=0x4357dd +runtime.(*lfstack).push(0x2?, 0xffff815ffbd8?) + runtime/lfstack.go:29 +0x125 fp=0xffff815ffb68 sp=0xffff815ffb28 pc=0x40b4c5 +runtime.(*spanSetBlockAlloc).free(...) + runtime/mspanset.go:322 +runtime.(*spanSet).reset(0xd6bed0) + runtime/mspanset.go:264 +0x87 fp=0xffff815ffb98 sp=0xffff815ffb68 pc=0x42f747 +runtime.finishsweep_m() + runtime/mgcsweep.go:260 +0x9c fp=0xffff815ffbd8 sp=0xffff815ffb98 pc=0x42377c +runtime.gcStart.func1() + runtime/mgc.go:668 +0x17 fp=0xffff815ffbe8 sp=0xffff815ffbd8 pc=0x463397 +runtime.systemstack() + runtime/asm_amd64.s:496 +0x49 fp=0xffff815ffbf0 sp=0xffff815ffbe8 pc=0x467dc9 + +goroutine 81 [running]: +runtime.systemstack_switch() + runtime/asm_amd64.s:463 fp=0xc00021c690 sp=0xc00021c688 pc=0x467d60 +runtime.gcStart({0xc000302000?, 0xc00?, 0xc00?}) + runtime/mgc.go:667 +0x319 fp=0xc00021c718 sp=0xc00021c690 pc=0x4191b9 +runtime.mallocgc(0xc00, 0x9a40c0, 0x1) + runtime/malloc.go:1172 +0x777 fp=0xc00021c780 sp=0xc00021c718 pc=0x40d377 +runtime.growslice(0xc000294000, 0x40d20a?, 0xc00057425f?, 0xb?, 0x9a40c0) + runtime/slice.go:274 +0x4e9 fp=0xc00021c7e0 sp=0xc00021c780 pc=0x44db09 +github.com/evanw/esbuild/internal/js_parser.(*parser).newSymbol(...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:1228 +github.com/evanw/esbuild/internal/js_parser.(*parser).declareSymbol(0xc000222000, 0x15, {0x0?}, {0xc00057425f, 0xb}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:1368 +0x105 fp=0xc00021c8a0 sp=0xc00021c7e0 pc=0x6fed05 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmt(0xc000222000, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:7827 +0x80b6 fp=0xc00021d588 sp=0xc00021c8a0 pc=0x72fc76 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmtsUpTo(0xc000222000, 0x0, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:8151 +0xe5 fp=0xc00021d6d0 sp=0xc00021d588 pc=0x730a25 +github.com/evanw/esbuild/internal/js_parser.Parse({_, _, _, _, _, _}, {{0xc0003be98d, 0x24}, {0xc00032e170, 0x8}, ...}, ...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:17024 +0x386 fp=0xc00021e3b0 sp=0xc00021d6d0 pc=0x7707c6 +github.com/evanw/esbuild/internal/cache.(*JSCache).Parse(_, {_, _, _, _, _, _}, {{0xc0003be98d, 0x24}, {0xc00032e170, ...}, ...}, ...) + github.com/evanw/esbuild/internal/cache/cache_ast.go:170 +0x29e fp=0xc00021eb80 sp=0xc00021e3b0 pc=0x7c859e +github.com/evanw/esbuild/internal/bundler.parseFile({{0xab5770, 0xc000398270}, {0xc0000d4070, 0xc000388060, 0xc000388078, 0xc000320000, 0x6, 0xc000398090}, 0xc000110900, 0xc0001800c0, ...}) + github.com/evanw/esbuild/internal/bundler/bundler.go:261 +0x1485 fp=0xc000221b30 sp=0xc00021eb80 pc=0x805565 +github.com/evanw/esbuild/internal/bundler.(*scanner).maybeParseFile.func1() + github.com/evanw/esbuild/internal/bundler/bundler.go:1494 +0x45 fp=0xc000221fe0 sp=0xc000221b30 pc=0x810c45 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000221fe8 sp=0xc000221fe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/bundler.(*scanner).maybeParseFile + github.com/evanw/esbuild/internal/bundler/bundler.go:1494 +0xaeb + +goroutine 1 [syscall]: +syscall.Syscall(0xd737c0?, 0x1?, 0x800000?, 0x7ffff800000?) + syscall/syscall_linux.go:69 +0x27 fp=0xc00017bbe8 sp=0xc00017bb78 pc=0x49c8c7 +syscall.read(0xc000108000?, {0xc00017bda8?, 0xc000?, 0xc00017bca8?}) + syscall/zsyscall_linux_amd64.go:711 +0x45 fp=0xc00017bc28 sp=0xc00017bbe8 pc=0x49b025 +syscall.Read(...) + syscall/syscall_unix.go:178 +internal/poll.ignoringEINTRIO(...) + internal/poll/fd_unix.go:794 +internal/poll.(*FD).Read(0xc000108000?, {0xc00017bda8?, 0x4000?, 0x4000?}) + internal/poll/fd_unix.go:163 +0x2ce fp=0xc00017bcc0 sp=0xc00017bc28 pc=0x4b1c8e +os.(*File).read(...) + os/file_posix.go:31 +os.(*File).Read(0xc000012010, {0xc00017bda8?, 0x4000?, 0x4000?}) + os/file.go:118 +0x5e fp=0xc00017bd18 sp=0xc00017bcc0 pc=0x4b645e +main.runService(0x1) + github.com/evanw/esbuild/cmd/esbuild/service.go:134 +0x38f fp=0xc00017fe38 sp=0xc00017bd18 pc=0x8ede4f +main.main() + github.com/evanw/esbuild/cmd/esbuild/main.go:241 +0xa29 fp=0xc00017ff80 sp=0xc00017fe38 pc=0x8ec449 +runtime.main() + runtime/proc.go:250 +0x207 fp=0xc00017ffe0 sp=0xc00017ff80 pc=0x438107 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc00017ffe8 sp=0xc00017ffe0 pc=0x469e41 + +goroutine 2 [force gc (idle)]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + runtime/proc.go:381 +0xd6 fp=0xc000050fb0 sp=0xc000050f90 pc=0x438536 +runtime.goparkunlock(...) + runtime/proc.go:387 +runtime.forcegchelper() + runtime/proc.go:305 +0xb0 fp=0xc000050fe0 sp=0xc000050fb0 pc=0x438370 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000050fe8 sp=0xc000050fe0 pc=0x469e41 +created by runtime.init.6 + runtime/proc.go:293 +0x25 + +goroutine 17 [runnable]: +runtime.goschedIfBusy() + runtime/proc.go:344 +0x30 fp=0xc00004c780 sp=0xc00004c768 pc=0x438430 +runtime.bgsweep(0x0?) + runtime/mgcsweep.go:308 +0x15d fp=0xc00004c7c8 sp=0xc00004c780 pc=0x42395d +runtime.gcenable.func1() + runtime/mgc.go:178 +0x26 fp=0xc00004c7e0 sp=0xc00004c7c8 pc=0x418bc6 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc00004c7e8 sp=0xc00004c7e0 pc=0x469e41 +created by runtime.gcenable + runtime/mgc.go:178 +0x6b + +goroutine 18 [runnable]: +runtime.(*scavengerState).run(0xd439c0) + runtime/mgcscavenge.go:551 +0x1a5 fp=0xc00004cfa0 sp=0xc00004cf98 pc=0x421d25 +runtime.bgscavenge(0x0?) + runtime/mgcscavenge.go:631 +0x51 fp=0xc00004cfc8 sp=0xc00004cfa0 pc=0x421d91 +runtime.gcenable.func2() + runtime/mgc.go:179 +0x26 fp=0xc00004cfe0 sp=0xc00004cfc8 pc=0x418b66 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc00004cfe8 sp=0xc00004cfe0 pc=0x469e41 +created by runtime.gcenable + runtime/mgc.go:179 +0xaa + +goroutine 3 [finalizer wait]: +runtime.gopark(0x4388b2?, 0xffff89711b88?, 0x0?, 0x0?, 0xc000050770?) + runtime/proc.go:381 +0xd6 fp=0xc000050628 sp=0xc000050608 pc=0x438536 +runtime.runfinq() + runtime/mfinal.go:193 +0x107 fp=0xc0000507e0 sp=0xc000050628 pc=0x417c07 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0000507e8 sp=0xc0000507e0 pc=0x469e41 +created by runtime.createfing + runtime/mfinal.go:163 +0x45 + +goroutine 4 [chan receive]: +runtime.gopark(0xc0003be940?, 0x0?, 0x0?, 0x0?, 0x0?) + runtime/proc.go:381 +0xd6 fp=0xc0000516c8 sp=0xc0000516a8 pc=0x438536 +runtime.chanrecv(0xc0000780c0, 0xc0000517b8, 0x1) + runtime/chan.go:583 +0x49d fp=0xc000051758 sp=0xc0000516c8 pc=0x4067bd +runtime.chanrecv2(0xc000012018?, 0xc0003be940?) + runtime/chan.go:447 +0x18 fp=0xc000051780 sp=0xc000051758 pc=0x4062f8 +main.runService.func1() + github.com/evanw/esbuild/cmd/esbuild/service.go:98 +0x4a fp=0xc0000517e0 sp=0xc000051780 pc=0x8ee32a +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0000517e8 sp=0xc0000517e0 pc=0x469e41 +created by main.runService + github.com/evanw/esbuild/cmd/esbuild/service.go:97 +0x1e5 + +goroutine 5 [sleep]: +runtime.gopark(0xcb9c7a9d9a?, 0x94bc00?, 0xd0?, 0xb1?, 0xc0003be940?) + runtime/proc.go:381 +0xd6 fp=0xc0003d6760 sp=0xc0003d6740 pc=0x438536 +time.Sleep(0x3b9aca00) + runtime/time.go:195 +0x135 fp=0xc0003d67a0 sp=0xc0003d6760 pc=0x466df5 +main.runService.func3() + github.com/evanw/esbuild/cmd/esbuild/service.go:124 +0x45 fp=0xc0003d67e0 sp=0xc0003d67a0 pc=0x8ee1e5 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d67e8 sp=0xc0003d67e0 pc=0x469e41 +created by main.runService + github.com/evanw/esbuild/cmd/esbuild/service.go:122 +0x31c + +goroutine 44 [GC worker (idle)]: +runtime.gopark(0xc0003d4fa0?, 0x1?, 0x98?, 0xc2?, 0xc0003d4f90?) + runtime/proc.go:381 +0xd6 fp=0xc0003d4f50 sp=0xc0003d4f30 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d4fe0 sp=0xc0003d4f50 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d4fe8 sp=0xc0003d4fe0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 67 [runnable]: +runtime.asyncPreempt2() + runtime/preempt.go:307 +0x3f fp=0xc000696060 sp=0xc000696040 pc=0x436c3f +runtime.asyncPreempt() + runtime/preempt_amd64.s:53 +0xdb fp=0xc0006961e8 sp=0xc000696060 pc=0x46b47b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseParenExpr(0xc000170a80, {0x0?}, 0x1, {{{0x6966b8?}, 0xc0?}, 0xa?}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3114 +0x11cb fp=0xc000696640 sp=0xc0006961e8 pc=0x70d42b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseAsyncPrefixExpr(0xc000170a80, {{0x59ec20?}, 0xc0?}, 0x1, 0x68?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:2915 +0x815 fp=0xc0006967f8 sp=0xc000696640 pc=0x70b775 +github.com/evanw/esbuild/internal/js_parser.(*parser).parsePrefix(0xc000170a80, 0x1, 0xc000697348, 0xd8?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3424 +0x1b7d fp=0xc000696f60 sp=0xc0006967f8 pc=0x71095d +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x1, 0x1?, 0x98?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4043 +0x3b fp=0xc000696fb8 sp=0xc000696f60 pc=0x713d3b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprOrBindings(0xc0001710b8?, 0x1?, 0xc0006971b8?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4030 +0x1b fp=0xc000696fe8 sp=0xc000696fb8 pc=0x713bfb +github.com/evanw/esbuild/internal/js_parser.(*parser).parsePrefix(0xc000170a80, 0x1, 0xc000697b38, 0xc8?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3697 +0x3a51 fp=0xc000697750 sp=0xc000696fe8 pc=0x712831 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x1, 0x0?, 0x0?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4043 +0x3b fp=0xc0006977a8 sp=0xc000697750 pc=0x713d3b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprOrBindings(0xc0001710b8?, 0x38?, 0x3ff0000000000001?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4030 +0x1b fp=0xc0006977d8 sp=0xc0006977a8 pc=0x713bfb +github.com/evanw/esbuild/internal/js_parser.(*parser).parsePrefix(0xc000170a80, 0x1, 0x0, 0x48?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3697 +0x3a51 fp=0xc000697f40 sp=0xc0006977d8 pc=0x712831 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x1, 0x1?, 0x48?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4043 +0x3b fp=0xc000697f98 sp=0xc000697f40 pc=0x713d3b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExpr(...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4034 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseCallArgs(0xc000170a80) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4915 +0x14c fp=0xc000698058 sp=0xc000697f98 pc=0x719acc +github.com/evanw/esbuild/internal/js_parser.(*parser).parseSuffix(0xc000170a80, {{0xaaf6e0?, 0xc00059f490?}, {0x590440?}}, 0x0, 0x0, 0x0) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4318 +0x20cd fp=0xc000698458 sp=0xc000698058 pc=0x715f4d +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x0, 0xc000596910?, 0x1?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4070 +0x127 fp=0xc0006984b0 sp=0xc000698458 pc=0x713e27 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprOrLetOrUsingStmt(0xc000170a80?, {0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4813 +0xa17 fp=0xc0006985a8 sp=0xc0006984b0 pc=0x719917 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmt(0xc000170a80, {0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:7917 +0x66b6 fp=0xc000699290 sp=0xc0006985a8 pc=0x72e276 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmtsUpTo(0xc000170a80, 0x12, {0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:8151 +0xe5 fp=0xc0006993d8 sp=0xc000699290 pc=0x730a25 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseFnBody(0xc000170a80, {0x0, 0x0, {{0x0}, 0x0}, {0x21f}, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:8110 +0x17b fp=0xc0006994b8 sp=0xc0006993d8 pc=0x7305fb +github.com/evanw/esbuild/internal/js_parser.(*parser).parseArrowBody(0xc000170a80, {0xd728e0, 0x0, 0x0}, {0x0, 0x0, {{0x0}, 0x0}, {0x21f}, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:2809 +0x1aa fp=0xc000699678 sp=0xc0006994b8 pc=0x70a7ca +github.com/evanw/esbuild/internal/js_parser.(*parser).parseParenExpr(0xc000170a80, {0x0?}, 0x1, {{{0x0?}, 0x0?}, 0x0?}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3140 +0x14a5 fp=0xc000699ad0 sp=0xc000699678 pc=0x70d705 +github.com/evanw/esbuild/internal/js_parser.(*parser).parsePrefix(0xc000170a80, 0x1, 0x0, 0x1?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3369 +0x1305 fp=0xc00069a238 sp=0xc000699ad0 pc=0x7100e5 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x1, 0x0?, 0x0?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4043 +0x3b fp=0xc00069a290 sp=0xc00069a238 pc=0x713d3b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExpr(...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4034 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseCallArgs(0xc000170a80) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4915 +0x14c fp=0xc00069a350 sp=0xc00069a290 pc=0x719acc +github.com/evanw/esbuild/internal/js_parser.(*parser).parseSuffix(0xc000170a80, {{0xaaf6e0?, 0xc00059e560?}, {0x412f65?}}, 0x0, 0x0, 0x0) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4318 +0x20cd fp=0xc00069a750 sp=0xc00069a350 pc=0x715f4d +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc000170a80, 0x0, 0x20?, 0x0?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4070 +0x127 fp=0xc00069a7a8 sp=0xc00069a750 pc=0x713e27 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprOrLetOrUsingStmt(0xd?, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4813 +0xa17 fp=0xc00069a8a0 sp=0xc00069a7a8 pc=0x719917 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmt(0xc000170a80, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:7917 +0x66b6 fp=0xc00069b588 sp=0xc00069a8a0 pc=0x72e276 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmtsUpTo(0xc000170a80, 0x0, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:8151 +0xe5 fp=0xc00069b6d0 sp=0xc00069b588 pc=0x730a25 +github.com/evanw/esbuild/internal/js_parser.Parse({_, _, _, _, _, _}, {{0xc0003b205d, 0x41}, {0xc0005aa060, 0x25}, ...}, ...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:17024 +0x386 fp=0xc00069c3b0 sp=0xc00069b6d0 pc=0x7707c6 +github.com/evanw/esbuild/internal/cache.(*JSCache).Parse(_, {_, _, _, _, _, _}, {{0xc0003b205d, 0x41}, {0xc0005aa060, ...}, ...}, ...) + github.com/evanw/esbuild/internal/cache/cache_ast.go:170 +0x29e fp=0xc00069cb80 sp=0xc00069c3b0 pc=0x7c859e +github.com/evanw/esbuild/internal/bundler.parseFile({{0xab5770, 0xc00058e7b0}, {0xc00060a000, 0xc00019cf00, 0xc00019cf18, 0xc0001b2320, 0x6, 0xc00019b380}, 0xc0000e4900, 0xc000180240, ...}) + github.com/evanw/esbuild/internal/bundler/bundler.go:261 +0x1485 fp=0xc00069fb30 sp=0xc00069cb80 pc=0x805565 +github.com/evanw/esbuild/internal/bundler.(*scanner).maybeParseFile.func1() + github.com/evanw/esbuild/internal/bundler/bundler.go:1494 +0x45 fp=0xc00069ffe0 sp=0xc00069fb30 pc=0x810c45 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc00069ffe8 sp=0xc00069ffe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/bundler.(*scanner).maybeParseFile + github.com/evanw/esbuild/internal/bundler/bundler.go:1494 +0xaeb + +goroutine 65 [GC worker (idle)]: +runtime.gopark(0x0?, 0xc000332240?, 0x0?, 0xf0?, 0x0?) + runtime/proc.go:381 +0xd6 fp=0xc0003d8750 sp=0xc0003d8730 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d87e0 sp=0xc0003d8750 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d87e8 sp=0xc0003d87e0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 9 [GC worker (idle)]: +runtime.gopark(0xc00004d7a0?, 0x1?, 0x38?, 0x82?, 0xc00004d790?) + runtime/proc.go:381 +0xd6 fp=0xc00004d750 sp=0xc00004d730 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc00004d7e0 sp=0xc00004d750 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc00004d7e8 sp=0xc00004d7e0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 66 [runnable]: +runtime.asyncPreempt2() + runtime/preempt.go:307 +0x3f fp=0xc0002d9560 sp=0xc0002d9540 pc=0x436c3f +runtime.asyncPreempt() + runtime/preempt_amd64.s:53 +0xdb fp=0xc0002d96e8 sp=0xc0002d9560 pc=0x46b47b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseParenExpr(0xc00045c000, {0x0?}, 0x1, {{{0x0?}, 0x0?}, 0x41?}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3118 +0x12c6 fp=0xc0002d9b40 sp=0xc0002d96e8 pc=0x70d526 +github.com/evanw/esbuild/internal/js_parser.(*parser).parsePrefix(0xc00045c000, 0x1, 0x0, 0x0?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:3369 +0x1305 fp=0xc0002da2a8 sp=0xc0002d9b40 pc=0x7100e5 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExprCommon(0xc00045c000, 0x1, 0x100?, 0x40?) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4043 +0x3b fp=0xc0002da300 sp=0xc0002da2a8 pc=0x713d3b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseExpr(...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:4034 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseAndDeclareDecls(0xc00045c000, 0x1, {0x0, 0x1, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:5397 +0x30b fp=0xc0002da3f0 sp=0xc0002da300 pc=0x71db8b +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmt(0xc00045c000, {0x0, 0x1, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:7243 +0x3910 fp=0xc0002db0d8 sp=0xc0002da3f0 pc=0x72b4d0 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmt(0xc00045c000, {0x0, 0x1, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:6872 +0x6450 fp=0xc0002dbdc0 sp=0xc0002db0d8 pc=0x72e010 +github.com/evanw/esbuild/internal/js_parser.(*parser).parseStmtsUpTo(0xc00045c000, 0x0, {0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...}) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:8151 +0xe5 fp=0xc0002dbf08 sp=0xc0002dbdc0 pc=0x730a25 +github.com/evanw/esbuild/internal/js_parser.Parse({_, _, _, _, _, _}, {{0x9bfb16, 0x9}, {0x9befac, 0x7}, ...}, ...) + github.com/evanw/esbuild/internal/js_parser/js_parser.go:17024 +0x386 fp=0xc0002dcbe8 sp=0xc0002dbf08 pc=0x7707c6 +github.com/evanw/esbuild/internal/bundler.(*runtimeCache).parseRuntime(_, _) + github.com/evanw/esbuild/internal/bundler/bundler.go:3168 +0x33e fp=0xc0002dd6a0 sp=0xc0002dcbe8 pc=0x82239e +github.com/evanw/esbuild/internal/bundler.ScanBundle.func2() + github.com/evanw/esbuild/internal/bundler/bundler.go:1301 +0x88 fp=0xc0002ddfe0 sp=0xc0002dd6a0 pc=0x80fb68 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0002ddfe8 sp=0xc0002ddfe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/bundler.ScanBundle + github.com/evanw/esbuild/internal/bundler/bundler.go:1300 +0x93b + +goroutine 42 [GC worker (idle)]: +runtime.gopark(0xc0003d47a0?, 0x1?, 0xb8?, 0xc3?, 0xc0003d4790?) + runtime/proc.go:381 +0xd6 fp=0xc0003d4750 sp=0xc0003d4730 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d47e0 sp=0xc0003d4750 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d47e8 sp=0xc0003d47e0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 27 [runnable]: +github.com/evanw/esbuild/internal/bundler.(*Bundle).computeDataForSourceMapsInParallel.func4() + github.com/evanw/esbuild/internal/bundler/bundler.go:3038 fp=0xc000255fe0 sp=0xc000255fd8 pc=0x820dc0 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000255fe8 sp=0xc000255fe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/bundler.(*Bundle).computeDataForSourceMapsInParallel + github.com/evanw/esbuild/internal/bundler/bundler.go:3038 +0x227 + +goroutine 43 [GC worker (idle)]: +runtime.gopark(0xc0003d57a0?, 0x1?, 0x58?, 0xc3?, 0xc0003d5790?) + runtime/proc.go:381 +0xd6 fp=0xc0003d5750 sp=0xc0003d5730 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d57e0 sp=0xc0003d5750 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d57e8 sp=0xc0003d57e0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 58 [semacquire]: +runtime.gopark(0xc0004cc480?, 0x18?, 0x80?, 0xa1?, 0xffff80c39aa8?) + runtime/proc.go:381 +0xd6 fp=0xc0000eb1c8 sp=0xc0000eb1a8 pc=0x438536 +runtime.goparkunlock(...) + runtime/proc.go:387 +runtime.semacquire1(0xc0004c3148, 0x18?, 0x1, 0x0, 0xb1?) + runtime/sema.go:160 +0x20f fp=0xc0000eb230 sp=0xc0000eb1c8 pc=0x449a6f +sync.runtime_Semacquire(0xc0004cc4c8?) + runtime/sema.go:62 +0x27 fp=0xc0000eb268 sp=0xc0000eb230 pc=0x465f67 +sync.(*WaitGroup).Wait(0xd728e0?) + sync/waitgroup.go:116 +0x4b fp=0xc0000eb290 sp=0xc0000eb268 pc=0x48268b +github.com/evanw/esbuild/internal/graph.CloneLinkerGraph({0xc0004d8000, 0x2, 0x2}, {0xc0004c30f8, 0x2, 0x2}, {0xc0000a81b0?, 0x1?, 0x1?}, 0x0) + github.com/evanw/esbuild/internal/graph/graph.go:255 +0x40c fp=0xc0000eb4a0 sp=0xc0000eb290 pc=0x80102c +github.com/evanw/esbuild/internal/linker.Link(0xc00023f400, 0x0, {0xc0005926e0, 0xc0004cc468, 0xc0000a8108, 0xc000098040, 0x6, 0xc0000a2360}, {0xab5770, 0xc0000a2540}, ...) + github.com/evanw/esbuild/internal/linker/linker.go:253 +0x465 fp=0xc0000eb948 sp=0xc0000eb4a0 pc=0x867ee5 +github.com/evanw/esbuild/internal/bundler.(*Bundle).Compile(0xc000080400, {0xc0000d4000, 0xc0000a80f0, 0xc0000a8108, 0xc000098040, 0x6, 0xc0000a2360}, 0x0, 0x0, 0xa05b10) + github.com/evanw/esbuild/internal/bundler/bundler.go:2849 +0x936 fp=0xc0000ec0c8 sp=0xc0000eb948 pc=0x81ec56 +github.com/evanw/esbuild/pkg/api.transformImpl({_, _}, {0x0, 0x0, 0x0, 0xc0000a2300, 0x3, {0x0, 0x0}, 0x0, ...}) + github.com/evanw/esbuild/pkg/api/api_impl.go:1763 +0x1445 fp=0xc0000ed8c0 sp=0xc0000ec0c8 pc=0x8ba705 +github.com/evanw/esbuild/pkg/api.Transform(...) + github.com/evanw/esbuild/pkg/api/api.go:465 +main.(*serviceType).handleTransformRequest(0xc000176060?, 0x1?, 0xc00019b230?) + github.com/evanw/esbuild/cmd/esbuild/service.go:1167 +0x358 fp=0xc0000edf78 sp=0xc0000ed8c0 pc=0x8f8838 +main.(*serviceType).handleIncomingPacket.func3() + github.com/evanw/esbuild/cmd/esbuild/service.go:242 +0x68 fp=0xc0000edfe0 sp=0xc0000edf78 pc=0x8f25a8 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0000edfe8 sp=0xc0000edfe0 pc=0x469e41 +created by main.(*serviceType).handleIncomingPacket + github.com/evanw/esbuild/cmd/esbuild/service.go:240 +0x16ec + +goroutine 10 [GC worker (idle)]: +runtime.gopark(0xcbadc31da4?, 0x0?, 0x0?, 0x0?, 0x0?) + runtime/proc.go:381 +0xd6 fp=0xc000052f50 sp=0xc000052f30 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc000052fe0 sp=0xc000052f50 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000052fe8 sp=0xc000052fe0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 45 [GC worker (idle)]: +runtime.gopark(0xcbadac01f3?, 0x1?, 0x28?, 0x3d?, 0xc0003b02e3?) + runtime/proc.go:381 +0xd6 fp=0xc0003d8f50 sp=0xc0003d8f30 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d8fe0 sp=0xc0003d8f50 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d8fe8 sp=0xc0003d8fe0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 26 [GC worker (idle)]: +runtime.gopark(0xa05f10?, 0xc0003a0b40?, 0x1a?, 0x14?, 0x0?) + runtime/proc.go:381 +0xd6 fp=0xc0003d5f50 sp=0xc0003d5f30 pc=0x438536 +runtime.gcBgMarkWorker() + runtime/mgc.go:1275 +0xf1 fp=0xc0003d5fe0 sp=0xc0003d5f50 pc=0x41a7f1 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d5fe8 sp=0xc0003d5fe0 pc=0x469e41 +created by runtime.gcBgMarkStartWorkers + runtime/mgc.go:1199 +0x25 + +goroutine 46 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0001034b0?) + runtime/proc.go:381 +0xd6 fp=0xc000103430 sp=0xc000103410 pc=0x438536 +runtime.chanrecv(0xc0001fe060, 0xc0001038c8, 0x1) + runtime/chan.go:583 +0x49d fp=0xc0001034c0 sp=0xc000103430 pc=0x4067bd +runtime.chanrecv1(0x0?, 0x11?) + runtime/chan.go:442 +0x18 fp=0xc0001034e8 sp=0xc0001034c0 pc=0x4062b8 +github.com/evanw/esbuild/internal/bundler.(*scanner).scanAllDependencies(0xc0000e4d80) + github.com/evanw/esbuild/internal/bundler/bundler.go:2024 +0x239 fp=0xc000103a08 sp=0xc0001034e8 pc=0x815d59 +github.com/evanw/esbuild/internal/bundler.ScanBundle(_, {_, _, _, _, _, _}, {_, _}, 0xc000180240, ...) + github.com/evanw/esbuild/internal/bundler/bundler.go:1371 +0xb36 fp=0xc0001040c8 sp=0xc000103a08 pc=0x80f396 +github.com/evanw/esbuild/pkg/api.transformImpl({_, _}, {0x0, 0x0, 0x0, 0xc00019b320, 0x3, {0x0, 0x0}, 0x0, ...}) + github.com/evanw/esbuild/pkg/api/api_impl.go:1758 +0x1365 fp=0xc0001058c0 sp=0xc0001040c8 pc=0x8ba625 +github.com/evanw/esbuild/pkg/api.Transform(...) + github.com/evanw/esbuild/pkg/api/api.go:465 +main.(*serviceType).handleTransformRequest(0xc000176060?, 0x2?, 0xc000398cc0?) + github.com/evanw/esbuild/cmd/esbuild/service.go:1167 +0x358 fp=0xc000105f78 sp=0xc0001058c0 pc=0x8f8838 +main.(*serviceType).handleIncomingPacket.func3() + github.com/evanw/esbuild/cmd/esbuild/service.go:242 +0x68 fp=0xc000105fe0 sp=0xc000105f78 pc=0x8f25a8 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000105fe8 sp=0xc000105fe0 pc=0x469e41 +created by main.(*serviceType).handleIncomingPacket + github.com/evanw/esbuild/cmd/esbuild/service.go:240 +0x16ec + +goroutine 47 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00018f4b0?) + runtime/proc.go:381 +0xd6 fp=0xc00018f430 sp=0xc00018f410 pc=0x438536 +runtime.chanrecv(0xc0003bc000, 0xc00018f8c8, 0x1) + runtime/chan.go:583 +0x49d fp=0xc00018f4c0 sp=0xc00018f430 pc=0x4067bd +runtime.chanrecv1(0x0?, 0x11?) + runtime/chan.go:442 +0x18 fp=0xc00018f4e8 sp=0xc00018f4c0 pc=0x4062b8 +github.com/evanw/esbuild/internal/bundler.(*scanner).scanAllDependencies(0xc000110d80) + github.com/evanw/esbuild/internal/bundler/bundler.go:2024 +0x239 fp=0xc00018fa08 sp=0xc00018f4e8 pc=0x815d59 +github.com/evanw/esbuild/internal/bundler.ScanBundle(_, {_, _, _, _, _, _}, {_, _}, 0xc0001800c0, ...) + github.com/evanw/esbuild/internal/bundler/bundler.go:1371 +0xb36 fp=0xc0001900c8 sp=0xc00018fa08 pc=0x80f396 +github.com/evanw/esbuild/pkg/api.transformImpl({_, _}, {0x0, 0x0, 0x0, 0xc000398030, 0x3, {0x0, 0x0}, 0x0, ...}) + github.com/evanw/esbuild/pkg/api/api_impl.go:1758 +0x1365 fp=0xc0001918c0 sp=0xc0001900c8 pc=0x8ba625 +github.com/evanw/esbuild/pkg/api.Transform(...) + github.com/evanw/esbuild/pkg/api/api.go:465 +main.(*serviceType).handleTransformRequest(0xc000176060?, 0x3?, 0xc000398d20?) + github.com/evanw/esbuild/cmd/esbuild/service.go:1167 +0x358 fp=0xc000191f78 sp=0xc0001918c0 pc=0x8f8838 +main.(*serviceType).handleIncomingPacket.func3() + github.com/evanw/esbuild/cmd/esbuild/service.go:242 +0x68 fp=0xc000191fe0 sp=0xc000191f78 pc=0x8f25a8 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc000191fe8 sp=0xc000191fe0 pc=0x469e41 +created by main.(*serviceType).handleIncomingPacket + github.com/evanw/esbuild/cmd/esbuild/service.go:240 +0x16ec + +goroutine 28 [runnable]: +github.com/evanw/esbuild/internal/graph.CloneLinkerGraph.func2() + github.com/evanw/esbuild/internal/graph/graph.go:153 fp=0xc0003d6fe0 sp=0xc0003d6fd8 pc=0x801860 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d6fe8 sp=0xc0003d6fe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/graph.CloneLinkerGraph + github.com/evanw/esbuild/internal/graph/graph.go:153 +0x235 + +goroutine 29 [runnable]: +runtime.gcTrigger.test({0x0?, 0x0?, 0x0?}) + runtime/mgc.go:547 +0xfb fp=0xc0000c5c20 sp=0xc0000c5c18 pc=0x418e7b +runtime.mallocgc(0x1fe0, 0x9a40c0, 0x1) + runtime/malloc.go:1171 +0x768 fp=0xc0000c5c88 sp=0xc0000c5c20 pc=0x40d368 +runtime.growslice(0xd728e0, 0x0?, 0x0?, 0x0?, 0x9a40c0) + runtime/slice.go:274 +0x4e9 fp=0xc0000c5ce8 sp=0xc0000c5c88 pc=0x44db09 +github.com/evanw/esbuild/internal/graph.CloneLinkerGraph.func1(0x1) + github.com/evanw/esbuild/internal/graph/graph.go:167 +0x2bc fp=0xc0000c5fc8 sp=0xc0000c5ce8 pc=0x801b7c +github.com/evanw/esbuild/internal/graph.CloneLinkerGraph.func2() + github.com/evanw/esbuild/internal/graph/graph.go:253 +0x29 fp=0xc0000c5fe0 sp=0xc0000c5fc8 pc=0x801889 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0000c5fe8 sp=0xc0000c5fe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/graph.CloneLinkerGraph + github.com/evanw/esbuild/internal/graph/graph.go:153 +0x235 + +goroutine 48 [runnable]: +github.com/evanw/esbuild/internal/bundler.ScanBundle.func2() + github.com/evanw/esbuild/internal/bundler/bundler.go:1300 fp=0xc0003d9fe0 sp=0xc0003d9fd8 pc=0x80fae0 +runtime.goexit() + runtime/asm_amd64.s:1598 +0x1 fp=0xc0003d9fe8 sp=0xc0003d9fe0 pc=0x469e41 +created by github.com/evanw/esbuild/internal/bundler.ScanBundle + github.com/evanw/esbuild/internal/bundler/bundler.go:1300 +0x93b + ❯ tests/authored-flow.test.ts (0 test) + ❯ tests/authored-flow-lifecycle-executor.test.ts (0 test) + ❯ tests/cli.test.ts (0 test) + ❯ tests/local-dev-ux.test.ts (0 test) + ❯ tests/local-agent-live.test.ts (0 test) + +⎯⎯⎯⎯⎯⎯ Failed Suites 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-flow-lifecycle-executor.test.ts [ tests/authored-flow-lifecycle-executor.test.ts ] + FAIL tests/authored-flow.test.ts [ tests/authored-flow.test.ts ] + FAIL tests/cli.test.ts [ tests/cli.test.ts ] +Error: The service was stopped + Plugin: vite:esbuild + File: /work/packages/sdk/tests/authored-flow-lifecycle-executor.test.ts + ❯ node_modules/esbuild/lib/main.js:737:38 + ❯ responseCallbacks. node_modules/esbuild/lib/main.js:622:9 + ❯ Socket.afterClose node_modules/esbuild/lib/main.js:613:28 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯ + + FAIL tests/local-agent-live.test.ts [ tests/local-agent-live.test.ts ] + FAIL tests/local-dev-ux.test.ts [ tests/local-dev-ux.test.ts ] +Error: The service is no longer running + Plugin: vite:esbuild + File: /work/packages/sdk/tests/local-agent-live.test.ts + ❯ node_modules/esbuild/lib/main.js:737:38 + ❯ sendRequest node_modules/esbuild/lib/main.js:618:36 + ❯ start node_modules/esbuild/lib/main.js:736:9 + ❯ Object.transform2 [as transform] node_modules/esbuild/lib/main.js:797:5 + ❯ node_modules/esbuild/lib/main.js:2040:77 + ❯ Object.transform node_modules/esbuild/lib/main.js:2040:36 + ❯ transform node_modules/esbuild/lib/main.js:1875:62 + ❯ transformWithEsbuild node_modules/vite/dist/node/chunks/dep-BK3b2jBa.js:19232:26 + ❯ TransformPluginContext.transform node_modules/vite/dist/node/chunks/dep-BK3b2jBa.js:19297:24 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯ + + Test Files 5 failed (5) + Tests no tests + Start at 13:31:27 + Duration 7.95s (transform 609ms, setup 0ms, collect 0ms, tests 0ms, environment 30ms, prepare 8.96s) + diff --git a/docs/evidence/ws13/focused-tests.txt b/docs/evidence/ws13/focused-tests.txt new file mode 100644 index 00000000..039d3ce0 --- /dev/null +++ b/docs/evidence/ws13/focused-tests.txt @@ -0,0 +1,18 @@ +$ cd packages/sdk && node node_modules/vitest/vitest.mjs run tests/local-dev-ux.test.ts tests/cli.test.ts tests/authored-flow.test.ts tests/authored-flow-lifecycle-executor.test.ts + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-dev-ux.test.ts (7 tests) 72ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 567ms + ✓ tests/authored-flow.test.ts (24 tests) 672ms + ✓ tests/cli.test.ts (63 tests) 4134ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 480ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 308ms + ✓ flows check CLI > passes all three canonical ladder flows and prints their resolved CLI 540ms + ✓ flows check CLI > refuses cli-unauthenticated.flow.yaml with typed kind cli_unauthenticated and exit 2 396ms + + Test Files 4 passed (4) + Tests 121 passed (121) + Start at 15:07:12 + Duration 9.89s (transform 1.48s, setup 0ms, collect 12.60s, tests 5.45s, environment 4ms, prepare 2.63s) + diff --git a/docs/evidence/ws13/launcher-before-fix.txt b/docs/evidence/ws13/launcher-before-fix.txt new file mode 100644 index 00000000..2a8fbe08 --- /dev/null +++ b/docs/evidence/ws13/launcher-before-fix.txt @@ -0,0 +1,4 @@ +$ cd /tmp/ws13-consumer/hello && npm start +npm notice run start +npm notice run flows run hello.flow.ts --local-agent --input '{}' +REFUSED [invalid_invocation] Usage: flows check [--json] flows run [--json] [--no-spawn] [--data-dir ] flows run [--json] [--no-spawn] [--data-dir ] --input flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] flows resume [--json] [--no-spawn] [--data-dir ] flows hn-monitor start [--data-dir ] [--poll-interval-ms ] diff --git a/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt b/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt new file mode 100644 index 00000000..c36ff0cc --- /dev/null +++ b/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt @@ -0,0 +1,40 @@ +$ cd packages/sdk +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ❯ tests/local-agent-live.test.ts (4 tests | 1 failed) 74821ms + × built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 30594ms + → WAITING [worker_lease] Run "01M20NWN9DGY8FEX5CCT77CBRW" step "agent-1" (agent) is running under a worker lease until 1788876869141. +: expected null to be +0 // Object.is equality + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 19438ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 16028ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 8649ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped +AssertionError: WAITING [worker_lease] Run "01M20NWN9DGY8FEX5CCT77CBRW" step "agent-1" (agent) is running under a worker lease until 1788876869141. +: expected null to be +0 // Object.is equality + +- Expected: +0 + ++ Received: +null + + ❯ tests/local-agent-live.test.ts:54:58 + 52| const f = fixture(); + 53| const result = f.invoke('--json'); + 54| expect(result.status, result.stderr + result.stdout).toBe(0); + | ^ + 55| expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, status… + 56| expect(readFileSync(f.marker, 'utf8')).toBe('hello'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 3 passed (4) + Start at 16:12:04 + Duration 156.08s (transform 27.33s, setup 0ms, collect 26.50s, tests 74.82s, environment 339ms, prepare 13.46s) + diff --git a/docs/evidence/ws13/local-agent-tests-final.txt b/docs/evidence/ws13/local-agent-tests-final.txt new file mode 100644 index 00000000..51775480 --- /dev/null +++ b/docs/evidence/ws13/local-agent-tests-final.txt @@ -0,0 +1,15 @@ +$ cd packages/sdk +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts (4 tests) 42350ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 31085ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 9612ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 946ms + + Test Files 1 passed (1) + Tests 4 passed (4) + Start at 16:22:23 + Duration 60.26s (transform 1.86s, setup 0ms, collect 2.35s, tests 42.35s, environment 1ms, prepare 1.99s) + diff --git a/docs/evidence/ws13/local-agent-tests-first-attempt.txt b/docs/evidence/ws13/local-agent-tests-first-attempt.txt new file mode 100644 index 00000000..a6ef0a99 --- /dev/null +++ b/docs/evidence/ws13/local-agent-tests-first-attempt.txt @@ -0,0 +1,79 @@ +$ cd packages/sdk && RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ❯ tests/local-agent-live.test.ts (4 tests | 3 failed) 118611ms + × built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 30739ms + → WAITING [worker_lease] Run "01M20JQAF8A3T7CN0MVJVTNMPC" step "agent-1" (agent) is running under a worker lease until 1788873548470. +: expected null to be +0 // Object.is equality + × built CLI local agent against a real daemon > renders actual agent completion in text output 35168ms + → expected null to be +0 // Object.is equality + × built CLI local agent against a real daemon > returns a failed run when the agent process fails 34277ms + → expected null to be 1 // Object.is equality + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 18385ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped +AssertionError: WAITING [worker_lease] Run "01M20JQAF8A3T7CN0MVJVTNMPC" step "agent-1" (agent) is running under a worker lease until 1788873548470. +: expected null to be +0 // Object.is equality + +- Expected: +0 + ++ Received: +null + + ❯ tests/local-agent-live.test.ts:46:58 + 44| const f = fixture(); + 45| const result = f.invoke('--json'); + 46| expect(result.status, result.stderr + result.stdout).toBe(0); + | ^ + 47| expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, status… + 48| expect(readFileSync(f.marker, 'utf8')).toBe('hello'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/3]⎯ + + FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > renders actual agent completion in text output +AssertionError: expected null to be +0 // Object.is equality + +- Expected: +0 + ++ Received: +null + + ❯ tests/local-agent-live.test.ts:53:58 + 51| it('renders actual agent completion in text output', () => { + 52| const result = fixture().invoke(); + 53| expect(result.status, result.stderr + result.stdout).toBe(0); + | ^ + 54| expect(result.stderr).toContain('✓ agent-1 (agent) [agent: complet… + 55| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ + + FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > returns a failed run when the agent process fails +AssertionError: expected null to be 1 // Object.is equality + +- Expected: +1 + ++ Received: +null + + ❯ tests/local-agent-live.test.ts:58:58 + 56| it('returns a failed run when the agent process fails', () => { + 57| const result = fixture(7).invoke(); + 58| expect(result.status, result.stderr + result.stdout).toBe(1); + | ^ + 59| expect(result.stderr).toContain('✗ agent-1'); + 60| expect(result.stderr).not.toContain('[agent: completed]'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ + + Test Files 1 failed (1) + Tests 3 failed | 1 passed (4) + Start at 15:18:04 + Duration 129.86s (transform 1.30s, setup 0ms, collect 569ms, tests 118.61s, environment 0ms, prepare 2.13s) + diff --git a/docs/evidence/ws13/local-agent-tests.txt b/docs/evidence/ws13/local-agent-tests.txt new file mode 100644 index 00000000..5edc4737 --- /dev/null +++ b/docs/evidence/ws13/local-agent-tests.txt @@ -0,0 +1,15 @@ +$ cd packages/sdk && RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts (4 tests) 57107ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 11808ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 10853ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 27030ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 7415ms + + Test Files 1 passed (1) + Tests 4 passed (4) + Start at 15:28:46 + Duration 88.57s (transform 13.19s, setup 0ms, collect 4.58s, tests 57.11s, environment 0ms, prepare 12.36s) + diff --git a/docs/evidence/ws13/record.py b/docs/evidence/ws13/record.py index 71851759..1af73d36 100644 --- a/docs/evidence/ws13/record.py +++ b/docs/evidence/ws13/record.py @@ -42,7 +42,7 @@ text = decoder.decode(data) cast.write(json.dumps([round(time.monotonic() - started, 6), 'o', text]) + '\n') cast.flush() - transcript.write(text) + transcript.write(text.replace('\r\n', '\n')) transcript.flush() if timed_out: time.sleep(0.2) diff --git a/docs/evidence/ws13/research-typecheck.txt b/docs/evidence/ws13/research-typecheck.txt new file mode 100644 index 00000000..a6d5e50e --- /dev/null +++ b/docs/evidence/ws13/research-typecheck.txt @@ -0,0 +1,3 @@ +$ npm --prefix examples/research run typecheck +npm notice run typecheck +npm notice run ../../packages/sdk/node_modules/.bin/tsc -p tsconfig.json diff --git a/docs/evidence/ws13/test-types-final.txt b/docs/evidence/ws13/test-types-final.txt new file mode 100644 index 00000000..540c4d3c --- /dev/null +++ b/docs/evidence/ws13/test-types-final.txt @@ -0,0 +1,3 @@ +$ cd packages/sdk +$ node node_modules/typescript/bin/tsc -p tsconfig.tests.json +EXIT_CODE=0 diff --git a/docs/evidence/ws13/typechecks.txt b/docs/evidence/ws13/typechecks.txt new file mode 100644 index 00000000..da541029 --- /dev/null +++ b/docs/evidence/ws13/typechecks.txt @@ -0,0 +1,3 @@ +$ cd packages/sdk +$ node node_modules/typescript/bin/tsc --noEmit && node node_modules/typescript/bin/tsc -p tsconfig.type-tests.json && node node_modules/typescript/bin/tsc -p tsconfig.tests.json +EXIT_CODE=0 diff --git a/packages/sdk/tests/local-agent-live.test.ts b/packages/sdk/tests/local-agent-live.test.ts index 7491a8bd..e0c063e2 100644 --- a/packages/sdk/tests/local-agent-live.test.ts +++ b/packages/sdk/tests/local-agent-live.test.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -8,6 +8,14 @@ const roots: string[] = []; const sdk = resolve('.'); const cli = join(sdk, 'dist/cli.js'); const wrapperHelper = resolve('../../testdata/preflight/wrapper-session.mjs'); +// Ask the existing build wrapper for its target directory. A temp fixture's +// cwd cannot discover the checkout, and test:prep's child-shell exports do not +// survive into vitest. Do not select another worktree's most recent binary. +const relayflowd = process.env['RELAYFLOWD_BIN'] ?? join(JSON.parse(execFileSync('sh', [ + resolve('../../ops/cargo.sh'), 'metadata', '--format-version=1', '--no-deps', '--locked', '--offline', +], { cwd: resolve('../../kernel'), encoding: 'utf8', + env: { ...process.env, RELAYFLOWS_NO_TOOLCHAIN_INSTALL: '1' }, +})).target_directory, 'debug', 'relayflowd'); afterEach(() => { for (const root of roots.splice(0)) { const connection = join(root, 'data/connection.json'); @@ -34,9 +42,12 @@ function fixture(exitCode = 0, workspace?: string) { writeFileSync(join(root, 'flows.json'), JSON.stringify({ cli: wrapper })); writeFileSync(join(root, 'package.json'), '{"type":"module"}'); writeFileSync(join(root, 'hello.flow.ts'), `import { flow } from '@relayflows/surface';\nexport default flow('hello', async f => { await f.agent('greeter', ${JSON.stringify({ task: 'hello', ...(workspace ? { workspace } : {}) })}); f.done('success'); });\n`); + // Bound a stuck fixture process, allowing startup/preflight before the + // kernel's independently enforced worker lease. UX timing is measured by + // the separate empty-cache cold-start transcript, not this cleanup ceiling. return { root, marker, invoke: (...flags: string[]) => spawnSync(process.execPath, [cli, 'run', 'hello.flow.ts', '--input', '{}', '--local-agent', '--data-dir', join(root, 'data'), ...flags], - { cwd: root, encoding: 'utf8', timeout: 30000 }) }; + { cwd: root, encoding: 'utf8', timeout: 90000, env: { ...process.env, RELAYFLOWD_BIN: relayflowd } }) }; } describe('built CLI local agent against a real daemon', () => { From 3f225f78c8d6db7c44dc022bf26a2b7e31f741c7 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 16:32:11 +0200 Subject: [PATCH 06/16] docs: normalize captured transcript whitespace Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/README.md | 2 +- docs/evidence/ws13/container-tests.txt | 1 - docs/evidence/ws13/focused-tests.txt | 1 - .../evidence/ws13/local-agent-tests-30s-ceiling.txt | 5 ++--- docs/evidence/ws13/local-agent-tests-final.txt | 1 - .../ws13/local-agent-tests-first-attempt.txt | 13 ++++++------- docs/evidence/ws13/local-agent-tests.txt | 1 - 7 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md index 5d2050b2..561270cd 100644 --- a/docs/evidence/ws13/README.md +++ b/docs/evidence/ws13/README.md @@ -32,7 +32,7 @@ The recording uses the initial packed implementation plus the npm bin fix. Its agent step invokes the real installed Claude CLI. The host already had Node, provider authentication, and dependencies; this is **not** a cold-machine measurement. The recorded command does not include a clone or installation. -Text transcripts normalize terminal CRLF to LF; the `.cast` files retain the +Text transcripts normalize terminal CRLF to LF and trim trailing whitespace; the `.cast` files retain the captured terminal bytes and elapsed timestamps. The functional CLI fixture has a 90-second cleanup ceiling. Its original diff --git a/docs/evidence/ws13/container-tests.txt b/docs/evidence/ws13/container-tests.txt index 705ee106..1827c4a8 100644 --- a/docs/evidence/ws13/container-tests.txt +++ b/docs/evidence/ws13/container-tests.txt @@ -519,4 +519,3 @@ Error: The service is no longer running Tests no tests Start at 13:31:27 Duration 7.95s (transform 609ms, setup 0ms, collect 0ms, tests 0ms, environment 30ms, prepare 8.96s) - diff --git a/docs/evidence/ws13/focused-tests.txt b/docs/evidence/ws13/focused-tests.txt index 039d3ce0..f5bac76a 100644 --- a/docs/evidence/ws13/focused-tests.txt +++ b/docs/evidence/ws13/focused-tests.txt @@ -15,4 +15,3 @@ $ cd packages/sdk && node node_modules/vitest/vitest.mjs run tests/local-dev-ux. Tests 121 passed (121) Start at 15:07:12 Duration 9.89s (transform 1.48s, setup 0ms, collect 12.60s, tests 5.45s, environment 4ms, prepare 2.63s) - diff --git a/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt b/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt index c36ff0cc..d1001da9 100644 --- a/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt +++ b/docs/evidence/ws13/local-agent-tests-30s-ceiling.txt @@ -17,10 +17,10 @@ $ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwi AssertionError: WAITING [worker_lease] Run "01M20NWN9DGY8FEX5CCT77CBRW" step "agent-1" (agent) is running under a worker lease until 1788876869141. : expected null to be +0 // Object.is equality -- Expected: +- Expected: 0 -+ Received: ++ Received: null ❯ tests/local-agent-live.test.ts:54:58 @@ -37,4 +37,3 @@ null Tests 1 failed | 3 passed (4) Start at 16:12:04 Duration 156.08s (transform 27.33s, setup 0ms, collect 26.50s, tests 74.82s, environment 339ms, prepare 13.46s) - diff --git a/docs/evidence/ws13/local-agent-tests-final.txt b/docs/evidence/ws13/local-agent-tests-final.txt index 51775480..0967851a 100644 --- a/docs/evidence/ws13/local-agent-tests-final.txt +++ b/docs/evidence/ws13/local-agent-tests-final.txt @@ -12,4 +12,3 @@ $ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwi Tests 4 passed (4) Start at 16:22:23 Duration 60.26s (transform 1.86s, setup 0ms, collect 2.35s, tests 42.35s, environment 1ms, prepare 1.99s) - diff --git a/docs/evidence/ws13/local-agent-tests-first-attempt.txt b/docs/evidence/ws13/local-agent-tests-first-attempt.txt index a6ef0a99..d240c786 100644 --- a/docs/evidence/ws13/local-agent-tests-first-attempt.txt +++ b/docs/evidence/ws13/local-agent-tests-first-attempt.txt @@ -18,10 +18,10 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relay AssertionError: WAITING [worker_lease] Run "01M20JQAF8A3T7CN0MVJVTNMPC" step "agent-1" (agent) is running under a worker lease until 1788873548470. : expected null to be +0 // Object.is equality -- Expected: +- Expected: 0 -+ Received: ++ Received: null ❯ tests/local-agent-live.test.ts:46:58 @@ -37,10 +37,10 @@ null FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > renders actual agent completion in text output AssertionError: expected null to be +0 // Object.is equality -- Expected: +- Expected: 0 -+ Received: ++ Received: null ❯ tests/local-agent-live.test.ts:53:58 @@ -56,10 +56,10 @@ null FAIL tests/local-agent-live.test.ts > built CLI local agent against a real daemon > returns a failed run when the agent process fails AssertionError: expected null to be 1 // Object.is equality -- Expected: +- Expected: 1 -+ Received: ++ Received: null ❯ tests/local-agent-live.test.ts:58:58 @@ -76,4 +76,3 @@ null Tests 3 failed | 1 passed (4) Start at 15:18:04 Duration 129.86s (transform 1.30s, setup 0ms, collect 569ms, tests 118.61s, environment 0ms, prepare 2.13s) - diff --git a/docs/evidence/ws13/local-agent-tests.txt b/docs/evidence/ws13/local-agent-tests.txt index 5edc4737..1447b955 100644 --- a/docs/evidence/ws13/local-agent-tests.txt +++ b/docs/evidence/ws13/local-agent-tests.txt @@ -12,4 +12,3 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relay Tests 4 passed (4) Start at 15:28:46 Duration 88.57s (transform 13.19s, setup 0ms, collect 4.58s, tests 57.11s, environment 0ms, prepare 12.36s) - From dcde07212a03e5eebaba1afaf3968fafc9edcc18 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 20:42:06 +0200 Subject: [PATCH 07/16] fix: show research preflight activity while provider checks run Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- examples/research/shims/headless.ts | 8 +++++++- examples/research/shims/run.ts | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/research/shims/headless.ts b/examples/research/shims/headless.ts index e82ef917..0a67568e 100644 --- a/examples/research/shims/headless.ts +++ b/examples/research/shims/headless.ts @@ -383,7 +383,11 @@ function probe(bin: string, args: string[], timeout: number): Promise { +export async function preflightHeadless( + targets: readonly PreflightTarget[], + binaries?: HeadlessBinaries, + onProbe?: (label: string, timeoutMs: number) => void, +): Promise { const findings: PreflightFinding[] = []; const seen = new Set(); const authChecked = new Set(); @@ -409,6 +413,7 @@ export async function preflightHeadless(targets: readonly PreflightTarget[], bin if (auth && !authChecked.has(cli)) { authChecked.add(cli); const label = `${cli} ${auth.join(" ")}`; + onProbe?.(label, 10_000); const result = await probe(bin, auth, 10_000); const early = classify(label, result); if (early) { findings.push(early); continue; } @@ -421,6 +426,7 @@ export async function preflightHeadless(targets: readonly PreflightTarget[], bin // 2. live round-trip with the declared model const args = roundTripArgs(cli, model); const label = `${cli} round-trip${model ? ` with model ${model}` : ""}`; + onProbe?.(label, 90_000); const result = await probe(bin, args, 90_000); const early = classify(label, result); if (early) { findings.push(early); continue; } diff --git a/examples/research/shims/run.ts b/examples/research/shims/run.ts index adcafffa..886231bc 100644 --- a/examples/research/shims/run.ts +++ b/examples/research/shims/run.ts @@ -228,7 +228,9 @@ export async function main(argv: readonly string[], deps: MainDeps): Promise a.cli).filter((cli) => !isHeadlessCli(cli)); if (unknown.length > 0) throw new Refused(`no headless adapter for cli: ${unknown.join(", ")}`); const targets = agents.filter((a) => isHeadlessCli(a.cli)).map((a) => ({ cli: a.cli, model: a.model })) as PreflightTarget[]; - const findings = await preflightHeadless(targets, deps.binaries); + const findings = await preflightHeadless(targets, deps.binaries, (label, timeoutMs) => { + deps.stderr(`research: checking ${label} (timeout ${timeoutMs / 1000}s)`); + }); if (findings.length > 0) { throw new Refused(findings.map((f) => `[${f.kind}] ${f.message}`).join("\n")); } From f14645096b0028f4464f9c41ecfed1ad87fd15f1 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 20:49:04 +0200 Subject: [PATCH 08/16] docs: report gallery refusals and move timing to accepted evidence Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- README.md | 15 +- docs/evidence/ws13/README.md | 41 ++-- .../gallery-dependency-upgrade-bot.txt | 12 ++ .../followup/gallery-pr-review-pipeline.txt | 12 ++ .../ws13/followup/gallery-research.txt | 16 ++ .../ws13/followup/gallery-results.json | 20 ++ .../evidence/ws13/followup/research-tests.txt | 176 ++++++++++++++++++ .../ws13/followup/research-typecheck.txt | 8 + docs/evidence/ws13/followup/run-gallery.py | 63 +++++++ .../ws13/gallery-social-post-pipeline.txt | 6 - examples/README.md | 49 ++--- examples/dependency-upgrade-bot/README.md | 11 +- examples/pr-review-pipeline/README.md | 17 +- examples/research/README.md | 6 +- examples/social-post-pipeline/README.md | 8 +- 15 files changed, 391 insertions(+), 69 deletions(-) create mode 100644 docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt create mode 100644 docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt create mode 100644 docs/evidence/ws13/followup/gallery-research.txt create mode 100644 docs/evidence/ws13/followup/gallery-results.json create mode 100644 docs/evidence/ws13/followup/research-tests.txt create mode 100644 docs/evidence/ws13/followup/research-typecheck.txt create mode 100644 docs/evidence/ws13/followup/run-gallery.py delete mode 100644 docs/evidence/ws13/gallery-social-post-pipeline.txt diff --git a/README.md b/README.md index f250eafc..b04b2634 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,13 @@ npm start **Release status:** `create-flow` is not published yet. The commands above are the intended released entry point; use the [candidate artifact procedure](docs/evidence/ws13/README.md) -to try this branch. A clean-machine first-agent run under 60 seconds has not -been established. The [clone + deterministic starter measurement](docs/evidence/ws13/cold-clone-direct.txt) -completed in 49.975 seconds in a fresh Linux container with Node and Git -provisioned before the timer; it does not measure an agent invocation. +to try this branch. The [clone + deterministic starter measurement](docs/evidence/ws13/cold-clone-direct.txt) +completed in **49.975 seconds** in a fresh Linux container with Node and Git +provisioned before the timer. The [real Claude command](docs/evidence/ws13/agent-run.txt) +completed in **132.637 seconds** on an authenticated development host; its +agent step took 28.95 seconds, including the provider round trip. The total +also includes CLI startup and preflight, whose costs were not separately +measured. The agent starter requires Node 22.18+ and an installed, authenticated Claude CLI. Use `--cli codex` to select Codex, or `--template deterministic` for a @@ -61,8 +64,8 @@ See the [example gallery and individual run results](examples/README.md). [Watch the captured agent run](docs/evidence/ws13/agent-run.cast) ([text transcript](docs/evidence/ws13/agent-run.txt)). -The larger examples currently refuse or time out; their intended budgets, -artifact gates, workspace restrictions, and human approvals are preserved. +The gallery reports each requested example as PASS or BLOCKED, with its +command, output, timing, and any capability or provider requirement still missing. Give your agent a skill to write a flow: diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md index 561270cd..3984a1ea 100644 --- a/docs/evidence/ws13/README.md +++ b/docs/evidence/ws13/README.md @@ -1,10 +1,24 @@ # WS-13 local development evidence -**Acceptance is incomplete.** This branch implements SDK scaffolding, terminal -progress, an opt-in local agent worker, and the npm launcher fix. It does not -establish a clean-machine first-agent run under 60 seconds or a green gallery. - -## Captured results +**Timing is accepted by Khaliq's ruling, not a blocker.** The measured cold +deterministic loop is 49.975s; the existing-host real Claude command is +132.637s. Its agent step is 28.95s including the provider round trip; the +remaining startup/preflight time was not separately measured, so the evidence +does not attribute most of the total to the provider. + +**Named handoffs:** the release-gate owner must register `create-flow` in +versioning/packaging/publishing. The review-swarm/CI owner must restore fresh +maintainability, history and structure transcripts; all three are missing and +there is no independent review signoff. Neither handoff is a reason to keep +the PR in draft once the gallery results are reported. No publishing work or +Cloud run-publication API is part of this follow-up. + +The [current three-entry gallery](../../../examples/README.md) supersedes the +initial invocation results below. Research now reports each provider probe +and timeout on stderr. [Research regression tests](followup/research-tests.txt) +and [typecheck](followup/research-typecheck.txt) contain the commands/output. + +## Initial captured results | Check | Result | Evidence | |---|---|---| @@ -21,12 +35,11 @@ establish a clean-machine first-agent run under 60 seconds or a green gallery. | Linux container test runner | esbuild Go runtime crashed under amd64 emulation before collecting tests | [Command, script and full output](container-tests.txt) | | Research typecheck after correcting its compiler path | No type errors reported | [Command and output](research-typecheck.txt) | -All four existing gallery entries were invoked separately. See the -[gallery table](../../../examples/README.md) for individual timings and literal -commands. Three refuse the unsupported `budget` header. Research reached the -outer 150-second verification limit without captured output; this does not -identify whether its preflight or execution was responsible. No gallery flow -was weakened or represented as successful. +The [gallery table](../../../examples/README.md) reports the three requested +entries individually. Unsupported budget headers remain a capability-owner +handoff. The initial research attempt reached an outer 150-second limit with +no captured output; the follow-up now exposes preflight progress and captures +the shim's own failure or success result. No gallery declaration was weakened. The recording uses the initial packed implementation plus the npm bin fix. Its agent step invokes the real installed Claude CLI. The host already had @@ -115,6 +128,6 @@ and publishing, and to `scripts/pack-release.mjs`, which currently refuses that package name. That script also requires the legacy runtime executable. Those gates were not edited. No package was published and no merge is allowed. -Veto tools were not exposed in this session. Several status DMs encountered server/overload errors. A later status to -`session-thread-rollout` received queue receipt `223075216797716480`; reading -was not confirmed. A coordination reply to WS-14 timed out. +Veto tools were not exposed. Relay queue receipts did not establish delivery; +the coordinator confirmed the original handoff never arrived. The PR and this +evidence directory are the durable handoff. diff --git a/docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt b/docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt new file mode 100644 index 00000000..c5b7f1a1 --- /dev/null +++ b/docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt @@ -0,0 +1,12 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-followup/node_modules/relayflows/bin/flows.js run examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts --local-agent --input '{}' --data-dir /tmp/ws13-followup-upgrade-daemon +OUTER_TIMEOUT_SECONDS=120 +(node:57800) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///private/tmp/ws13-gallery-followup/examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts is not specified and it doesn't parse as CommonJS. +Reparsing as ES module because module syntax was detected. This incurs a performance overhead. +To eliminate this warning, add "type": "module" to /private/tmp/ws13-gallery-followup/package.json. +(Use `node --trace-warnings ...` to show where the warning was created) +REFUSED [invalid_spec] unsupported_header: flow "dependency-upgrade-bot" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=1.224 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt b/docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt new file mode 100644 index 00000000..4f900298 --- /dev/null +++ b/docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt @@ -0,0 +1,12 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-followup/node_modules/relayflows/bin/flows.js run examples/pr-review-pipeline/pr-review-pipeline.flow.ts --local-agent --input '{"diffRange":"origin/main...HEAD"}' --data-dir /tmp/ws13-followup-review-daemon +OUTER_TIMEOUT_SECONDS=120 +(node:57861) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///private/tmp/ws13-gallery-followup/examples/pr-review-pipeline/pr-review-pipeline.flow.ts is not specified and it doesn't parse as CommonJS. +Reparsing as ES module because module syntax was detected. This incurs a performance overhead. +To eliminate this warning, add "type": "module" to /private/tmp/ws13-gallery-followup/package.json. +(Use `node --trace-warnings ...` to show where the warning was created) +REFUSED [invalid_spec] unsupported_header: flow "pr-review-pipeline" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=0.252 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/gallery-research.txt b/docs/evidence/ws13/followup/gallery-research.txt new file mode 100644 index 00000000..b578de79 --- /dev/null +++ b/docs/evidence/ws13/followup/gallery-research.txt @@ -0,0 +1,16 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node --experimental-strip-types examples/research/shims/run.ts --slug ws13-followup --question 'Compare durable step journals with deterministic replay. Keep every report under 200 words.' --timeout-minutes 3 --runs-dir /tmp/ws13-research-followup-runs +OUTER_TIMEOUT_SECONDS=780 +research: checking claude auth status (timeout 10s) +research: checking claude round-trip with model sonnet (timeout 90s) +research: checking codex login status (timeout 10s) +research: checking codex round-trip with model gpt-5.6-sol (timeout 90s) +research: checking grok round-trip with model grok-4.6 (timeout 90s) +research: checking claude round-trip with model opus (timeout 90s) +research: run dir /tmp/ws13-research-followup-runs/2026-09-08-ws13-followup +research: killed 2 still-running agent step(s) after failure +FAILED step "grok" completionReason: timeout — agent step "grok" failed (timeout): exceeded 180000ms; transcript at /tmp/ws13-research-followup-runs/2026-09-08-ws13-followup/grok/grok.log + +EXIT_CODE=1 +ELAPSED_SECONDS=217.375 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/gallery-results.json b/docs/evidence/ws13/followup/gallery-results.json new file mode 100644 index 00000000..68d4378e --- /dev/null +++ b/docs/evidence/ws13/followup/gallery-results.json @@ -0,0 +1,20 @@ +[ + { + "example": "dependency-upgrade-bot", + "exitCode": 2, + "elapsedSeconds": 1.224, + "timedOut": false + }, + { + "example": "pr-review-pipeline", + "exitCode": 2, + "elapsedSeconds": 0.252, + "timedOut": false + }, + { + "example": "research", + "exitCode": 1, + "elapsedSeconds": 217.375, + "timedOut": false + } +] diff --git a/docs/evidence/ws13/followup/research-tests.txt b/docs/evidence/ws13/followup/research-tests.txt new file mode 100644 index 00000000..f5fb440c --- /dev/null +++ b/docs/evidence/ws13/followup/research-tests.txt @@ -0,0 +1,176 @@ +$ export PATH=/tmp/ws13-toolchain:$PATH +$ node --experimental-strip-types --test examples/research/tests/*.test.ts +TAP version 13 +# Subtest: claude: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded +ok 1 - claude: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded + --- + duration_ms: 356.488667 + type: 'test' + ... +# Subtest: codex: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded +ok 2 - codex: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded + --- + duration_ms: 195.035083 + type: 'test' + ... +# Subtest: grok: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded +ok 3 - grok: artifacts are the top-level files the agent wrote, minus the shim's own files; usage, session, trajectory recorded + --- + duration_ms: 160.548291 + type: 'test' + ... +# Subtest: the task never travels on argv: the stub dumps its argv and the brief text is not in it +ok 4 - the task never travels on argv: the stub dumps its argv and the brief text is not in it + --- + duration_ms: 218.826542 + type: 'test' + ... +# Subtest: no declared model: RELAYFLOW_MODEL is ABSENT in the child even if the host has it set +ok 5 - no declared model: RELAYFLOW_MODEL is ABSENT in the child even if the host has it set + --- + duration_ms: 163.477541 + type: 'test' + ... +# Subtest: a stray top-level file written by the agent IS an artifact, so the gate can see undeclared writes inside the workspace +ok 6 - a stray top-level file written by the agent IS an artifact, so the gate can see undeclared writes inside the workspace + --- + duration_ms: 286.867917 + type: 'test' + ... +# Subtest: non-zero exit, empty final message, and missing usage are each worker_error, never an empty success +ok 7 - non-zero exit, empty final message, and missing usage are each worker_error, never an empty success + --- + duration_ms: 297.109208 + type: 'test' + ... +# Subtest: a CLI that outlives its timeout is killed and reported as timeout +ok 8 - a CLI that outlives its timeout is killed and reported as timeout + --- + duration_ms: 572.231708 + type: 'test' + ... +# Subtest: preflight is per (cli, model): auth failure is cli_unauthenticated; an unresolvable declared model is model_unavailable; a silent round-trip is not ready; a missing binary is cli_missing +ok 9 - preflight is per (cli, model): auth failure is cli_unauthenticated; an unresolvable declared model is model_unavailable; a silent round-trip is not ready; a missing binary is cli_missing + --- + duration_ms: 1112.544625 + type: 'test' + ... +# Subtest: workspace dir must be absolute and normalized +ok 10 - workspace dir must be absolute and normalized + --- + duration_ms: 0.793625 + type: 'test' + ... +# Subtest: when one lane fails, the still-running sibling lanes are killed instead of spending until their timeout +ok 11 - when one lane fails, the still-running sibling lanes are killed instead of spending until their timeout + --- + duration_ms: 543.095709 + type: 'test' + ... +# Subtest: main(): every refusal is exit 2 and happens before anything is created; a fake run is exit 0 +ok 12 - main(): every refusal is exit 2 and happens before anything is created; a fake run is exit 0 + --- + duration_ms: 290.977917 + type: 'test' + ... +# Subtest: a lane that has not spawned yet when a sibling fails is aborted, never started +ok 13 - a lane that has not spawned yet when a sibling fails is aborted, never started + --- + duration_ms: 39.001584 + type: 'test' + ... +# Subtest: a same-size rewrite of an existing file IS an artifact (content, not size or mtime, decides) +ok 14 - a same-size rewrite of an existing file IS an artifact (content, not size or mtime, decides) + --- + duration_ms: 164.3295 + type: 'test' + ... +# Subtest: SIGINT to the entry point stops every live agent (exit 130), instead of orphaning permission-bypassed CLIs +ok 15 - SIGINT to the entry point stops every live agent (exit 130), instead of orphaning permission-bypassed CLIs + --- + duration_ms: 730.086333 + type: 'test' + ... +# Subtest: preflight: a probe terminated by a signal is probe_failed, not cli_unauthenticated +ok 16 - preflight: a probe terminated by a signal is probe_failed, not cli_unauthenticated + --- + duration_ms: 277.96675 + type: 'test' + ... +# Subtest: a symlinked entrypoint still runs main (realpath comparison), exit 2 on a bad argument +ok 17 - a symlinked entrypoint still runs main (realpath comparison), exit 2 on a bad argument + --- + duration_ms: 102.591459 + type: 'test' + ... +# Subtest: three lanes are dispatched concurrently, then one synthesis +ok 18 - three lanes are dispatched concurrently, then one synthesis + --- + duration_ms: 1.616792 + type: 'test' + ... +# Subtest: a lane that writes no report fails its gate and synthesis never runs +ok 19 - a lane that writes no report fails its gate and synthesis never runs + --- + duration_ms: 0.727875 + type: 'test' + ... +# Subtest: the header pins every agent's CLI and model exactly; a changed or dropped model fails here +ok 20 - the header pins every agent's CLI and model exactly; a changed or dropped model fails here + --- + duration_ms: 0.141 + type: 'test' + ... +# Subtest: every failure class reports a completionReason from COMPLETION_REASONS, and the set is exactly the documented one +ok 21 - every failure class reports a completionReason from COMPLETION_REASONS, and the set is exactly the documented one + --- + duration_ms: 0.138917 + type: 'test' + ... +# Subtest: headless invocations use structured output and never put the task on argv +ok 22 - headless invocations use structured output and never put the task on argv + --- + duration_ms: 0.1995 + type: 'test' + ... +# Subtest: parseHeadless reads final text, usage, session and subagents from each CLI's verified shape +ok 23 - parseHeadless reads final text, usage, session and subagents from each CLI's verified shape + --- + duration_ms: 0.450667 + type: 'test' + ... +# Subtest: a CLI that exits without a readable, non-empty final message and a usage record is unreadable, not an empty success +ok 24 - a CLI that exits without a readable, non-empty final message and a usage record is unreadable, not an empty success + --- + duration_ms: 0.256125 + type: 'test' + ... +# Subtest: usage counters must be finite numbers: missing or string-valued input/output tokens are a parse error, absent cache counters are 0 +ok 25 - usage counters must be finite numbers: missing or string-valued input/output tokens are a parse error, absent cache counters are 0 + --- + duration_ms: 0.257959 + type: 'test' + ... +# Subtest: a gate registered after the step was awaited throws instead of silently never running +ok 26 - a gate registered after the step was awaited throws instead of silently never running + --- + duration_ms: 0.322417 + type: 'test' + ... +# Subtest: a lane that reports no usage fails its budget gate +ok 27 - a lane that reports no usage fails its budget gate + --- + duration_ms: 0.438083 + type: 'test' + ... +1..27 +# tests 27 +# suites 0 +# pass 27 +# fail 0 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 5695.376209 + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/research-typecheck.txt b/docs/evidence/ws13/followup/research-typecheck.txt new file mode 100644 index 00000000..d6308bbf --- /dev/null +++ b/docs/evidence/ws13/followup/research-typecheck.txt @@ -0,0 +1,8 @@ +$ export PATH=/tmp/ws13-toolchain:$PATH +$ npm --prefix examples/research run typecheck + +> typecheck +> ../../packages/sdk/node_modules/.bin/tsc -p tsconfig.json + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/run-gallery.py b/docs/evidence/ws13/followup/run-gallery.py new file mode 100644 index 00000000..38679196 --- /dev/null +++ b/docs/evidence/ws13/followup/run-gallery.py @@ -0,0 +1,63 @@ +"""Capture each requested gallery invocation, including nonzero exits and timeouts. +Usage: python3 run-gallery.py /absolute/gallery-clone /absolute/evidence-directory [research-default] +""" +from pathlib import Path +import json +import os +import shlex +import signal +import subprocess +import sys +import time + +root, evidence = (Path(p).resolve() for p in sys.argv[1:3]) +evidence.mkdir(parents=True, exist_ok=True) +node = '/tmp/ws13-toolchain/node' +cli = str(root / 'node_modules/relayflows/bin/flows.js') +cases = [ + ('dependency-upgrade-bot', 120, [node, cli, 'run', + 'examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts', '--local-agent', + '--input', '{}', '--data-dir', '/tmp/ws13-followup-upgrade-daemon']), + ('pr-review-pipeline', 120, [node, cli, 'run', + 'examples/pr-review-pipeline/pr-review-pipeline.flow.ts', '--local-agent', + '--input', '{"diffRange":"origin/main...HEAD"}', + '--data-dir', '/tmp/ws13-followup-review-daemon']), + ('research', 780, [node, '--experimental-strip-types', 'examples/research/shims/run.ts', + '--slug', 'ws13-followup', '--question', + 'Compare durable step journals with deterministic replay. Keep every report under 200 words.', + '--timeout-minutes', '3', '--runs-dir', '/tmp/ws13-research-followup-runs']), +] +if sys.argv[3:] == ['research-default']: + command = cases[-1][2].copy() + command[command.index('ws13-followup')] = 'ws13-default-budget' + index = command.index('--timeout-minutes') + del command[index:index + 2] + cases = [('research', 3900, command)] +results = [] +for name, timeout, command in cases: + with (evidence / f'gallery-{name}.txt').open('w') as output: + output.write(f'$ cd {shlex.quote(str(root))}\n$ {shlex.join(command)}\n') + output.write(f'OUTER_TIMEOUT_SECONDS={timeout}\n') + output.flush() + started = time.monotonic() + process = subprocess.Popen(command, cwd=root, + env={**os.environ, 'PATH': '/tmp/ws13-toolchain:' + os.environ['PATH']}, + stdout=output, stderr=subprocess.STDOUT, start_new_session=True) + timed_out = False + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + code = 124 + elapsed = round(time.monotonic() - started, 3) + output.write(f'\nEXIT_CODE={code}\nELAPSED_SECONDS={elapsed:.3f}\nTIMED_OUT={timed_out}\n') + result = {'example': name, 'exitCode': code, 'elapsedSeconds': elapsed, 'timedOut': timed_out} + results.append(result) + (evidence / 'gallery-results.json').write_text(json.dumps(results, indent=2) + '\n') + print(json.dumps(result), flush=True) diff --git a/docs/evidence/ws13/gallery-social-post-pipeline.txt b/docs/evidence/ws13/gallery-social-post-pipeline.txt deleted file mode 100644 index 48daddc3..00000000 --- a/docs/evidence/ws13/gallery-social-post-pipeline.txt +++ /dev/null @@ -1,6 +0,0 @@ -$ cd /tmp/ws13-gallery -$ node /tmp/ws13-consumer/hello/node_modules/relayflows/bin/flows.js run examples/social-post-pipeline/social-post-pipeline.flow.ts --local-agent --input '{"brand": "Relayflows", "topic": "durable steps", "approver": "local-reviewer"}' --data-dir /tmp/ws13-gallery-socia -REFUSED [invalid_spec] unsupported_header: flow "social-post-pipeline" uses unsupported header fields: budget - -EXIT_CODE=2 -ELAPSED_SECONDS=6.698 diff --git a/examples/README.md b/examples/README.md index d1c06c37..340a9e21 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,30 +1,31 @@ -# Example gallery +# Example gallery status -These four examples describe larger flows. **None has a green end-to-end -result in the WS-13 verification run.** The table records time until refusal -or the verification timeout, not time to successful completion. +The three entries below are the requested WS-13 gallery scope. They are +advanced examples, not a promise that every surface feature is executable. +For a working local starting point, use the [small agent starter](../README.md) +([recorded run](../docs/evidence/ws13/agent-run.txt)). -| Example | What it demonstrates | Observed result | Time | +| Example | Status | Observed result | Elapsed | |---|---|---|---:| -| [dependency-upgrade-bot](dependency-upgrade-bot/) | Upgrade → independent verification → PR | Refused: unsupported `budget` header, exit 2 | [6.596s](../docs/evidence/ws13/gallery-dependency-upgrade-bot.txt) | -| [pr-review-pipeline](pr-review-pipeline/) | Three review lenses → consensus | Refused: unsupported `budget` header, exit 2 | [4.990s](../docs/evidence/ws13/gallery-pr-review-pipeline.txt) | -| [social-post-pipeline](social-post-pipeline/) | Research → draft → fact-check → graphic → human approval | Refused: unsupported `budget` header, exit 2 | [6.698s](../docs/evidence/ws13/gallery-social-post-pipeline.txt) | -| [research](research/) | Claude/Codex/Grok fan-out → synthesis via existing shims | Verification timed out with no output captured | [150.067s](../docs/evidence/ws13/gallery-research.txt) | +| [dependency-upgrade-bot](dependency-upgrade-bot/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [1.224s](../docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt) | +| [pr-review-pipeline](pr-review-pipeline/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.252s](../docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt) | +| [research](research/) | **RETRY IN PROGRESS** | Preflight passed; first run failed at the supplied 3-minute step limit. Retrying with the documented default budget | [217.375s for the first attempt](../docs/evidence/ws13/followup/gallery-research.txt) | -The first three were invoked individually with the packed candidate CLI, -`--local-agent`, explicit inputs, and separate local daemon directories. -Research was invoked through its documented shim with a one-minute per-step -bound and a 150-second outer verification bound; the latter does not establish -whether preflight or execution was responsible. Exact commands and captured -output are linked in the table. These are runs on an existing development -host, not a clean machine. +Each link contains the literal command, captured output, exit code and timing. +These are individual runs from a separate clone on an authenticated macOS +host, against the packed candidate CLI. Research uses its documented source +shim. These timings are not clean-machine measurements. -Removing the unsupported headers or weakening the examples' artifact gates -would change what they promise. Further runtime work is required before these -can be advertised as runnable. Social-post-pipeline additionally depends on -`f.human`; workspace permission annotations and postfix gates also remain -unsupported by the authored executor. +**Dependency-upgrade-bot and pr-review-pipeline need the SDK/kernel capability +owner.** Their authored budgets are currently rejected. Their postfix artifact +gates and workspace permission declarations also require runtime support. +Removing those requirements would weaken what the examples promise; this +branch leaves them intact. The local agent worker handles stream-only steps +and cannot supply workspace isolation. -The research example's `npm run typecheck` command now uses the actual -`packages/sdk` compiler path. Its shim tests and typecheck are separate from -an end-to-end run. +**Research now prints provider preflight activity.** Each CLI/model probe names +its timeout on stderr, while stdout remains the final structured result. All +four model probes passed in the first follow-up run; the three-minute limit +then expired during research. That failure is not evidence of missing provider +authentication or an unsupported model. The earlier outer timeout is retained +in the [historical evidence](../docs/evidence/ws13/gallery-research.txt). diff --git a/examples/dependency-upgrade-bot/README.md b/examples/dependency-upgrade-bot/README.md index d7eb4f96..18ac4c11 100644 --- a/examples/dependency-upgrade-bot/README.md +++ b/examples/dependency-upgrade-bot/README.md @@ -1,5 +1,12 @@ # dependency-upgrade-bot +**BLOCKED — not runnable on the current authored executor.** The candidate +CLI refuses the `budget` header before any step runs (exit 2, 1.224s). +[Exact command and captured output](../../docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt). +The SDK/kernel capability owner must supply budget-header support, postfix +artifact gates, and the declared workspace behavior before this example can +be advertised as working. Its existing requirements remain intact. + **Like I'm 5:** A checklist notices a library is out of date. A robot tries upgrading it, but only in its own sandboxed corner where it can't break anything real. A *second*, completely separate robot — in its own sandbox @@ -46,8 +53,8 @@ first refusal is resolved. cd packages/surface && npm run typecheck:examples ``` -- `f.agent(...)` builds a real step but parks without a worker attached, - same as every other example in this repo today. +- `--local-agent` attaches a stream-only worker; it cannot provide the + workspace revision pins and isolation declared by this example. - **The sandbox isolation is declared, not enforced.** RFC-0001 Appendix A rule 1 (workspace-scoped permissions) is gate-8 kernel work; today nothing stops the `upgrader` step from reading `sandbox/verify/` if the underlying diff --git a/examples/pr-review-pipeline/README.md b/examples/pr-review-pipeline/README.md index 02b7a098..f60d85d0 100644 --- a/examples/pr-review-pipeline/README.md +++ b/examples/pr-review-pipeline/README.md @@ -1,5 +1,12 @@ # pr-review-pipeline +**BLOCKED — not runnable on the current authored executor.** The candidate +CLI refuses the `budget` header before any step runs (exit 2, 0.252s). +[Exact command and captured output](../../docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt). +The SDK/kernel capability owner must supply budget-header support, postfix +artifact gates, and the declared workspace behavior before this example can +be advertised as working. Its existing requirements remain intact. + **Like I'm 5:** Instead of one reviewer reading your whole pull request, three little reviewers each look for one thing — one only checks for security holes, one only checks for logic bugs, one only checks for slow @@ -29,9 +36,6 @@ mechanism — My Senior Dev's multi-agent PR review — in two layers: `looksLikeFalsePositiveDispute`) rather than just concatenating three reports into one. -Unlike the other two examples in this directory, this one never calls -`f.human` — nothing here needs it to make sense as a demonstration. - ## Status: refused before execution WS-13 invoked this example with the packed CLI and `--local-agent`. It @@ -44,7 +48,6 @@ first refusal is resolved. cd packages/surface && npm run typecheck:examples ``` -`f.agent(...)` builds a real step but parks without a worker attached, same -as every other example in this repo today. Everything else in this flow — -the fan-out, the gates, the reconciliation step — is otherwise ordinary use -of the shipped `@relayflows/surface` contract. +`--local-agent` attaches a stream-only worker. Budget headers, postfix gates +and workspace permission annotations are still refused by the authored +executor, even though the surface package can represent their types. diff --git a/examples/research/README.md b/examples/research/README.md index a2e19791..a805f867 100644 --- a/examples/research/README.md +++ b/examples/research/README.md @@ -42,7 +42,7 @@ node --experimental-strip-types examples/research/shims/run.ts \ Requires `claude`, `codex`, and `grok` on `PATH`, each authenticated. The entry point preflights every declared (CLI, model) pair before creating -anything: a cheap auth probe where one exists (`claude auth status`, +anything, printing each check and its timeout to stderr: a cheap auth probe where one exists (`claude auth status`, `codex login status`), then a live one-line round-trip with the declared model flag that must answer exactly `OK` (trimmed; "NOT OK" and "OK." fail). An authenticated CLI that cannot resolve a declared model (`opus` on a host that does not know the alias) is refused as @@ -79,8 +79,8 @@ npm --prefix examples/research test # node --experimental-strip-types - npm --prefix examples/research run typecheck # sdk's ./node_modules/.bin/tsc, not npx (npx would fetch an unrelated tsc and "pass") ``` -**These are not run by any gate.** This repository has no CI workflows and -`sdk`'s `npm test` is scoped to `sdk/`. The Safety properties below cite these +**These are not run by the current CI workflows.** The SDK's `npm test` is +scoped to `packages/sdk/`. The Safety properties below cite these tests as evidence; that evidence exists only when someone runs the two commands. Wiring `examples/*` into a runner is a follow-up. diff --git a/examples/social-post-pipeline/README.md b/examples/social-post-pipeline/README.md index 0e36a1cb..04ff0fa9 100644 --- a/examples/social-post-pipeline/README.md +++ b/examples/social-post-pipeline/README.md @@ -32,13 +32,7 @@ part isn't the agents — it's what each `.gate()` checks: - The human gate (`f.human`) is the last word. Everything upstream can pass every gate and the flow still won't publish without a yes. -## Status: refused before execution - -WS-13 invoked this example with the packed CLI and `--local-agent`. It -refused the unsupported `budget` header before entering the body. See the -[gallery](../README.md) for the exact command, output, and elapsed time. -The remaining limitations below describe what still needs to land after that -first refusal is resolved. +## Status: typechecks, does not run yet ```sh cd packages/surface && npm run typecheck:examples From 2479285c1c3762686df072ce402a8b4531a33461 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 21:15:45 +0200 Subject: [PATCH 09/16] fix: renew agent leases and capture completed research gallery run Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/README.md | 23 +- docs/evidence/ws13/followup/README.md | 63 +++ .../followup/default-budget/artifacts.json | 30 ++ .../default-budget/gallery-research.txt | 52 +++ .../default-budget/gallery-results.json | 8 + .../followup/default-budget/reports/claude.md | 59 +++ .../followup/default-budget/reports/codex.md | 29 ++ .../followup/default-budget/reports/grok.md | 188 ++++++++ .../default-budget/reports/synthesis.md | 420 ++++++++++++++++++ .../ws13/followup/final-sdk/artifact.json | 4 + .../gallery-dependency-upgrade-bot.txt | 8 + .../final-sdk/gallery-pr-review-pipeline.txt | 8 + .../followup/final-sdk/gallery-results.json | 14 + .../ws13/followup/heartbeat-abort.txt | 16 + .../evidence/ws13/followup/heartbeat-live.txt | 19 + .../ws13/followup/heartbeat-packed.txt | 15 + .../ws13/followup/heartbeat-tests.txt | 31 ++ .../ws13/followup/kernel-case-retry.txt | 19 + docs/evidence/ws13/followup/kernel-suite.txt | 70 +++ docs/evidence/ws13/followup/run-gallery.py | 15 +- .../evidence/ws13/followup/sdk-typechecks.txt | 4 + docs/evidence/ws13/stage-registry.mjs | 15 +- examples/README.md | 21 +- examples/dependency-upgrade-bot/README.md | 4 +- examples/pr-review-pipeline/README.md | 4 +- examples/research/README.md | 8 + packages/sdk/src/worker-cli.ts | 20 +- packages/sdk/src/worker-lease.ts | 73 +++ packages/sdk/src/worker.ts | 8 +- packages/sdk/src/wrapper-session.ts | 15 +- packages/sdk/tests/local-agent-live.test.ts | 13 +- packages/sdk/tests/worker-cli-abort.test.ts | 43 ++ packages/sdk/tests/worker-cli.test.ts | 6 +- packages/sdk/tests/worker-lease.test.ts | 105 +++++ 34 files changed, 1393 insertions(+), 37 deletions(-) create mode 100644 docs/evidence/ws13/followup/README.md create mode 100644 docs/evidence/ws13/followup/default-budget/artifacts.json create mode 100644 docs/evidence/ws13/followup/default-budget/gallery-research.txt create mode 100644 docs/evidence/ws13/followup/default-budget/gallery-results.json create mode 100644 docs/evidence/ws13/followup/default-budget/reports/claude.md create mode 100644 docs/evidence/ws13/followup/default-budget/reports/codex.md create mode 100644 docs/evidence/ws13/followup/default-budget/reports/grok.md create mode 100644 docs/evidence/ws13/followup/default-budget/reports/synthesis.md create mode 100644 docs/evidence/ws13/followup/final-sdk/artifact.json create mode 100644 docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt create mode 100644 docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt create mode 100644 docs/evidence/ws13/followup/final-sdk/gallery-results.json create mode 100644 docs/evidence/ws13/followup/heartbeat-abort.txt create mode 100644 docs/evidence/ws13/followup/heartbeat-live.txt create mode 100644 docs/evidence/ws13/followup/heartbeat-packed.txt create mode 100644 docs/evidence/ws13/followup/heartbeat-tests.txt create mode 100644 docs/evidence/ws13/followup/kernel-case-retry.txt create mode 100644 docs/evidence/ws13/followup/kernel-suite.txt create mode 100644 docs/evidence/ws13/followup/sdk-typechecks.txt create mode 100644 packages/sdk/src/worker-lease.ts create mode 100644 packages/sdk/tests/worker-cli-abort.test.ts create mode 100644 packages/sdk/tests/worker-lease.test.ts diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md index 3984a1ea..afc75082 100644 --- a/docs/evidence/ws13/README.md +++ b/docs/evidence/ws13/README.md @@ -9,12 +9,16 @@ does not attribute most of the total to the provider. **Named handoffs:** the release-gate owner must register `create-flow` in versioning/packaging/publishing. The review-swarm/CI owner must restore fresh maintainability, history and structure transcripts; all three are missing and -there is no independent review signoff. Neither handoff is a reason to keep +there is no approving independent signoff. After the PR left draft, Codex +and Cubic produced review findings; the lease-renewal P1 is addressed +in this branch, with [captured verification](followup/README.md). Those findings do not replace the missing swarm transcripts. Neither handoff is a reason to keep the PR in draft once the gallery results are reported. No publishing work or Cloud run-publication API is part of this follow-up. -The [current three-entry gallery](../../../examples/README.md) supersedes the -initial invocation results below. Research now reports each provider probe +The [current three-entry gallery](../../../examples/README.md) is **1 PASS, +2 BLOCKED** and supersedes the initial invocation results below. Research +completed with the default budget in [690.935s](followup/default-budget/gallery-research.txt); +the SDK flows still refuse unsupported budget headers. Research now reports each provider probe and timeout on stderr. [Research regression tests](followup/research-tests.txt) and [typecheck](followup/research-typecheck.txt) contain the commands/output. @@ -33,7 +37,7 @@ and [typecheck](followup/research-typecheck.txt) contain the commands/output. | Empty-cache install + deterministic run in fresh Debian Trixie container | Completed in 43.374s; deterministic template, no source clone or agent | [Command and output](cold-trixie.txt) | | Empty-cache install + deterministic run in fresh Debian Bookworm container | Refused: published Linux daemon requires GLIBC_2.39; 55.223s | [Command and output](cold-container.txt) | | Linux container test runner | esbuild Go runtime crashed under amd64 emulation before collecting tests | [Command, script and full output](container-tests.txt) | -| Research typecheck after correcting its compiler path | No type errors reported | [Command and output](research-typecheck.txt) | +| Research typecheck after correcting its compiler path | Superseded by the complete follow-up capture | [Command and output](followup/research-typecheck.txt) | The [gallery table](../../../examples/README.md) reports the three requested entries individually. Unsupported budget headers remain a capability-owner @@ -41,6 +45,11 @@ handoff. The initial research attempt reached an outer 150-second limit with no captured output; the follow-up now exposes preflight progress and captures the shim's own failure or success result. No gallery declaration was weakened. +The cold-container transcripts include provisioning output followed by the +inner command’s elapsed value; `record.py` was used for the separate PTY +agent recordings, not to time the cold Docker commands. Node/image/Git +provisioning is excluded from those cold command timings. + The recording uses the initial packed implementation plus the npm bin fix. Its agent step invokes the real installed Claude CLI. The host already had Node, provider authentication, and dependencies; this is **not** a cold-machine @@ -95,6 +104,9 @@ Serve all candidate tarballs locally: node docs/evidence/ws13/stage-registry.mjs /tmp/ws13-artifacts 48734 ``` +The registry binds to loopback by default. For Docker access, explicitly +add the bind host: `node docs/evidence/ws13/stage-registry.mjs /tmp/ws13-artifacts 48734 0.0.0.0`. + In a separate terminal, point npm at that registry; dependencies outside this branch redirect to the public npm registry: @@ -126,7 +138,8 @@ expanded into an unrelated refactor. The independent release-gate owner must add `create-flow` to package versioning and publishing, and to `scripts/pack-release.mjs`, which currently refuses that package name. That script also requires the legacy runtime executable. -Those gates were not edited. No package was published and no merge is allowed. +Those gates were not edited. No package was published and no merge is allowed. PR #247 is ready for review, +not in draft; publishing and review are named handoffs. Veto tools were not exposed. Relay queue receipts did not establish delivery; the coordinator confirmed the original handoff never arrived. The PR and this diff --git a/docs/evidence/ws13/followup/README.md b/docs/evidence/ws13/followup/README.md new file mode 100644 index 00000000..b7584304 --- /dev/null +++ b/docs/evidence/ws13/followup/README.md @@ -0,0 +1,63 @@ +# WS-13 follow-up: final gallery and lease correction + +PR #247 is out of draft. Timing is accepted by Khaliq's ruling. The existing +49.975s deterministic and 132.637s real-agent command transcripts remain in the +parent directory; no further timing experiment was completed after that ruling. + +The three requested gallery entries have individual, explicit outcomes: + +| Entry | Result | Elapsed | Command and output | +|---|---|---:|---| +| dependency-upgrade-bot | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 0.138s | [Final SDK capture](final-sdk/gallery-dependency-upgrade-bot.txt) | +| pr-review-pipeline | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 0.143s | [Final SDK capture](final-sdk/gallery-pr-review-pipeline.txt) | +| research | PASS: three lane reports and synthesis, `completionReason: synthesized`, exit 0 | 690.935s | [Default-budget capture](default-budget/gallery-research.txt) | + +The SDK/kernel capability owner must supply the two blocked flows' budget +headers, postfix artifact gates and declared workspace behavior. Those +requirements were not removed or weakened. Research uses its documented source +shim; the SDK examples use installed candidate npm artifacts. These are runs +on an authenticated development host in a separate clone, not cold benchmarks. +The final SDK artifact's [hash](final-sdk/artifact.json) identifies the package +used for the last two invocations; it includes the lease fix below. Research's +shim does not import AgentWorker and was unchanged by that fix. + +Research now prints each preflight probe and its timeout on stderr, leaving +stdout for the structured result. The first follow-up used a shorter three-minute +step bound and failed ([217.375s transcript](gallery-research.txt)). The retry +used the documented default 30-minute per-step budget and completed. Generated +[reports](default-budget/reports/) and [artifact hashes](default-budget/artifacts.json) +are retained as execution evidence; they are model-generated research output. + +After ready-for-review triggered Codex/Cubic comments, a P1 exposed that the +existing worker did not renew its 30-second lease. AgentWorker now renews +through `step.heartbeat`, confirms ownership before starting the CLI, and stops +the process group if renewal fails or its response does not arrive before lease +expiry. It drains outstanding renewals before `step.complete`. Renewal failures +never produce successful completions. There is no kernel protocol or gate change. +Existing fake worker clients gained the protocol heartbeat/deadline fields; +their assertions were preserved. + +| Verification | Result | Literal command and captured output | +|---|---|---| +| Research regression suite | 27 passed | [Transcript](research-tests.txt) | +| Research typecheck | Exit 0 | [Transcript](research-typecheck.txt) | +| Worker lease and existing wrapper suite | 17 passed | [Transcript](heartbeat-tests.txt) | +| Real raw/wrapper subprocess cancellation | 2 passed | [Transcript](heartbeat-abort.txt) | +| Built CLI + real daemon | 5 passed, including a 35s single-invocation case | [Transcript](heartbeat-live.txt) | +| Packed SDK + real daemon, same long case | 1 selected test passed; 4 not selected | [Transcript](heartbeat-packed.txt) | +| SDK/API/test-source types | Exit 0 | [Transcript](sdk-typechecks.txt) | +| Broader live-kernel suite | 29 passed, 1 hit its unchanged 5s timeout | [Failure retained](kernel-suite.txt) | +| Isolated rerun of that unchanged case | Passed; 29 other cases not selected | [Transcript](kernel-case-retry.txt) | + +No full-suite green or mutation verification is claimed. No package was +published. The release-gate owner must register `create-flow` for packaging and +publishing. The review-swarm/CI owner must obtain fresh maintainability, history +and structure transcripts; the old missing transcripts and new review comments +are not approving signoff at the final head. Lower-priority review comments +remain for review; this report does not claim every comment is resolved. + +`run-gallery.py` takes a clone and a fresh evidence directory. Use +`research-default` for the documented-budget research run or `sdk-only` for the +two SDK examples. It resolves Node from PATH and refuses to overwrite captures. +Earlier invocations used `/tmp/ws13-toolchain` to select the isolated Node 22 +installation; the captured argv retains those actual paths. diff --git a/docs/evidence/ws13/followup/default-budget/artifacts.json b/docs/evidence/ws13/followup/default-budget/artifacts.json new file mode 100644 index 00000000..6a3e1378 --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/artifacts.json @@ -0,0 +1,30 @@ +[ + { + "name": "claude", + "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/claude/report.md", + "file": "docs/evidence/ws13/followup/default-budget/reports/claude.md", + "bytes": 12957, + "sha256": "a6ecb1f63eed56aa6d1cc79b813c864950dc298fa5a2b5fb7c93d92cd7453f63" + }, + { + "name": "codex", + "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/codex/report.md", + "file": "docs/evidence/ws13/followup/default-budget/reports/codex.md", + "bytes": 1694, + "sha256": "cf02a078e799d7a291f01f81633c5aee8d8dda7b2e672b2f626eee38331914e7" + }, + { + "name": "grok", + "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/grok/report.md", + "file": "docs/evidence/ws13/followup/default-budget/reports/grok.md", + "bytes": 23777, + "sha256": "b733b7b6b0bb673dc5d8a2c4ba1b494d343585469f12fbdf507bdf733aa5d05e" + }, + { + "name": "synthesis", + "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/SYNTHESIS.md", + "file": "docs/evidence/ws13/followup/default-budget/reports/synthesis.md", + "bytes": 24831, + "sha256": "ebf4fbfddf85693dcd0ac500d41333b7e987a3e8279e730643d1a0834fdba239" + } +] diff --git a/docs/evidence/ws13/followup/default-budget/gallery-research.txt b/docs/evidence/ws13/followup/default-budget/gallery-research.txt new file mode 100644 index 00000000..d1d77e09 --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/gallery-research.txt @@ -0,0 +1,52 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node --experimental-strip-types examples/research/shims/run.ts --slug ws13-default-budget --question 'Compare durable step journals with deterministic replay. Keep every report under 200 words.' --runs-dir /tmp/ws13-research-followup-runs +OUTER_TIMEOUT_SECONDS=3900 +research: checking claude auth status (timeout 10s) +research: checking claude round-trip with model sonnet (timeout 90s) +research: checking codex login status (timeout 10s) +research: checking codex round-trip with model gpt-5.6-sol (timeout 90s) +research: checking grok round-trip with model grok-4.6 (timeout 90s) +research: checking claude round-trip with model opus (timeout 90s) +research: run dir /tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget +{ + "completionReason": "synthesized", + "synthesis": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/SYNTHESIS.md", + "reports": { + "claude": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/claude/report.md", + "codex": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/codex/report.md", + "grok": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/grok/report.md" + }, + "usage": { + "claude": { + "inputTokens": 18, + "outputTokens": 3368, + "cacheReadInputTokens": 365609, + "cacheCreationInputTokens": 21412, + "costUsd": "1.197919" + }, + "codex": { + "inputTokens": 948419, + "outputTokens": 3783, + "cacheReadInputTokens": 887936, + "cacheCreationInputTokens": 0 + }, + "grok": { + "inputTokens": 389652, + "outputTokens": 27334, + "cacheReadInputTokens": 1989888, + "cacheCreationInputTokens": 0, + "costUsd": "0.429503" + }, + "synthesizer": { + "inputTokens": 22, + "outputTokens": 14809, + "cacheReadInputTokens": 428952, + "cacheCreationInputTokens": 48875, + "costUsd": "1.078458" + } + } +} + +EXIT_CODE=0 +ELAPSED_SECONDS=690.935 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/default-budget/gallery-results.json b/docs/evidence/ws13/followup/default-budget/gallery-results.json new file mode 100644 index 00000000..4b9898c2 --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/gallery-results.json @@ -0,0 +1,8 @@ +[ + { + "example": "research", + "exitCode": 0, + "elapsedSeconds": 690.935, + "timedOut": false + } +] diff --git a/docs/evidence/ws13/followup/default-budget/reports/claude.md b/docs/evidence/ws13/followup/default-budget/reports/claude.md new file mode 100644 index 00000000..7d90300c --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/reports/claude.md @@ -0,0 +1,59 @@ +# Durable Step Journals vs. Deterministic Replay + +## 1. Executive summary + +**Recommendation: use a durable step journal (append-only log of step inputs/outputs + memoization) as the default; reserve deterministic replay for workflows whose in-process code is provably pure.** The two are alternative *recovery semantics* built on the same storage primitive (an append-only log), not competing storage technologies. Replay (Temporal, Azure Durable Functions) re-executes workflow code against recorded history, so it fast-forwards through already-completed calls but requires strict determinism — no ambient randomness, clocks, or I/O — and imposes real versioning pain (Temporal's `GetVersion`/Patch API). Step journals (DBOS, Restate, AWS Step Functions, and this repo's own `relayflowd`) persist step *results* only and never re-execute code, trading replay's microsecond resume for simpler mental models and safety with non-deterministic workloads — notably LLM/agent steps. For agentic systems specifically, the consensus is shifting toward step journals; this repo has already made that call explicitly (RFC-0001, settled decision #2). + +## 2. Landscape and best practices + +**Consensus.** A durable journal (append-only WAL/event log) is the storage primitive; deterministic replay is one possible *recovery mechanism* layered on it, alongside step-journal memoization as the other. Temporal's own docs describe Event History as "a complete and durable log of everything that has happened," with a Worker that, on recovery, "replays the code and recreates the state of the Workflow Execution to what it was immediately before the crash" [1, verified]. Azure Durable Functions uses the same event-sourced-history-plus-replay model, backed by Storage Tables/Netherite/MSSQL [2]. This dependence on replaying *code* is why determinism constraints exist at all: non-deterministic operations (random values, wall-clock reads, uncoordinated I/O) must be journaled or mediated by the SDK, or replay diverges from the original run [1, verified; 3]. + +Step-journal systems reject code replay entirely. DBOS checkpoints each step's result directly into Postgres — "workers checkpoint steps to Postgres themselves," and recovery means "another server can recover its workflows from their checkpoints," using Postgres's own integrity constraints to dedupe concurrent execution attempts [4, verified]. This traces to DBOS's Stanford/MIT VLDB 2021 paper on OS/database co-design for transactional durability [5, unverified — found via search, not fetched]. Restate and AWS Step Functions follow the same shape: a journal of step facts, resume = continue from the last completed fact, never re-run finished code [6, unverified — search snippet only]. + +**Contested/emerging.** Jack Vanlightly's November 2025 essay "Demystifying Determinism in Durable Execution" is the most careful public treatment of *why* the determinism requirement exists and where it bites (control-flow determinism vs. side-effect determinism) [3, unverified — not independently fetched, but content is consistent with Temporal's own docs]. ZenML's "No Journal, No Replay" post argues explicitly that journal-replay is the wrong model for AI agents: their Kitaru project caches step *outputs* in an artifact store instead of an operation log, explicitly avoiding "the determinism tax" — no restriction on randomness, timestamps, or external calls inside agent code — at the cost of higher resume latency (seconds vs. milliseconds), which the authors argue is irrelevant when LLM call latency already dominates [7, verified]. This is the same trade-off this repo's RFC makes independently. Temporal's versioning story (`GetVersion`/Patch, confirmed via Temporal's own docs [8, verified]) remains a genuinely unresolved pain point — old and new code paths must coexist indefinitely for long-running workflows — and is cited by multiple practitioners (ZenML, this repo's RFC) as the structural cost of the replay model. TigerBeetle's and FoundationDB's deterministic-simulation-testing work (seed-based replay for *chaos-testing*, not production recovery) is a separate but related use of "deterministic replay" and shouldn't be conflated with workflow recovery replay [9, unverified — described by subagent, not independently fetched by lead]. + +**Marketing vs. substance.** Vendor claims that "durable execution changes everything" are common across an increasingly crowded market (8+ vendors: Temporal, Restate, DBOS, Inngest, Azure Durable Functions, AWS Step Functions, Hatchet, and others); practitioner critiques (cited by the landscape subagent as Chris Riccomini's "Justifying the Bubble" and a Hatchet blog post, neither independently fetched by the lead — treat as unverified) argue the market is oversaturated and that determinism/idempotency operational burden is understated in vendor marketing. This is plausible directionally but not independently confirmed here. + +## 3. Recommended approach + +Default to a **step journal**: an append-only, fsync'd log keyed by `(run_id, step_id, attempt)`, storing each step's completion fact (`completionReason`, output, timestamps) — never the code. Resume reads the journal forward from the last checkpoint/epoch summary and skips any step already marked complete; it never re-executes in-process code. Idempotency keys (`sha256(run_id‖step_id)`, stable across retries) dedupe side effects at the point of write, not at the point of read. Bound journal growth with periodic compaction: close a segment, write a summary entry restating all still-live state, archive the closed segment losslessly (never rewrite it). This is exactly the design already adopted in this repo's kernel (see §5). + +Reserve **deterministic replay** for narrow, provably pure sub-computations — e.g., a bounded retry loop inside a single step — rather than as the durability substrate for an entire agentic workflow. If a workload genuinely has no non-determinism (pure orchestration of typed API calls, no LLM steps, no filesystem mutation), Temporal-style replay is a legitimate, mature choice with strong tooling; but agent steps that call LLMs, touch a workspace, or produce non-repeatable output make full-workflow replay actively unsafe, since any divergence between recorded and re-executed control flow silently corrupts recovered state. + +Test both models by construction, not by hope: crash-inject at every step boundary (`SIGKILL` between steps) and assert that resumed token/dollar spend equals exactly one execution per step; for any replay-based component, run periodic replay-divergence checks (replay recorded histories against current code in a sandbox worker) to catch determinism regressions before they hit production, and version any behavior change behind an explicit patch marker. + +## 4. Trade-offs and risks + +**Step journal risks:** journal bloat without compaction (mitigate with segment/epoch rollover); side-effect duplication on retry if idempotency keys aren't stable across attempts; step-payload schema drift as the system evolves (mitigate with additive-only fields and versioned readers). None of these are exotic — they're solved problems with known patterns — but they require discipline to implement correctly the first time. + +**Deterministic replay risks:** non-determinism bugs (map iteration order, `Date.now()`, random IDs, unmediated I/O) are the dominant real-world failure class, and they're insidious — a workflow can run correctly for months and then fail replay only when a specific historical branch is hit after a code change. Detecting this requires replay-history regression tests in CI, which many teams skip. Versioning is the other structural cost: Temporal's `GetVersion` API works but requires permanent branches in code for the lifetime of any long-running execution that started under the old version — this is a real, ongoing maintenance tax, not a one-time migration cost. + +**What would make this recommendation wrong:** if the target workload is overwhelmingly deterministic orchestration (financial transaction sequencing, typed API composition with no LLM/agent steps) and needs microsecond-latency resume with heavy reliance on Temporal's mature ecosystem (Cloud, observability, SDKs across 6+ languages), full replay may be the better-supported choice despite its versioning cost — the maturity gap between Temporal and newer step-journal-first tools (DBOS, Restate) is real and worth weighing against architectural purity. + +## 5. What we can leverage + +- **This repo's own kernel design (`docs/RFC-0001-everything-is-a-relayflow.md`, `kernel/DESIGN.md`, `kernel/relayflowd-journal/`)** — a settled, documented decision (RFC-0001 §6, decision #2, verified by direct read) to use step journal + memoization, explicitly rejecting deterministic replay as "semantically wrong for agents." The journal is SQLite-backed, append-only (`entries`/`segments` tables), with idempotency keys `sha256(run_id‖step_id)`, segment-per-epoch compaction (decision #8), and a full agent-step starting-state contract (Appendix A) covering pin-on-start, effect journaling, and crash recovery modes (`reset`/`inspect`/`manual`). **Fit: this is not prior art to adopt — it is the already-chosen architecture; the applied design work here should extend it, not replace it.** +- **DBOS Transact** — Postgres-native step checkpointing, MIT-licensed, library not server. **Fit: closest external analogue to this repo's model; worth studying its checkpoint/dedupe SQL patterns, but adopting it wholesale would mean abandoning the Rust/SQLite kernel already built.** +- **Temporal** — mature, widely deployed, strong tooling and multi-language SDKs. **Fit: poor for this repo's agent-step-heavy workload given the RFC's explicit rejection of replay; useful only as the comparison baseline the RFC's capability table already uses.** +- **Restate** — journal+durable-steps hybrid, newer, smaller community. **Fit: architecturally closer to this repo's approach than Temporal, but not evaluated in depth here; a follow-up could compare its journal schema against `kernel/DESIGN.md` directly.** +- **AWS Step Functions / Azure Durable Functions** — zero-ops managed options. **Fit: poor — cloud-locked, ASL/replay-model constraints don't match the multi-language, self-hostable kernel goal stated in RFC-0001 §4.** + +## 6. Open questions + +- Does `kernel/relayflowd-journal/` currently have automated replay-divergence or crash-injection tests matching RFC-0001's stated acceptance gate ("resumed run's token spend equals one execution of each step")? Not verified in this pass — only the design was read, not test coverage. +- How does this repo's segment-per-epoch compaction handle a step whose output payload schema changes across a kernel upgrade, in practice (RFC-0001 §7 leaves this "open until a real kernel upgrade has been executed")? +- Is there a quantified cost/latency comparison between this repo's journal-only resume and Temporal-style replay for a representative agent workflow, or is the "replay is semantically wrong for agents" decision based on qualitative reasoning alone? + +## 7. Sources + +1. https://docs.temporal.io/encyclopedia/event-history — Temporal Event History and replay mechanics — verified (fetched) +2. https://learn.microsoft.com/en-us/azure/durable-task/common/durable-task-orchestrations — Azure Durable Functions event-sourced orchestration model — unverified (reported by subagent, not independently fetched) +3. Jack Vanlightly, "Demystifying Determinism in Durable Execution" (Nov 2025) — unverified (reported by subagent; URL not independently confirmed) +4. https://www.dbos.dev/blog/postgres-is-all-you-need-for-durable-execution — DBOS Postgres-backed step checkpointing — verified (fetched) +5. DBOS VLDB 2021 paper, https://dl.acm.org/doi/10.14778/3485450.3485454 — foundational OS/DB co-design work behind DBOS — unverified (URL reported by subagent, not independently fetched) +6. Restate durable steps docs, https://docs.restate.dev/develop/go/durable-steps — unverified (reported by subagent via search snippet) +7. https://www.zenml.io/blog/no-journal-replay — argument against journal-replay for AI agents (Kitaru project) — verified (fetched) +8. https://docs.temporal.io/develop/go/workflows/versioning — Temporal GetVersion/Patch API for replay-safe code evolution — verified (fetched) +9. TigerBeetle deterministic simulation testing blog posts (tigerbeetle.com/blog) — unverified (reported by subagent, not independently fetched) +10. /private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md — local repo RFC documenting the step-journal-over-replay decision — verified (read directly, full text) +11. /private/tmp/ws13-gallery-followup/kernel/DESIGN.md — local kernel journal entry-type design — unverified in this pass (reported by subagent; not independently re-read by lead) diff --git a/docs/evidence/ws13/followup/default-budget/reports/codex.md b/docs/evidence/ws13/followup/default-budget/reports/codex.md new file mode 100644 index 00000000..d0ca0bc0 --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/reports/codex.md @@ -0,0 +1,29 @@ +# Executive summary + +Keep Relayflow’s durable step journal; use deterministic replay only for pure orchestration. Agent/LLM outputs, effects, artifacts, and spend are nondeterministic facts, not code to rerun. + +# Landscape and best practices + +Temporal reruns deterministic workflow code against event history; external work belongs in Activities, and incompatible changes require versioning [1]. Inngest instead injects persisted step results on recovery [2]; Restate applies this to LLM/tool calls [3]. Consensus: persist outcomes and make effects idempotent. “Exactly once” without provider cooperation is marketing. + +# Recommended approach + +Fold append-only entries into `RunState`; schedule only unfinished steps. Persist leases, pins, outputs, `completionReason`, budgets, and effect keys in per-run SQLite; compact live state into `epoch.summary` (`kernel/DESIGN.md:3-11,65-78,135-149,162-206`). + +# Trade-offs and risks + +Journals expose step boundaries and storage/atomicity costs; replay preserves natural control flow but imposes determinism and deployment constraints. Elect→perform→confirm can duplicate provider success after a pre-confirmation crash (`kernel/DESIGN.md:121-133`). + +# What we can leverage + +Reuse the Rust/rusqlite journal and pure state machine; DBOS/Inngest validate the pattern, but migration adds little. + +# Open questions + +Can every adapter enforce provider idempotency? What retention and resume-latency bounds pass crash injection? + +# Sources + +1. https://docs.temporal.io/workflow-definition — verified +2. https://www.inngest.com/docs/learn/how-functions-are-executed — verified +3. https://docs.restate.dev/ai/patterns/durable-agents — verified diff --git a/docs/evidence/ws13/followup/default-budget/reports/grok.md b/docs/evidence/ws13/followup/default-budget/reports/grok.md new file mode 100644 index 00000000..6ea23497 --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/reports/grok.md @@ -0,0 +1,188 @@ +# Durable step journals vs deterministic replay + +Lane: grok. Date: 2026-09-08. + +Two subagents (Landscape, Applied) ran in parallel. This report merges them after parent spot-checks of local kernel code and primary vendor docs. Disagreements resolved in §3. The research question asked to keep every report under 200 words; §1 is that comparison. The remaining sections exist because the lane protocol required them. + +## 1. Executive summary + +Keep a durable step journal as source of truth; recover by memoizing recorded results, not by re-executing workflow source. RFC-0001 decision #2: Temporal-style code replay is semantically wrong for `llm` and `agent` steps. This kernel already does journal + memoization: `RunState::fold` injects completed outputs as facts; `crash_resume` asserts completed effects are not replayed as code. + +A journal without skip-on-resume double-charges. Code replay without a log dies with the process. They complement only if "replay" means folding facts. Temporal re-runs Workflow functions and matches Commands to Event History (determinism + versioning). Inngest, Restate, and DBOS inject stored step outputs — closer. LangGraph checkpointers snapshot state; they are not a command log. + +Do not adopt Temporal/Restate/DBOS. The journal protocol is the product boundary. Finish epoch compaction, two-phase effects under crash, and Gate-5 archival; gate with existing crash-injection tests. + +## 2. Landscape and best practices + +### What the two phrases actually name + +**Durable step journal.** An append-only, ordered log of facts about a run: this step started, this wait armed, this effect was elected, this attempt completed with `completionReason` and output. The log is the source of truth. Event sourcing is the general pattern [1]. Temporal Event History is one such log [2][3]. Restate, Inngest, and DBOS each persist per-step results in a journal or checkpoint table [4][5][6]. Fowler’s classic constraint still applies: external side effects must be gated so replay of the log does not re-send them [1]. + +**Deterministic replay.** Overloaded. Three distinct mechanisms share the word: + +1. **Code replay (Temporal / Cadence).** On resume, the worker starts the Workflow function from the top. Commands emitted by that re-execution are matched against Event History. A mismatch is a *non-deterministic error*. Activities, timers, and signals are not re-done; their recorded results are fed back. Workflow code must be deterministic given that history: no raw `Date.now()`, RNG, or I/O outside Activities [3][7]. Versioning (patches, Worker Versioning) exists because deployed code is part of the recovery path [7]. +2. **Result replay / step memoization (Inngest, Restate, DBOS, this kernel).** The handler or scheduler is re-entered, but completed steps are short-circuited: stored outputs are injected, unfinished work runs. Inngest is explicit that this is *not* Temporal’s model [5]. Restate journals `ctx.run` / equivalent and “replays the journal” as recorded results [4]. DBOS restarts the workflow function with checkpointed inputs and returns checkpointed step outputs [6]. +3. **Snapshot restore (LangGraph checkpointers).** Persist graph state after a super-step; resume from the last snapshot. Durability modes include `exit` / `async` / `sync`; `InMemorySaver` does not survive process restart [8]. This is a checkpoint, not a command log. + +Inngest’s own “Durable Agents” page calls (2) “deterministic replay” [9]. That naming is the main source of confusion in the last 18 months. RFC-0001 uses “deterministic replay” to mean (1), and rejects it [10]. + +### Consensus (multiple independent primary sources) + +- Persist progress *before* the caller observes a result; the log is what happened [1][2][4]. +- Isolate side effects from control flow. Temporal: Activities. Inngest/Restate/DBOS: `step.run` / `ctx.run` / `@DBOS.step`. Relayflows: `deterministic` | `llm` | `agent` with effects journaled separately [3][4][5][6][10]. +- On recovery, do not re-execute completed side effects. Exactly-once *effects* is the claim; attempts may run more than once [6][10][11]. +- Control flow, given recorded results, must be stable enough to reach the first unfinished step. That is weaker than “the source file is a pure function of history.” +- A journal that is only an observability trace (OpenAI/Anthropic session traces, unverified here) is not a recovery mechanism. +- Simulated-clock deterministic simulation (FoundationDB, TigerBeetle VOPR — Landscape cited; parent did not re-fetch) is a *test* technique, not production recovery. This kernel uses a simulated clock in `relayflowd-core` for that reason [12]. + +### Contested / emerging + +- **Code replay vs memoization for agents.** Temporal’s 2025 blog argues agents are fine if LLM calls live in Activities [13, unverified]. RFC-0001 and Inngest argue the opposite for agent loops: the graph is drawn at runtime; forcing a hermetic Workflow function plus Activity split is the wrong authoring model [9][10]. This is the live industry split, not a settled science. +- **Snapshots vs event logs.** LangGraph time-travel wants snapshots [8]. Resident runs that must answer “which agent, under which credential, why” want an append-only journal [10]. You can project snapshots *from* a journal; you cannot reconstruct a journal from a snapshot. +- **Exactly-once vs at-least-once + idempotency.** Hatchet’s architecture docs state at-least-once and require idempotent tasks [14]. DBOS claims exactly-once for steps that share a Postgres transaction with the checkpoint [6]. Relayflows split election from provider call (two-phase `effect.recorded` / `effect.confirmed`) because the mount is not yet the writer [12]. Anyone selling “exactly-once” without naming the crash window is contested. +- **Workflow-as-code immutability.** Restate’s older write-up treats versioned deployments as the escape from Temporal’s patching [15, unverified by parent]. RFC-0001’s escape is different: replay results, not code, so an old segment needs only an old *reader* [10]. + +### Marketing (dropped or discounted) + +- Hatchet marketing copy about a “transactionally-safe event log” implying no duplicates. Their own guarantees page says at-least-once [14]. +- “Durable agents” on a `MemorySaver` or an LLM trace store. Persistence that dies with the process, or that cannot resume a killed run, is not durable execution [8]. +- Vendor “exactly-once” without an elect-before-call or transactional piggyback story. + +Canonical older work still in force: Fowler event sourcing and external-system gateways [1]; CQRS as a *read* projection, not a substitute for the write log [16, search only]; ARIES write-ahead logging and repeating history (Landscape fetched the PDF; parent did not, so the PDF is `unverified` here); sagas compensate rather than replay effects (ACM paper not fetched). + +## 3. Recommended approach + +**One sentence.** Treat the journal as the run; recover by folding it into `RunState` and dispatching only unfinished work. Do not re-run completed step code. That is already Gate 1 in this repo. + +Landscape said crash-safe exactly-once “needs both” a journal and replay. Applied said this repo forbids deterministic replay. **Resolved:** need a durable log *and* a recovery procedure that consumes it. The recovery procedure is memoized result-fold, not Temporal code replay. Inngest’s use of “deterministic replay” for memoization is a naming collision; this report uses RFC vocabulary. + +### 3.1 What already exists (do not redesign) + +**Components.** `relayflowd-core` is a pure state machine on a `Clock` trait (`SimClock` in tests). `relayflowd-journal` is the SQLite implementation. `relayflowd` interprets `Action`s (append, exec deterministic, dispatch, arm timer, complete run). The TypeScript SDK speaks journal protocol v0 over a unix socket; it does not reach around the protocol [12][17]. + +**Write path.** One SQLite file per run: `/runs/.sqlite3`, `PRAGMA journal_mode=WAL`, `PRAGMA synchronous=FULL`. One transaction per logical append. A failed commit returns `Err`; the protocol maps that to `journal_write_failed` and the step fails. No fallback [12][18][17]. Envelope fields: `seq`, `segment_id`, `entry_type`, `run_id`, `step_id`, `attempt`, `at_ms`, canonical JSON `payload` [12]. + +Entry types that matter for this comparison: `run.spawned`, `step.attempt.started` (pins, idempotency key, lease), `step.completed` (`completionReason`, `disposition`, memoized `output`, budget), waits/sleeps, `stream.appended` / channel facts, two-phase `effect.recorded` then `effect.confirmed`, `memory.injected`, `epoch.summary`, `segment.closed`, `run.completed` [12]. + +**Retrieval / resume.** `Engine::resume` opens the run file, loads spec, folds current-segment entries via `RunState::fold`, runs `recovery_actions_filtered` for dead attempts, then continues scheduling [19][20][21]. Completed steps with `disposition=step_done` become `Done`; their `output` is injected as fact, spending zero tokens and appending zero entries [12]. A `step.attempt.started` without `step.completed` is abandoned as `crashed` or `lease_expired` unless a live worker still holds the lease [21]. Open waits re-arm; elapsed timers fire. + +**Effects.** Appendix A rule 5: exactly-once *effects*, not exactly-once execution. v0 is elect → perform → confirm. An unconfirmed election does not suppress the next attempt (the winner may have died before the provider call). A successful completion holding an unconfirmed election is refused [12][22]. Idempotency key is `sha256(run_id ‖ step_id)`, stable across attempts [12]. + +**Agent starting state.** `step.attempt.started` pins workspace revisions and stream offsets. Recovery modes: `reset` (default, restore pins), `inspect` (dirty workspace + trajectory tail), `manual` (`needs_human`) [10][12]. + +**Channels.** Replay reads recorded deliveries in journal sequence; it does not execute consumer code or invoke receive again [23]. + +**Memory.** Crash recovery reuses the journaled `memory.injected` pack; the provider is not called again. The current provider is `FixedMemoryProvider` (synthetic pack) — substrate stub, not retrieval quality [24]. + +**Consolidation / forgetting.** Decision #8: segment-per-epoch. Rollover appends `segment.closed` + `epoch.summary` in one transaction; closed segments are never rewritten [10][12]. `rollover_is_atomic_scaffolding_for_epoch_resume` exists [25]. Gate-5 archival of closed segments to relayhistory is specified, not implemented as a live reader (Applied; parent did not find a reader either). + +**Evaluation already in-tree.** `kernel/relayflowd/tests/crash_resume.rs`: SIGKILL at every hello-ladder boundary and mid-step, then `resume` CLI. Assertions: completed marker effects are not re-executed; journal attempt counts match (`assert_exact_journal`); mid-step dead attempt is explained and retried [26]. That is the gate, not a nice-to-have. + +### 3.2 Architecture to keep building (not a new engine) + +``` +spec (data) ──run.start──► journal append (run.spawned) + │ + ▼ + fold → RunState + │ + ┌───────────────┼────────────────┐ + ▼ ▼ ▼ + ExecDeterministic Dispatch llm/agent ArmTimer / wait + │ │ + │ effect.record ─► provider ─► effect.confirm + │ │ + └──── step.completed (memo) ────┘ + │ + resume = fold + dispatch unfinished +``` + +Control flow lives in the spec + kernel machine, not in user source that must re-emit the same Commands. Non-determinism is recorded as facts: LLM output, agent pins, memory pack, routing decision, effect election. + +**Do not add Temporal.** Adopting it would require wrapping every `llm`/`agent` step as an Activity and keeping Workflow source deterministic — the thing decision #2 forbids. Parent grep found no Temporal/Inngest/Restate/DBOS adapters in kernel or SDK; do not create them. + +**Do copy the useful idea from the memoization family:** named step boundaries whose outputs are the memo table. This kernel already has that as `step.completed.output`. Inngest’s extra trick — defining steps *at runtime* inside an agent loop — is useful for Gate 4 resident loops, but those loops must still journal each iteration as a step, not as Temporal history events. + +## 4. Trade-offs and risks + +**What this gets right for agents.** An `llm` or `agent` step is not a pure function. Re-running its source to rebuild locals would either re-call the model (budget invariant fails) or require the author to have split every non-deterministic call into an Activity (authoring friction, Temporal versioning hell). Journal + memoization records the *result* and never re-enters completed work. RFC §7’s versioning story follows: old segments need old readers, not old code [10]. + +**What Temporal still does better.** Fine-grained locals and branches inside one long Workflow function, without declaring a spec step for each. Signal/query as first-class. A large ecosystem (Nexus, multi-language workers, patching libraries). If this product were only hermetic activities with no agents, code replay would be the conservative choice. + +**What would make journal + memoization wrong.** + +- All steps become deterministic, hermetic, and cheap to re-enter — then code replay’s “workflow is a function” DX wins and the spec compiler is overhead. +- The journal protocol is abandoned for a vendor runtime. Then pins, `completionReason`, two-phase effects, per-step token budgets, and tenant-unaware cells (decision #15) have to be re-expressed in someone else’s model. They will not fit. +- Epoch summaries drift from the folded log (`steps_done` / `budget_spent` disagree). Resume would skip or double-run. Detect by folding the current segment and comparing to `epoch.summary`. +- Unconfirmed effects complete successfully — the crash window between elect and provider call becomes “zero provider calls.” The kernel already refuses this; a regression is a P0. +- Silent re-exec of `step_done` work. Detect: crash tests plus “resumed spend equals one success per step” [12][26]. +- Treating LangGraph `durability="async"` or in-memory checkpointers as equivalent. They are not fail-closed [8]. +- Calling simulated-clock DST “production replay.” Core tests on `SimClock` pin the machine; they do not replace SQLite crash-injection. + +**Operational cost of staying custom.** You own fsync discipline, compaction, leases, and worker dispatch. That is the point of a small Rust kernel. The cost is real: epoch archival is still scaffolding [25]; memory is a stub [24]; mount-as-writer (collapsing two-phase effects) waits on later gates [12]. + +## 5. What we can leverage + +| Item | Fit assessment | +|---|---| +| This kernel’s journal + `RunState::fold` (`kernel/relayflowd-core/src/state.rs`, `machine.rs`, `machine/recovery.rs`) | **Use as-is.** This *is* the recommended approach, already implemented. | +| SQLite WAL + `synchronous=FULL` (`kernel/relayflowd-journal/src/append.rs`) | **Keep.** Fail-closed append; one file per run matches decision #15 (sleeping cell costs storage only). | +| Crash-injection suite (`kernel/relayflowd/tests/crash_resume.rs` and submodules) | **The evaluation harness.** Extend; do not replace with vendor replay testers. | +| Journal protocol v0 (`packages/sdk/src/protocol.ts`, `kernel/DESIGN.md` §5) | **The product boundary.** SDKs speak it; nothing reaches around it. | +| Two-phase effects (`kernel/relayflowd/src/engine/effects.rs`) | **Keep.** Honest about the elect/perform crash window; closer to exactly-once than “at-least-once + hope.” | +| Epoch rollover (`kernel/relayflowd-journal/src/segment.rs`, test in `lib.rs`) | **Scaffolding, not forgetting.** Finish archival to relayhistory; do not rewrite closed segments. | +| Durable channels (`kernel/DURABLE-CHANNELS.md`) | **Result-replay of messages.** Offsets are facts; receive is not re-executed. | +| Step memory (`kernel/MEMORY.md`) | **Journaled pack reuse on resume is right.** Provider is a stub; do not confuse it with Gate 5 quality. | +| Temporal (Event History + code replay) [2][3][7] | **Do not adopt.** Contradicts decision #2. Useful as the negative example and as the competitor Gate 1 must match on durability, not on mechanism. | +| Inngest step memoization [5][9] | **Closest commercial analog.** MIT/SSPL mix and HTTP-invoke model; no artifact/pins/budget kernel. Steal the *explanation*, not the service. | +| Restate journals + Virtual Objects [4] | **Similar durability, wrong protocol.** Extra runtime in front of services; would replace `relayflowd`. License not re-verified here. | +| DBOS Transact on Postgres [6] | **Apache-2.0, library-in-process.** Good fit for DB-local steps; does not give `llm`/`agent` rails, pins, or a journal protocol. Do not replace the kernel with it. | +| Hatchet [14] | **Postgres task log, at-least-once.** Fine as a queue; weaker effect story than Appendix A. | +| LangGraph checkpointers [8] | **Snapshots for graph agents, not a run journal.** `MemorySaver` is not durable. Do not use as the kernel store. | +| Fowler event sourcing + gateways [1] | **Prior art for “replay results, disable external gateways.”** Already encoded as memoization + effect election. | +| 12-factor-agents factor 5 (thread as state) [27] | **Aligned at slogan level** (unify execution and business state as events). Their “thread is the context window” is Gate 4’s *view*, not the journal. History stays complete; context is assembled per wake [10]. | +| Cadence replayer/shadower | **Unverified** (parent did not fetch). Temporal’s ancestor; same code-replay family. | +| FoundationDB / TigerBeetle DST | **Unverified by parent.** Relevant to `SimClock` tests, not to production resume. | + +## 6. Open questions + +1. **Has a live resident run crossed an epoch boundary on a new `journal_version`?** RFC §7 leaves spec/journal/protocol versioning open until that happens [10]. Result-replay is the claimed escape from Temporal versioning; it is unproven in production in this repo. +2. **When does Gate 4 collapse elect/confirm into “the mount write is the effect record”?** Until then, exactly-once is two-phase and the crash window is real [12]. +3. **Channel compaction.** Channel replay currently scans retained segments; there is no bounded snapshot for deleting old segments [23]. Resident runs will hit this. +4. **Dynamic steps inside an agent loop.** Inngest allows `step.run` names decided at runtime [9]. Relayflow specs are compiled, content-addressed bundles (decision #14) [10]. Can a Gate 4 loop journal iteration N as data without minting a new digest every iteration? +5. **Semantic retry vs Temporal retry.** Kernel retries `verification_failed` as a new attempt with a new model call, bounded by `max_iterations`, charging each attempt [12]. Confirm the budget invariant still holds when `inspect` recovery re-enters a dirty workspace. +6. **Vendor lock-in if we ever *embed* Restate/DBOS for a subset of deterministic steps.** Probably not worth it; the protocol would fork. + +## 7. Sources + +1. https://martinfowler.com/eaaDev/EventSourcing.html — Event log as source of truth; rebuild; external gateways on replay. `verified` +2. https://docs.temporal.io/encyclopedia/event-history/ — Event History; Commands mapped to Events; crash recovery via replay. `verified` +3. https://docs.temporal.io/workflows — Resume re-runs Workflow code from the beginning against history; Activities not re-executed. `verified` +4. https://restate.dev/what-is-durable-execution — Journaled steps; restart and replay recorded results. `verified` +5. https://www.inngest.com/docs/learn/how-functions-are-executed — Step memoization vs Temporal deterministic replay; each step a separate HTTP invocation. `verified` +6. https://docs.dbos.dev/architecture — Postgres checkpoints; recover by restarting the workflow and skipping checkpointed steps; workflow must be deterministic given step outputs. `verified` +7. https://docs.temporal.io/workflow-definition — Determinism constraints; Command/Event matching; non-deterministic errors; versioning. `verified` +8. https://docs.langchain.com/oss/python/langgraph/persistence — Checkpointers as graph-state snapshots; in-memory saver is not durable. `verified` +9. https://www.inngest.com/docs/learn/durable-agents — Calls memoization “deterministic replay”; dynamic agent loops. `verified` +10. `/private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md` — Decision #2 no deterministic replay; journal + memoization; Appendix A; epoch compaction. `verified` +11. `/private/tmp/ws13-gallery-followup/kernel/DESIGN.md` — Entry types, SQLite schema, memoized resume algorithm, protocol v0. `verified` +12. Same as [11] plus `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs`, `machine.rs`. `verified` +13. https://temporal.io/blog/of-course-you-can-build-dynamic-ai-agents-with-temporal — Agents via Activities. `unverified` (search snippet only) +14. https://docs.hatchet.run/v1/architecture-and-guarantees — Postgres state; **at-least-once**; tasks must be idempotent. `verified` +15. https://restate.dev/blog/solving-durable-executions-immutability-problem/ — Versioned deployments vs patching. `unverified` (parent did not fetch) +16. https://martinfowler.com/bliki/CQRS.html — CQRS. `unverified` (search only) +17. `/private/tmp/ws13-gallery-followup/packages/sdk/src/protocol.ts` — Verb set including `run.resume`, `effect.record`/`confirm`, `journal.read`. `verified` +18. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/append.rs` — Immediate transaction, fail-closed. `verified` +19. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine.rs` — `resume` / `resume_filtered`. `verified` +20. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs` — `RunState::fold`. `verified` +21. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/machine/recovery.rs` — Dead attempts → `crashed` / `lease_expired`. `verified` +22. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine/effects.rs` — Elect / confirm. `verified` +23. `/private/tmp/ws13-gallery-followup/kernel/DURABLE-CHANNELS.md` — Replay deliveries, do not re-execute receive. `verified` +24. `/private/tmp/ws13-gallery-followup/kernel/MEMORY.md` — Pack reused on resume; provider stub. `verified` +25. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/lib.rs` — `rollover_is_atomic_scaffolding_for_epoch_resume`. `verified` +26. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/tests/crash_resume.rs` — SIGKILL then resume; “completed effects must not be replayed as code”. `verified` +27. https://github.com/humanlayer/12-factor-agents/blob/main/content/factor-05-unify-execution-state.md — Unify execution/business state; resume by loading the thread. `verified` +28. https://web.stanford.edu/class/cs345d-01/rl/aries.pdf — ARIES WAL. `unverified` (Landscape claimed fetch; parent did not) +29. https://cadenceworkflow.io/docs/go-client/workflow-replay-shadowing — Cadence replayer. `unverified` +30. https://apple.github.io/foundationdb/testing.html — Deterministic simulation. `unverified` (parent did not fetch) +31. https://docs.tigerbeetle.com/concepts/safety/ — VOPR / WAL. `unverified` (parent did not fetch) diff --git a/docs/evidence/ws13/followup/default-budget/reports/synthesis.md b/docs/evidence/ws13/followup/default-budget/reports/synthesis.md new file mode 100644 index 00000000..27abebdd --- /dev/null +++ b/docs/evidence/ws13/followup/default-budget/reports/synthesis.md @@ -0,0 +1,420 @@ +# Synthesis — Durable step journals vs. deterministic replay + +Editor's note on lane compliance: the question asked for reports under 200 words. +Only **codex** complied (~190 words). **grok** wrote a compliant §1 and then ~2,900 +words under a lane protocol it says overrode the limit; **claude** wrote ~1,900 +words with no acknowledgement of the limit. Length did not track quality: +codex's short report is correct but thin, grok's long report carries the most +verified primary-source and in-repo grounding, and claude's mid-length report +contains the one factual error found in this pass (see §3.1). + +--- + +## 1. Recommendation + +Keep the durable step journal as the source of truth and recover by folding +recorded facts into state — do not adopt Temporal-style code replay. All three +lanes reach this conclusion independently, and it is already the settled +decision in this repo (RFC-0001, decision #2 [16]). Nothing here argues for a +new engine; the work is finishing the one that exists. + +Build, in order: + +1. **Close the epoch loop.** Rollover is scaffolding today [23]; finish Gate-5 + archival to relayhistory and add a live reader. Never rewrite a closed + segment — our escape from Temporal's `GetVersion` tax [4] is *old readers*, + not old code paths kept alive forever. +2. **Add a summary-vs-fold divergence check.** Fold the current segment and + assert it equals `epoch.summary`. Drift here silently skips or double-runs + steps on resume — the highest-severity failure mode this design has. +3. **Harden two-phase effects.** `effect.recorded` → provider → `effect.confirmed` + [21] leaves a real crash window. Keep refusing a successful completion that + holds an unconfirmed election; treat regression as P0. Audit which adapters + actually pass a provider idempotency key (codex). +4. **Extend crash injection, don't replace it.** `crash_resume.rs` [24] is the + acceptance gate: SIGKILL at every step boundary, assert resumed spend equals + exactly one success per step. Extend it across epoch boundaries and channels. +5. **Bound channel replay.** Replay scans retained segments with no snapshot + [25]; resident runs will hit this. + +Do not build Temporal, Restate, DBOS, or Inngest adapters. Do borrow their +vocabulary when explaining the design. + +## 2. Where the lanes agree + +- **Journal + memoization beats code replay for agent/LLM workloads.** An `llm` + or `agent` step is not a pure function; re-entering its code either re-bills + the provider or forces every non-deterministic call into an Activity-shaped + split. (claude, codex, grok) +- **The two are not competing storage technologies.** An append-only log is the + shared primitive; code replay and result memoization are two *recovery + procedures* over it. A journal with no skip-on-resume double-charges; code + replay with no log dies with the process. (claude, grok; codex implicitly) +- **Temporal is the canonical code-replay system, and determinism is the price.** + Workflow code re-runs from the top against Event History; Activities are not + re-executed, their results are fed back; incompatible changes need versioning + [1][2][3][4]. (claude, codex, grok) +- **Isolate side effects from control flow.** Temporal Activities, Inngest + `step.run`, Restate `ctx.run`, DBOS `@DBOS.step`, Relayflows' + `deterministic | llm | agent` with effects journaled separately. + [3][5][7][9][16] (claude, codex, grok) +- **"Exactly-once" is a claim about *effects*, not executions.** Attempts may + run more than once; idempotency keys and provider cooperation are what make + the effect single. Vendor "exactly-once" copy that never names the crash + window is marketing. Hatchet's own guarantees page says at-least-once + [11]. [9][11][16] (claude, codex, grok) +- **Stable idempotency key across attempts.** `sha256(run_id ‖ step_id)`. + (claude, grok) +- **Compaction is required and must be lossless.** Segment-per-epoch rollover + with a summary entry; closed segments archived, never rewritten + (decision #8) [16][23]. (claude, grok) +- **Don't adopt a vendor runtime.** DBOS/Inngest/Restate validate the pattern; + migrating buys little and would cost the pins, budgets, `completionReason`, + and protocol boundary. (claude, codex, grok) +- **Deterministic *simulation* testing (FoundationDB, TigerBeetle) is a + different thing** wearing the same word — a test technique, not production + recovery. (claude, grok — both lanes marked their sources unverified, so no + entry appears in §8) + +## 3. Where the lanes disagree + +### 3.1 Do step-journal systems re-execute workflow code? — resolved against claude + +**claude:** step-journal systems "reject code replay entirely"; DBOS, Restate, +AWS Step Functions "persist step *results* only and never re-execute code." +**grok:** result memoization *does* re-enter the handler — "DBOS restarts the +workflow function with checkpointed inputs and returns checkpointed step +outputs" — and DBOS therefore still requires the workflow function to be +deterministic given step outputs [9]. + +**Ruling: grok is right, and this is the one substantive error in the pass.** +I re-fetched the DBOS architecture doc to settle it. It states: "DBOS restarts +each interrupted workflow by calling it with its checkpointed inputs. As the +workflow re-executes, it checks before each step if that step's output is +checkpointed in Postgres. If there is a checkpoint, the step returns the +checkpointed output instead of executing," and "The workflow function must be +**deterministic**: if executed multiple times, with the same arguments and step +return values, the workflow should invoke the same steps with the same inputs +in the same order." [9] + +This matters beyond pedantry. claude used "never re-executes code" as the +generic property of the step-journal family, which would imply DBOS/Inngest +carry no determinism constraint at all. They carry a *weaker* one — +determinism of control flow given recorded step outputs, rather than +determinism of the whole function including clocks and RNG. The property +claude describes is real, but it belongs to **this kernel specifically**, and +it comes from a different design choice: control flow lives in a declarative +spec folded by `RunState::fold` [18], not in user source that must re-emit the +same calls. That is a stronger position than DBOS's, and it should be argued +on that basis rather than on a false generalization. + +### 3.2 Is "deterministic replay" one thing or three? — grok + +**grok** splits the term into (1) code replay (Temporal/Cadence), (2) result +replay / step memoization (Inngest, Restate, DBOS, this kernel), (3) snapshot +restore (LangGraph checkpointers), and notes Inngest itself calls (2) +"deterministic replay" [6]. **claude** and **codex** treat the term as +Temporal's meaning only. + +**Ruling: adopt grok's taxonomy.** It is the difference between a naming +collision and a disagreement, and it dissolves 3.1's confusion. Use RFC-0001 +vocabulary in our own docs — "deterministic replay" means (1) and is rejected — +but expect readers arriving from Inngest to mean (2). + +### 3.3 Are agents fine under Temporal if LLM calls are Activities? + +**grok** surfaces Temporal's 2025 position that they are (marked unverified, +search snippet only), against RFC-0001's and Inngest's position that agent +loops draw their graph at runtime and the hermetic-function-plus-Activity +split is the wrong authoring model [6][16]. claude and codex do not engage the +counter-argument; claude asserts full-workflow replay is "actively unsafe" for +agent steps. + +**Ruling: grok states it fairly and claude overstates.** Temporal-with- +Activities is *workable* — it is not unsafe, it is expensive in authoring +friction and versioning. The honest form of our claim is: the mechanism costs +more than it returns for runtime-shaped agent graphs, not that it corrupts +state. Note this is an unsettled industry split, and grok's citation for +Temporal's side is unverified. + +### 3.4 What would make this recommendation wrong? + +**claude:** mature-ecosystem pull — Temporal Cloud, observability, six-plus +language SDKs — could outweigh the versioning cost for deterministic +orchestration. **grok:** the falsifiers are internal — all steps becoming +cheap and hermetic, epoch summaries drifting, unconfirmed effects completing. +**codex** does not address it. + +**Ruling: both, and they are not in tension.** claude's is the "should we have +started here" question and is now moot; grok's are live regression detectors +and belong in CI. Item 2 of §1 comes from grok's list. + +### 3.5 Depth and scope + +**codex** answered the question asked, at the length asked, and its terse +claims all hold. **grok** exceeded the limit tenfold and returned the only +report with a per-file map of the existing implementation. **claude** exceeded +it fivefold and returned the weakest source verification (five of eleven +sources unverified, including two it built argument on). + +**Ruling: grok's report is the spine of this synthesis, codex's is the correct +answer in miniature, claude's contributes the ZenML citation and the +external-maturity framing.** Length was not what separated them — verification +discipline was. + +## 4. Single-source claims worth keeping + +- **ZenML's "No Journal, No Replay" / Kitaru** (claude, verified [13]) — the + clearest public statement of the counter-position: cache step *outputs* in an + artifact store, accept seconds-not-milliseconds resume, on the argument that + LLM latency dominates anyway so the "determinism tax" buys nothing. This is + the best external corroboration of decision #2 and the only lane to find it. +- **Temporal's versioning tax has a name and an API** (claude, verified [4]) — + `GetVersion`/Patch requires permanent code branches for the lifetime of any + long-running execution started under the old version. This is the concrete + cost our "old readers, not old code" story avoids. +- **Hatchet is explicitly at-least-once** (grok, verified [11]) — the cleanest + citation for why "transactionally-safe event log" marketing copy does not + imply no duplicates. +- **LangGraph checkpointers are snapshots, not a command log** (grok, verified + [12]) — `InMemorySaver` does not survive restart; `durability="async"` is not + fail-closed. You can project a snapshot from a journal, not the reverse. Worth + keeping because "durable agents" claims in this space often rest on this. +- **The elect→perform→confirm duplication window** (codex and grok, from + `kernel/DESIGN.md` [17][21]) — codex names it as the headline risk in five + words; grok explains that an unconfirmed election deliberately does not + suppress the next attempt because the winner may have died before the + provider call. Both matter: it is the sharpest honest weakness in the current + design. +- **Provider idempotency is per-adapter, not global** (codex, open question) — + the kernel's stable key is worth nothing on an adapter whose provider ignores + it. No other lane asked. +- **12-factor-agents factor 5** (grok, verified [15]) — aligned at slogan level + only; their "thread is the context window" is a *view*, not the journal. +- **Fowler's external-gateway rule** (grok, verified [14]) — the 2005 statement + of the same constraint: replaying the log must not re-send external effects. + Useful as prior art when explaining that none of this is novel. + +## 5. The plan + +**Repository mapping.** The question named no repositories; all three lanes +grounded it in this repo, and I follow them. The system under discussion is the +`relayflowd` kernel (`kernel/relayflowd-core`, `kernel/relayflowd-journal`, +`kernel/relayflowd`) plus the TypeScript SDK (`packages/sdk`), governed by +`docs/RFC-0001-everything-is-a-relayflow.md` and `kernel/DESIGN.md`. External +repos (temporalio, dbos-inc, inngest, restatedev, langchain-ai/langgraph) are +reference material, not integration targets — the lanes are unanimous that no +adapter should be written, and grok's grep confirms none exists. + +### Phase 0 — hold the line (already done; do not redesign) + +Verified present: pure state machine on a `Clock` trait with `SimClock` in +tests [18]; SQLite per run with `journal_mode=WAL`, `synchronous=FULL`, one +transaction per append, fail-closed to `journal_write_failed` [22]; envelope of +`seq`, `segment_id`, `entry_type`, `run_id`, `step_id`, `attempt`, `at_ms`, +canonical JSON payload [17]; `Engine::resume` folding entries and filtering +dead attempts [20][19]; crash-injection suite [24]. + +### Phase 1 — data model + +Keep the entry set: `run.spawned`, `step.attempt.started` (pins, idempotency +key, lease), `step.completed` (`completionReason`, `disposition`, memoized +`output`, budget), waits/sleeps, `stream.appended`, `effect.recorded` / +`effect.confirmed`, `memory.injected`, `epoch.summary`, `segment.closed`, +`run.completed` [17]. Additive-only payload fields with versioned readers +(claude's mitigation for schema drift, which is the mechanism RFC §7 leaves +open). One SQLite file per run — a sleeping cell then costs storage only +(decision #15). + +### Phase 2 — write path + +Unchanged in shape: append before the caller observes a result; a failed commit +fails the step with no fallback [22]. Effects stay two-phase until Gate 4 makes +the mount the writer and the election *is* the record. Until then, document the +window rather than claiming exactly-once. + +### Phase 3 — retrieval / resume + +`resume` = open run file, load spec, fold current segment via `RunState::fold`, +run recovery for dead attempts (`crashed` / `lease_expired` unless a live +worker holds the lease), dispatch only unfinished work [18][19][20]. Completed +steps become `Done` with `output` injected as fact: zero tokens, zero new +entries. Open waits re-arm, elapsed timers fire. Channel replay reads recorded +deliveries in journal sequence and does not re-invoke receive [25]. Memory +reuses the journaled pack rather than re-calling the provider [26] — noting the +provider is currently `FixedMemoryProvider`, a substrate stub, and its being a +stub says nothing about retrieval quality. + +### Phase 4 — consolidation / forgetting + +Segment-per-epoch. Rollover appends `segment.closed` + `epoch.summary` in one +transaction; closed segments are never rewritten [16][17]. `segment.rs` and the +`rollover_is_atomic_scaffolding_for_epoch_resume` test exist [23]; Gate-5 +archival to relayhistory is specified but has no live reader — that is the gap. +Add the fold-vs-summary equality check here. + +### Phase 5 — evaluation + +`kernel/relayflowd/tests/crash_resume.rs` [24] is the gate, not a nice-to-have: +SIGKILL at every hello-ladder boundary and mid-step, then `resume`; assert +completed marker effects are not re-executed, journal attempt counts match +(`assert_exact_journal`), and mid-step dead attempts are explained and retried. +Extend with: epoch-boundary crashes, channel-replay crashes, an +unconfirmed-election regression test, and the resumed-spend invariant stated as +an explicit assertion. `SimClock` tests pin the machine and do not substitute +for SQLite crash injection. claude's suggested "replay-divergence checks" +apply only to any code-replay component we adopt — we have none, so the +equivalent here is the fold-vs-summary check. + +### Phase 6 — boundary discipline + +Journal protocol v0 (`packages/sdk/src/protocol.ts` [27], `kernel/DESIGN.md` +§5) is the product boundary: SDKs speak it, nothing reaches around it. Grok's +framing is right — the protocol, not the storage engine, is what would be lost +by adopting a vendor runtime. + +## 6. Leverage, ranked + +1. **This kernel's journal + `RunState::fold`** [16][17][18] — *use as-is*; it + is the recommendation, already implemented. (claude, codex, grok) +2. **Crash-injection suite** [24] — the evaluation harness; extend, never + replace with a vendor replay tester. (grok, codex) +3. **SQLite WAL + `synchronous=FULL`, one file per run** [22] — keep; + fail-closed append matched to decision #15. (codex, grok) +4. **Journal protocol v0** [27] — the product boundary. (grok) +5. **Two-phase effects** [21] — keep; honest about the crash window, closer to + exactly-once than at-least-once-and-hope. (codex, grok) +6. **Inngest step memoization** [5][6] — closest commercial analog; steal the + *explanation* (including their runtime-defined step names for Gate 4 loops), + not the service. HTTP-invoke model, no artifact/pins/budget kernel. (codex, + grok) +7. **Fowler event sourcing + external gateways** [14] — the canonical prior art + for "replay results, disable external gateways"; already encoded here. + (grok) +8. **DBOS Transact** [9][10] — Apache-2.0, in-process library, Postgres + checkpoints. Good study material for checkpoint/dedupe SQL; adopting it + means abandoning the Rust/SQLite kernel and it gives no `llm`/`agent` rails, + pins, or protocol. Note it *does* re-enter workflow code (§3.1). (claude, + codex, grok) +9. **ZenML "No Journal, No Replay" / Kitaru** [13] — no code to adopt; the best + external write-up of our own position. (claude) +10. **Restate journals + Virtual Objects** [7][8] — similar durability, wrong + protocol; an extra runtime in front of services that would replace + `relayflowd`. License not re-verified by any lane. (claude, codex, grok) +11. **Hatchet** [11] — Postgres task log, explicitly at-least-once; fine as a + queue, weaker effect story than Appendix A. (grok) +12. **12-factor-agents factor 5** [15] — aligned at slogan level; useful + framing, no implementation. (grok) +13. **Temporal** [1][2][3][4] — *do not adopt*; contradicts decision #2. Its + genuine advantages (fine-grained locals inside one long function without + declaring a spec step, signals/queries as first-class, multi-language + workers, Nexus) are the honest case against us, and are worth naming in + docs rather than eliding. (claude, codex, grok) +14. **LangGraph checkpointers** [12] — snapshots, not a run journal; not + durable by default. Do not use as the kernel store. (grok) +15. **AWS Step Functions / Azure Durable Functions** — cloud-locked, wrong + model for a self-hostable multi-language kernel. Ranked last and cited by + claude only, whose sources for both were unverified — no §8 entry. + +## 7. Open questions + +1. **Has a live resident run crossed an epoch boundary on a new + `journal_version`?** RFC §7 leaves spec/journal/protocol versioning open + until one has. "Old readers, not old code" is our claimed escape from + Temporal's patching and is unproven here. (claude, grok) +2. **Do epoch summaries provably match the folded log?** No check exists. + Drift means skipped or double-run steps on resume. (grok) +3. **Can every adapter enforce provider idempotency?** A stable key is worth + nothing against a provider that ignores it; which adapters actually pass it + through? (codex) +4. **What retention and resume-latency bounds pass crash injection?** No + quantified resume-latency comparison against Temporal-style replay exists — + decision #2 rests on qualitative reasoning. That is defensible, but say so. + (claude, codex) +5. **When does Gate 4 collapse elect/confirm into "the mount write is the + effect record"?** Until then the crash window is real. (grok) +6. **Channel compaction.** Replay scans retained segments with no bounded + snapshot; resident runs will hit this. (grok) +7. **Dynamic steps inside an agent loop.** Can a Gate 4 loop journal iteration + N as data without minting a new content-addressed spec digest per iteration + (decision #14)? (grok) +8. **Does `inspect` recovery into a dirty workspace preserve the budget + invariant** when `verification_failed` retries charge each attempt? (grok) +9. **How does compaction handle a step whose output payload schema changes + across a kernel upgrade** in practice? (claude) +10. **Is embedding Restate/DBOS for a deterministic-step subset ever worth it?** + Grok's own answer — probably not, the protocol would fork — is convincing; + left open only because nobody has priced it. (grok) + +## 8. Sources + +Union of sources at least one lane marked **verified**. Sources the lanes +listed as unverified are excluded (Vanlightly's determinism essay, the DBOS +VLDB paper, Cadence replayer, ARIES, CQRS, FoundationDB, TigerBeetle, Temporal's +dynamic-agents blog, Restate's immutability post, Azure Durable Functions, +Restate Go durable-steps) except where I fetched them myself — see [9]. + +1. https://docs.temporal.io/encyclopedia/event-history/ — Event History as + durable log; Commands mapped to Events; recovery by replaying code. + (claude, grok) +2. https://docs.temporal.io/workflows — resume re-runs Workflow code from the + top against history; Activities are not re-executed. (grok) +3. https://docs.temporal.io/workflow-definition — determinism constraints, + Command/Event matching, non-deterministic errors, versioning. (codex, grok) +4. https://docs.temporal.io/develop/go/workflows/versioning — `GetVersion` / + Patch API for replay-safe code evolution. (claude) +5. https://www.inngest.com/docs/learn/how-functions-are-executed — step + memoization; explicitly distinguished from Temporal's model; each step a + separate invocation. (codex, grok) +6. https://www.inngest.com/docs/learn/durable-agents — calls memoization + "deterministic replay"; runtime-defined steps in agent loops. (grok) +7. https://restate.dev/what-is-durable-execution — journaled steps; restart and + replay recorded results. (grok) +8. https://docs.restate.dev/ai/patterns/durable-agents — the same applied to + LLM/tool calls. (codex) +9. https://docs.dbos.dev/architecture — Postgres checkpoints; recovery restarts + the workflow function with checkpointed inputs and short-circuits + checkpointed steps; workflow must be deterministic given step outputs. + (grok; **re-fetched by the editor** to resolve §3.1) +10. https://www.dbos.dev/blog/postgres-is-all-you-need-for-durable-execution — + workers checkpoint steps to Postgres; recovery from checkpoints; Postgres + constraints dedupe concurrent attempts. (claude) +11. https://docs.hatchet.run/v1/architecture-and-guarantees — Postgres state; + at-least-once; tasks must be idempotent. (grok) +12. https://docs.langchain.com/oss/python/langgraph/persistence — checkpointers + as graph-state snapshots; in-memory saver is not durable. (grok) +13. https://www.zenml.io/blog/no-journal-replay — the case against journal-replay + for AI agents; Kitaru caches step outputs in an artifact store. (claude) +14. https://martinfowler.com/eaaDev/EventSourcing.html — event log as source of + truth; external gateways must be gated on replay. (grok) +15. https://github.com/humanlayer/12-factor-agents/blob/main/content/factor-05-unify-execution-state.md + — unify execution and business state; resume by loading the thread. (grok) +16. `/private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md` + — decision #2 (no deterministic replay), journal + memoization, Appendix A, + epoch compaction, decisions #8/#14/#15. (claude, grok) +17. `/private/tmp/ws13-gallery-followup/kernel/DESIGN.md` — entry types, SQLite + schema, memoized resume algorithm, protocol v0, elect/perform/confirm. + (codex, grok) +18. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs`, + `machine.rs` — `RunState::fold`; pure state machine on a `Clock` trait. + (grok) +19. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/machine/recovery.rs` + — dead attempts resolved to `crashed` / `lease_expired`. (grok) +20. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine.rs` — + `resume` / `resume_filtered`. (grok) +21. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine/effects.rs` + — two-phase election and confirmation. (grok) +22. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/append.rs` + — immediate transaction, fail-closed append. (grok) +23. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/segment.rs`, + `lib.rs` — segment rollover; + `rollover_is_atomic_scaffolding_for_epoch_resume`. (grok) +24. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/tests/crash_resume.rs` + — SIGKILL at step boundaries then `resume`; completed effects must not be + replayed as code; `assert_exact_journal`. (grok) +25. `/private/tmp/ws13-gallery-followup/kernel/DURABLE-CHANNELS.md` — replay + recorded deliveries; do not re-execute receive. (grok) +26. `/private/tmp/ws13-gallery-followup/kernel/MEMORY.md` — journaled pack + reused on resume; `FixedMemoryProvider` is a stub. (grok) +27. `/private/tmp/ws13-gallery-followup/packages/sdk/src/protocol.ts` — verb set + including `run.resume`, `effect.record` / `confirm`, `journal.read`. (grok) diff --git a/docs/evidence/ws13/followup/final-sdk/artifact.json b/docs/evidence/ws13/followup/final-sdk/artifact.json new file mode 100644 index 00000000..9acce654 --- /dev/null +++ b/docs/evidence/ws13/followup/final-sdk/artifact.json @@ -0,0 +1,4 @@ +{ + "file": "/tmp/ws13-followup-artifacts/relayflows-sdk-2.0.8.tgz", + "sha256": "ba58cee2b966299e3c224c20097d8f336848aebbca26cf092a02aade6ea41bac" +} diff --git a/docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt b/docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt new file mode 100644 index 00000000..0b5e7547 --- /dev/null +++ b/docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt @@ -0,0 +1,8 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-followup/node_modules/relayflows/bin/flows.js run examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts --local-agent --input '{}' --data-dir /tmp/ws13-followup-upgrade-daemon +OUTER_TIMEOUT_SECONDS=120 +REFUSED [invalid_invocation] Usage: flows check [--json] flows run [--json] [--no-spawn] [--data-dir ] flows run [--json] [--no-spawn] [--data-dir ] --input flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] flows resume [--json] [--no-spawn] [--data-dir ] flows hn-monitor start [--data-dir ] [--poll-interval-ms ] + +EXIT_CODE=2 +ELAPSED_SECONDS=0.138 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt b/docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt new file mode 100644 index 00000000..d75bdb2f --- /dev/null +++ b/docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt @@ -0,0 +1,8 @@ +$ cd /private/tmp/ws13-gallery-followup +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-followup/node_modules/relayflows/bin/flows.js run examples/pr-review-pipeline/pr-review-pipeline.flow.ts --local-agent --input '{"diffRange":"origin/main...HEAD"}' --data-dir /tmp/ws13-followup-review-daemon +OUTER_TIMEOUT_SECONDS=120 +REFUSED [invalid_invocation] Usage: flows check [--json] flows run [--json] [--no-spawn] [--data-dir ] flows run [--json] [--no-spawn] [--data-dir ] --input flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] flows resume [--json] [--no-spawn] [--data-dir ] flows hn-monitor start [--data-dir ] [--poll-interval-ms ] + +EXIT_CODE=2 +ELAPSED_SECONDS=0.143 +TIMED_OUT=False diff --git a/docs/evidence/ws13/followup/final-sdk/gallery-results.json b/docs/evidence/ws13/followup/final-sdk/gallery-results.json new file mode 100644 index 00000000..73b81a5e --- /dev/null +++ b/docs/evidence/ws13/followup/final-sdk/gallery-results.json @@ -0,0 +1,14 @@ +[ + { + "example": "dependency-upgrade-bot", + "exitCode": 2, + "elapsedSeconds": 0.138, + "timedOut": false + }, + { + "example": "pr-review-pipeline", + "exitCode": 2, + "elapsedSeconds": 0.143, + "timedOut": false + } +] diff --git a/docs/evidence/ws13/followup/heartbeat-abort.txt b/docs/evidence/ws13/followup/heartbeat-abort.txt new file mode 100644 index 00000000..52b09530 --- /dev/null +++ b/docs/evidence/ws13/followup/heartbeat-abort.txt @@ -0,0 +1,16 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ node node_modules/vitest/vitest.mjs run tests/worker-cli-abort.test.ts --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/worker-cli-abort.test.ts > stops claude and its process group when lease ownership is lost 1500ms + ✓ tests/worker-cli-abort.test.ts > stops wrapper.mjs and its process group when lease ownership is lost 1352ms + + Test Files 1 passed (1) + Tests 2 passed (2) + Start at 21:05:35 + Duration 7.60s (transform 1.23s, setup 0ms, collect 629ms, tests 2.85s, environment 1ms, prepare 1.76s) + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/heartbeat-live.txt b/docs/evidence/ws13/followup/heartbeat-live.txt new file mode 100644 index 00000000..44e07944 --- /dev/null +++ b/docs/evidence/ws13/followup/heartbeat-live.txt @@ -0,0 +1,19 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1695ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 35728ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > renders actual agent completion in text output 639ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > returns a failed run when the agent process fails 598ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 21:03:54 + Duration 41.92s (transform 60ms, setup 0ms, collect 61ms, tests 38.84s, environment 0ms, prepare 164ms) + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/heartbeat-packed.txt b/docs/evidence/ws13/followup/heartbeat-packed.txt new file mode 100644 index 00000000..b99737bf --- /dev/null +++ b/docs/evidence/ws13/followup/heartbeat-packed.txt @@ -0,0 +1,15 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ FLOWS_TEST_CLI=/tmp/ws13-gallery-followup/node_modules/@relayflows/sdk/dist/cli.js RELAYFLOWD_BIN=/tmp/ws13-gallery-followup/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts -t 'runs beyond the initial 30-second lease' --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 36467ms + + Test Files 1 passed (1) + Tests 1 passed | 4 skipped (5) + Start at 21:10:58 + Duration 37.21s (transform 28ms, setup 0ms, collect 21ms, tests 36.47s, environment 0ms, prepare 80ms) + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/heartbeat-tests.txt b/docs/evidence/ws13/followup/heartbeat-tests.txt new file mode 100644 index 00000000..0c81dfd3 --- /dev/null +++ b/docs/evidence/ws13/followup/heartbeat-tests.txt @@ -0,0 +1,31 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ node node_modules/vitest/vitest.mjs run tests/worker-lease.test.ts tests/worker-cli.test.ts --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/worker-lease.test.ts > worker lease ownership > renews the same attempt through a long subprocess and drains before completing once + ✓ tests/worker-lease.test.ts > worker lease ownership > aborts execution and never completes after a rejected heartbeat + ✓ tests/worker-lease.test.ts > worker lease ownership > expires locally when a renewal response never arrives, without stranding close + ✓ tests/worker-lease.test.ts > worker lease ownership > does not spawn a process for an already-expired dispatch + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > passes an explicit safe environment at identification and execution 502ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 459ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > bounds wrapper execution after acknowledgement 441ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > bounds captured wrapper output 459ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > refuses a duplicate execute protocol frame + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2036ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1854ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3269ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11262ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > accepts an execute token and an over-8KiB payload flushed in one write 324ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 878ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > still bounds an un-terminated handshake buffer and names the bound 618ms + ✓ tests/worker-cli.test.ts > delivers the journaled memory pack to the real wrapper and excludes its charge from completion usage 395ms + + Test Files 2 passed (2) + Tests 17 passed (17) + Start at 21:03:49 + Duration 30.03s (transform 947ms, setup 0ms, collect 1.06s, tests 22.80s, environment 0ms, prepare 2.64s) + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/kernel-case-retry.txt b/docs/evidence/ws13/followup/kernel-case-retry.txt new file mode 100644 index 00000000..66b5c68e --- /dev/null +++ b/docs/evidence/ws13/followup/kernel-case-retry.txt @@ -0,0 +1,19 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/live-kernel.test.ts -t 'follows a live worker dispatch through flows run' --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > follows a live worker dispatch through flows run 655ms + + Test Files 1 passed (1) + Tests 1 passed | 29 skipped (30) + Start at 21:10:35 + Duration 1.99s (transform 398ms, setup 0ms, collect 826ms, tests 657ms, environment 0ms, prepare 90ms) + + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/followup/kernel-suite.txt b/docs/evidence/ws13/followup/kernel-suite.txt new file mode 100644 index 00000000..59ba39ce --- /dev/null +++ b/docs/evidence/ws13/followup/kernel-suite.txt @@ -0,0 +1,70 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/live-kernel.test.ts --reporter verbose + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5240ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32610ms + × tests/live-kernel.test.ts > built flows CLI against live relayflowd > follows a live worker dispatch through flows run + → Test timed out in 5000ms. +If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout". + × tests/live-kernel.test.ts > built flows CLI against live relayflowd > follows a live worker dispatch through flows run 5285ms + → Test timed out in 5000ms. +If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout". + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > cancels over the real socket and rejects the lease holder after closure + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs an agent CLI end to end through the SDK worker 867ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 598ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > f.agent's default flowPath anchors on cwd, not cwd's parent 487ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5716ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 4880ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) 841ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields 727ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text 524ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) 470ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) 633ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 857ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 491ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 567ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 376ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 408ms +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is directly relevant to AI agents and automation as it describes an autonomous agent system that performs self-directed software development tasks—specifically opening and reviewing pull requests without human intervention, which exemplifies the core capabilities of AI automation in development workflows.","relevance_score":10,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 31101ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir + ✓ tests/live-kernel.test.ts > JournalClient wire conformance against live relayflowd > exercises every protocol-v0 verb with the real server +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=67665 run=01M216RS49MCJ1YNRZW811WATA while step=two state=Running + + ✓ tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 378ms + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant 397ms + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > TWO ticks for ONE scheduled instant produce exactly ONE run + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a poller RESTART re-emitting a slot does not re-run it + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a MISSED interval is backfilled into its own run, not collapsed into the current one + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a tick for a DIFFERENT schedule id does not wake this flow + ✓ tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > journals the declared silence budget, so a dead schedule is not silently zero + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > follows a live worker dispatch through flows run +Error: Test timed out in 5000ms. +If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout". +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 29 passed (30) + Start at 21:07:16 + Duration 100.83s (transform 1.35s, setup 0ms, collect 5.29s, tests 94.45s, environment 0ms, prepare 569ms) + + +EXIT_CODE=1 diff --git a/docs/evidence/ws13/followup/run-gallery.py b/docs/evidence/ws13/followup/run-gallery.py index 38679196..f035e078 100644 --- a/docs/evidence/ws13/followup/run-gallery.py +++ b/docs/evidence/ws13/followup/run-gallery.py @@ -1,18 +1,23 @@ """Capture each requested gallery invocation, including nonzero exits and timeouts. -Usage: python3 run-gallery.py /absolute/gallery-clone /absolute/evidence-directory [research-default] +Usage: python3 run-gallery.py /absolute/gallery-clone /absolute/evidence-directory [research-default|sdk-only] """ from pathlib import Path import json import os import shlex import signal +import shutil import subprocess import sys import time root, evidence = (Path(p).resolve() for p in sys.argv[1:3]) evidence.mkdir(parents=True, exist_ok=True) -node = '/tmp/ws13-toolchain/node' +if any(evidence.glob('gallery-*.txt')): + raise SystemExit('Choose an empty evidence directory; existing captures will not be overwritten.') +node = shutil.which('node') +if node is None: + raise SystemExit('Node 22.18+ is required on PATH.') cli = str(root / 'node_modules/relayflows/bin/flows.js') cases = [ ('dependency-upgrade-bot', 120, [node, cli, 'run', @@ -27,7 +32,9 @@ 'Compare durable step journals with deterministic replay. Keep every report under 200 words.', '--timeout-minutes', '3', '--runs-dir', '/tmp/ws13-research-followup-runs']), ] -if sys.argv[3:] == ['research-default']: +if sys.argv[3:] == ['sdk-only']: + cases = cases[:2] +elif sys.argv[3:] == ['research-default']: command = cases[-1][2].copy() command[command.index('ws13-followup')] = 'ws13-default-budget' index = command.index('--timeout-minutes') @@ -41,7 +48,7 @@ output.flush() started = time.monotonic() process = subprocess.Popen(command, cwd=root, - env={**os.environ, 'PATH': '/tmp/ws13-toolchain:' + os.environ['PATH']}, + env=os.environ.copy(), stdout=output, stderr=subprocess.STDOUT, start_new_session=True) timed_out = False try: diff --git a/docs/evidence/ws13/followup/sdk-typechecks.txt b/docs/evidence/ws13/followup/sdk-typechecks.txt new file mode 100644 index 00000000..6f8aa186 --- /dev/null +++ b/docs/evidence/ws13/followup/sdk-typechecks.txt @@ -0,0 +1,4 @@ +$ cd packages/sdk +$ export PATH=/tmp/ws13-toolchain:$PATH +$ node node_modules/typescript/bin/tsc --noEmit && node node_modules/typescript/bin/tsc -p tsconfig.type-tests.json && node node_modules/typescript/bin/tsc -p tsconfig.tests.json +EXIT_CODE=0 diff --git a/docs/evidence/ws13/stage-registry.mjs b/docs/evidence/ws13/stage-registry.mjs index 0fb634a5..18abbb32 100644 --- a/docs/evidence/ws13/stage-registry.mjs +++ b/docs/evidence/ws13/stage-registry.mjs @@ -1,5 +1,5 @@ // Serve packed candidate packages locally; redirect other dependencies to npm. -// Usage: node stage-registry.mjs /absolute/artifact-directory [port] +// Usage: node stage-registry.mjs /absolute/artifact-directory [port] [bind-host; default 127.0.0.1] import { createServer } from 'node:http'; import { createHash } from 'node:crypto'; import { readFileSync, readdirSync } from 'node:fs'; @@ -17,10 +17,17 @@ for (const file of readdirSync(directory).filter(file => file.endsWith('.tgz'))) tarballs.set(`/tarballs/${file}`, data); } createServer((req, res) => { - const url = new URL(req.url, `http://${req.headers.host}`); + let url, name; + try { + url = new URL(req.url, `http://${req.headers.host}`); + name = decodeURIComponent(url.pathname.slice(1)); + } catch { + res.writeHead(400); + res.end('Bad request'); + return; + } const tarball = tarballs.get(url.pathname); if (tarball) { res.end(tarball); return; } - const name = decodeURIComponent(url.pathname.slice(1)); const candidate = packages.get(name); if (!candidate) { res.writeHead(302, { location: `https://registry.npmjs.org${req.url}` }); @@ -32,4 +39,4 @@ createServer((req, res) => { res.end(JSON.stringify({ name, 'dist-tags': { latest: manifest.version }, versions: { [manifest.version]: { ...manifest, dist: { tarball: `${url.origin}/tarballs/${file}`, integrity } }, } })); -}).listen(Number(process.argv[3] ?? 48731), '0.0.0.0', () => console.log('Candidate registry ready')); +}).listen(Number(process.argv[3] ?? 48731), process.argv[4] ?? '127.0.0.1', () => console.log('Candidate registry ready')); diff --git a/examples/README.md b/examples/README.md index 340a9e21..bf73b77a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,15 +1,16 @@ # Example gallery status -The three entries below are the requested WS-13 gallery scope. They are +**1 of 3 requested entries passed.** The two blocked entries fail explicitly +before their bodies execute. The entries below are advanced examples, not a promise that every surface feature is executable. For a working local starting point, use the [small agent starter](../README.md) ([recorded run](../docs/evidence/ws13/agent-run.txt)). | Example | Status | Observed result | Elapsed | |---|---|---|---:| -| [dependency-upgrade-bot](dependency-upgrade-bot/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [1.224s](../docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt) | -| [pr-review-pipeline](pr-review-pipeline/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.252s](../docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt) | -| [research](research/) | **RETRY IN PROGRESS** | Preflight passed; first run failed at the supplied 3-minute step limit. Retrying with the documented default budget | [217.375s for the first attempt](../docs/evidence/ws13/followup/gallery-research.txt) | +| [dependency-upgrade-bot](dependency-upgrade-bot/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.138s](../docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt) | +| [pr-review-pipeline](pr-review-pipeline/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.143s](../docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt) | +| [research](research/) | **PASS** | All model probes passed; three lane reports and synthesis produced; exit 0, `completionReason: synthesized` | [690.935s](../docs/evidence/ws13/followup/default-budget/gallery-research.txt) | Each link contains the literal command, captured output, exit code and timing. These are individual runs from a separate clone on an authenticated macOS @@ -24,8 +25,10 @@ branch leaves them intact. The local agent worker handles stream-only steps and cannot supply workspace isolation. **Research now prints provider preflight activity.** Each CLI/model probe names -its timeout on stderr, while stdout remains the final structured result. All -four model probes passed in the first follow-up run; the three-minute limit -then expired during research. That failure is not evidence of missing provider -authentication or an unsupported model. The earlier outer timeout is retained -in the [historical evidence](../docs/evidence/ws13/gallery-research.txt). +its timeout on stderr, while stdout remains the final structured result. +The run with the documented default budget completed in 690.935s; its +[three reports and synthesis](../docs/evidence/ws13/followup/default-budget/reports/) +are captured with [artifact hashes](../docs/evidence/ws13/followup/default-budget/artifacts.json). +The first follow-up's shorter three-minute step limit expired after 217.375s +([captured failure](../docs/evidence/ws13/followup/gallery-research.txt)). That +attempt is not evidence of missing authentication or an unsupported model. diff --git a/examples/dependency-upgrade-bot/README.md b/examples/dependency-upgrade-bot/README.md index 18ac4c11..7d60217d 100644 --- a/examples/dependency-upgrade-bot/README.md +++ b/examples/dependency-upgrade-bot/README.md @@ -1,8 +1,8 @@ # dependency-upgrade-bot **BLOCKED — not runnable on the current authored executor.** The candidate -CLI refuses the `budget` header before any step runs (exit 2, 1.224s). -[Exact command and captured output](../../docs/evidence/ws13/followup/gallery-dependency-upgrade-bot.txt). +CLI refuses the `budget` header before any step runs (exit 2, 0.138s). +[Exact command and captured output](../../docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt). The SDK/kernel capability owner must supply budget-header support, postfix artifact gates, and the declared workspace behavior before this example can be advertised as working. Its existing requirements remain intact. diff --git a/examples/pr-review-pipeline/README.md b/examples/pr-review-pipeline/README.md index f60d85d0..82eca89a 100644 --- a/examples/pr-review-pipeline/README.md +++ b/examples/pr-review-pipeline/README.md @@ -1,8 +1,8 @@ # pr-review-pipeline **BLOCKED — not runnable on the current authored executor.** The candidate -CLI refuses the `budget` header before any step runs (exit 2, 0.252s). -[Exact command and captured output](../../docs/evidence/ws13/followup/gallery-pr-review-pipeline.txt). +CLI refuses the `budget` header before any step runs (exit 2, 0.143s). +[Exact command and captured output](../../docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt). The SDK/kernel capability owner must supply budget-header support, postfix artifact gates, and the declared workspace behavior before this example can be advertised as working. Its existing requirements remain intact. diff --git a/examples/research/README.md b/examples/research/README.md index a805f867..823001ed 100644 --- a/examples/research/README.md +++ b/examples/research/README.md @@ -1,5 +1,13 @@ # examples/research — a fan-out research relayflow, authored on flows v2 +**PASS — 690.935s with the documented default budget**, using authenticated +Claude, Codex and Grok CLIs. The run produced three lane reports and a synthesis, +with `completionReason: synthesized` and exit 0. +[Command and full output](../../docs/evidence/ws13/followup/default-budget/gallery-research.txt), +[generated reports](../../docs/evidence/ws13/followup/default-budget/reports/). +An earlier verification attempt used a three-minute step limit and timed out; +that shorter limit was not enough for this multi-agent research workload. + Give it a research question. It fans the question out to three independent model lanes — **Claude** (sonnet), **Codex**, and **Grok** — each of which spawns **two subagents** (a *landscape* researcher over papers, frameworks, diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 70cbd935..77eb828e 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -32,7 +32,9 @@ export async function runAgentCli( wakeContext: unknown, model?: string, wrapperLimits?: Partial, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const kind = cliAdapterKind(cli); if (kind === 'relayflows-wrapper-v1') { @@ -43,6 +45,7 @@ export async function runAgentCli( model, wrapperEnvironment(process.env), wrapperLimits, + signal, ); } @@ -64,16 +67,20 @@ export async function runAgentCli( } if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - return spawnInvocation(cli, invocation, env); + return spawnInvocation(cli, invocation, env, signal); } function spawnInvocation( cli: string, invocation: CliInvocation, env: NodeJS.ProcessEnv, + signal?: AbortSignal, ): Promise { return new Promise((resolve) => { - const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); + const child = spawn(cli, invocation.args, { + stdio: ['ignore', 'pipe', 'pipe'], env, + detached: signal !== undefined && process.platform !== 'win32', + }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; let settled = false; @@ -82,8 +89,17 @@ function spawnInvocation( if (settled) return; settled = true; if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); resolve(result); }; + const onAbort = (): void => { + if (child.pid !== undefined && process.platform !== 'win32') { + try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } + } else child.kill('SIGKILL'); + finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' }); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) onAbort(); child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); child.once('error', (error) => finish({ diff --git a/packages/sdk/src/worker-lease.ts b/packages/sdk/src/worker-lease.ts new file mode 100644 index 00000000..6ce68561 --- /dev/null +++ b/packages/sdk/src/worker-lease.ts @@ -0,0 +1,73 @@ +import type { JournalClient } from './journal-client.js'; +import type { StepDispatchEvent } from './protocol.js'; + +/** Hold the dispatched lease only while its subprocess is still ours to run. */ +export async function withWorkerLease( + client: JournalClient, + dispatch: StepDispatchEvent, + execute: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + let stopped = false; + let renewalTimer: NodeJS.Timeout | undefined; + let expiryTimer: NodeJS.Timeout | undefined; + let pending: Promise = Promise.resolve(); + const fail = (error: unknown): void => { controller.abort(error); }; + const armExpiry = (deadline: number): number => { + const remaining = deadline - Date.now(); + if (!Number.isFinite(remaining) || remaining <= 0) { + throw new Error(`Agent lease is already expired for ${dispatch.run_id}/${dispatch.step_id}.`); + } + if (expiryTimer !== undefined) clearTimeout(expiryTimer); + expiryTimer = setTimeout(() => fail(new Error( + `Agent lease expired before renewal for ${dispatch.run_id}/${dispatch.step_id}.`, + )), remaining); + return remaining; + }; + const renew = async (): Promise => { + const result = await untilAborted(client.stepHeartbeat( + dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.lease_id, + ), controller.signal); + controller.signal.throwIfAborted(); + const remaining = armExpiry(result.lease_deadline_ms); + if (!stopped) { + renewalTimer = setTimeout(() => { + pending = renew().catch(fail); + }, Math.max(1, Math.floor(remaining / 3))); + } + }; + try { + armExpiry(dispatch.lease_deadline_ms); + // Establish ownership before starting an effectful process. + await renew(); + const result = await execute(controller.signal); + stopped = true; + if (renewalTimer !== undefined) clearTimeout(renewalTimer); + // Drain any renewal before the caller sends step.complete. A renewal + // racing after completion would otherwise report a spurious lease error. + await pending; + controller.signal.throwIfAborted(); + return result; + } finally { + stopped = true; + controller.abort(new Error('Worker lease scope ended.')); + if (renewalTimer !== undefined) clearTimeout(renewalTimer); + if (expiryTimer !== undefined) clearTimeout(expiryTimer); + await pending; + } +} + +function untilAborted(request: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = (): void => { reject(signal.reason); }; + signal.addEventListener('abort', abort, { once: true }); + request.then(value => { + signal.removeEventListener('abort', abort); + resolve(value); + }, error => { + signal.removeEventListener('abort', abort); + reject(error); + }); + if (signal.aborted) abort(); + }); +} diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 9f338936..a5fde8b3 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -3,6 +3,7 @@ import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; import type { KernelAgentStep } from './spec.js'; import { runAgentCli } from './worker-cli.js'; +import { withWorkerLease } from './worker-lease.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -90,9 +91,10 @@ export class AgentWorker extends EventEmitter { private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; - const result = typeof spec.cli === 'string' && typeof spec.instruction === 'string' - ? await runAgentCli(spec.cli, memoryInstruction(spec.instruction, dispatch.memory), dispatch.wake_context, spec.model) - : { exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' }; + const result = await withWorkerLease(this.client, dispatch, signal => + typeof spec.cli === 'string' && typeof spec.instruction === 'string' + ? runAgentCli(spec.cli, memoryInstruction(spec.instruction, dispatch.memory), dispatch.wake_context, spec.model, undefined, signal) + : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; // Output shape: if the CLI's stdout parses as JSON, promote THAT diff --git a/packages/sdk/src/wrapper-session.ts b/packages/sdk/src/wrapper-session.ts index 9d086d1e..82d5493d 100644 --- a/packages/sdk/src/wrapper-session.ts +++ b/packages/sdk/src/wrapper-session.ts @@ -48,7 +48,9 @@ export function runWrapperSession( model: string | undefined, env: NodeJS.ProcessEnv, overrides: Partial = {}, + signal?: AbortSignal, ): Promise { + if (signal?.aborted) return Promise.reject(signal.reason); const limits = sessionLimits(overrides); let request: string; try { @@ -72,7 +74,7 @@ export function runWrapperSession( )); } - return executePinnedWrapper(cli, identity, request, env, limits); + return executePinnedWrapper(cli, identity, request, env, limits, signal); } function executePinnedWrapper( @@ -81,11 +83,13 @@ function executePinnedWrapper( request: string, env: NodeJS.ProcessEnv, limits: WrapperSessionLimits, + signal?: AbortSignal, ): Promise { return new Promise((resolve) => { const child = spawn(identity.executable, [WRAPPER_IDENTIFY_ARG], { stdio: ['pipe', 'pipe', 'pipe'], env, + detached: signal !== undefined && process.platform !== 'win32', }); const stdout: string[] = []; const stderr: Buffer[] = []; @@ -108,8 +112,17 @@ function executePinnedWrapper( if (settled) return; settled = true; clearTimers(); + signal?.removeEventListener('abort', onAbort); resolve(result); }; + const onAbort = (): void => { + if (child.pid !== undefined && process.platform !== 'win32') { + try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } + } else child.kill('SIGKILL'); + finish(failure('Agent execution aborted: lease ownership lost.')); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) { onAbort(); return; } const terminate = (message: string): void => { if (protocolError !== undefined) return; protocolError = message; diff --git a/packages/sdk/tests/local-agent-live.test.ts b/packages/sdk/tests/local-agent-live.test.ts index e0c063e2..2f67dc2e 100644 --- a/packages/sdk/tests/local-agent-live.test.ts +++ b/packages/sdk/tests/local-agent-live.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it } from 'vitest'; const roots: string[] = []; const sdk = resolve('.'); -const cli = join(sdk, 'dist/cli.js'); +const cli = process.env['FLOWS_TEST_CLI'] ?? join(sdk, 'dist/cli.js'); const wrapperHelper = resolve('../../testdata/preflight/wrapper-session.mjs'); // Ask the existing build wrapper for its target directory. A temp fixture's // cwd cannot discover the checkout, and test:prep's child-shell exports do not @@ -31,13 +31,13 @@ afterEach(() => { } }); -function fixture(exitCode = 0, workspace?: string) { +function fixture(exitCode = 0, workspace?: string, delayMs = 0) { const root = mkdtempSync(join(tmpdir(), 'flows-local-agent-')); roots.push(root); symlinkSync(join(sdk, 'node_modules'), join(root, 'node_modules')); const marker = join(root, 'invoked'); const wrapper = join(root, 'agent.mjs'); - writeFileSync(wrapper, `#!/usr/bin/env node\nimport { receiveWrapperRequest } from ${JSON.stringify(wrapperHelper)};\nimport { writeFileSync } from 'node:fs';\nif (process.argv[2] === 'auth') process.exit(0);\nconst request = await receiveWrapperRequest();\nif (request) { writeFileSync(${JSON.stringify(marker)}, request.instruction); console.log('local-agent-ok'); process.exit(${exitCode}); }\n`); + writeFileSync(wrapper, `#!/usr/bin/env node\nimport { receiveWrapperRequest } from ${JSON.stringify(wrapperHelper)};\nimport { appendFileSync } from 'node:fs';\nif (process.argv[2] === 'auth') process.exit(0);\nconst request = await receiveWrapperRequest();\nif (request) { appendFileSync(${JSON.stringify(marker)}, request.instruction); await new Promise(resolve => setTimeout(resolve, ${delayMs})); console.log('local-agent-ok'); process.exit(${exitCode}); }\n`); chmodSync(wrapper, 0o755); writeFileSync(join(root, 'flows.json'), JSON.stringify({ cli: wrapper })); writeFileSync(join(root, 'package.json'), '{"type":"module"}'); @@ -59,6 +59,13 @@ describe('built CLI local agent against a real daemon', () => { expect(readFileSync(f.marker, 'utf8')).toBe('hello'); expect(result.stderr).not.toContain('✓'); }); + it('runs beyond the initial 30-second lease without a second invocation', () => { + const f = fixture(0, undefined, 35_000); + const result = f.invoke('--json'); + expect(result.status, result.stderr + result.stdout).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, completionReason: 'success' }); + expect(readFileSync(f.marker, 'utf8')).toBe('hello'); + }, 90_000); it('renders actual agent completion in text output', () => { const result = fixture().invoke(); expect(result.status, result.stderr + result.stdout).toBe(0); diff --git a/packages/sdk/tests/worker-cli-abort.test.ts b/packages/sdk/tests/worker-cli-abort.test.ts new file mode 100644 index 00000000..f8e61176 --- /dev/null +++ b/packages/sdk/tests/worker-cli-abort.test.ts @@ -0,0 +1,43 @@ +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { expect, it } from 'vitest'; +import { runAgentCli } from '../src/worker-cli.js'; + +it.each(['claude', 'wrapper.mjs'])('stops %s and its process group when lease ownership is lost', async name => { + const root = mkdtempSync(join(tmpdir(), 'lease-abort-')); + const controller = new AbortController(); + const parentPid = join(root, 'parent-pid'); + const childPid = join(root, 'child-pid'); + const effect = join(root, 'late-effect'); + const executable = join(root, name); + const helper = resolve('../../testdata/preflight/wrapper-session.mjs'); + const childSource = `require('node:fs').writeFileSync(${JSON.stringify(childPid)}, String(process.pid)); setTimeout(() => require('node:fs').writeFileSync(${JSON.stringify(effect)}, 'unexpected'), 800);`; + writeFileSync(join(root, 'package.json'), '{"type":"module"}'); + writeFileSync(executable, `#!/usr/bin/env node +import { writeFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +${name === 'wrapper.mjs' ? `import { receiveWrapperRequest } from ${JSON.stringify(helper)}; await receiveWrapperRequest();` : ''} +writeFileSync(${JSON.stringify(parentPid)}, String(process.pid)); +spawn(process.execPath, ['-e', ${JSON.stringify(childSource)}], { stdio: 'inherit' }); +setInterval(() => {}, 1000); +`); + chmodSync(executable, 0o755); + try { + const running = runAgentCli(executable, 'hello', undefined, undefined, undefined, controller.signal); + const deadline = Date.now() + 5000; + while (!existsSync(childPid) && Date.now() < deadline) await new Promise(resolveWait => setTimeout(resolveWait, 10)); + expect(existsSync(childPid)).toBe(true); + controller.abort(new Error('lease rejected')); + expect((await running).exit_code).toBeNull(); + await new Promise(resolveWait => setTimeout(resolveWait, 900)); + expect(existsSync(effect)).toBe(false); + for (const file of [parentPid, childPid]) { + const pid = Number(readFileSync(file, 'utf8')); + expect(() => process.kill(pid, 0)).toThrow(); + } + } finally { + controller.abort(); + rmSync(root, { recursive: true, force: true }); + } +}, 10_000); diff --git a/packages/sdk/tests/worker-cli.test.ts b/packages/sdk/tests/worker-cli.test.ts index 6133c69e..481aecb9 100644 --- a/packages/sdk/tests/worker-cli.test.ts +++ b/packages/sdk/tests/worker-cli.test.ts @@ -345,6 +345,7 @@ process.exit(0); const completions: unknown[][] = []; const client = new EventEmitter() as EventEmitter & Record; client.workerAttach = async (): Promise => ({ ok: true }); + client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); client.stepComplete = async (...args: unknown[]): Promise => { completions.push(args); return { ok: true }; @@ -364,7 +365,7 @@ process.exit(0); attempt: 1, step_type: 'agent', spec: { cli: wrapper, instruction: 'instruction' }, - lease_id: 'lease-leak', + lease_id: 'lease-leak', lease_deadline_ms: Date.now() + 30_000, idempotency_key: 'idem-leak', pins: {} as Pins, }); @@ -478,6 +479,7 @@ process.stdin.on('end', () => { const completions: unknown[][] = []; const client = new EventEmitter() as EventEmitter & Record; client.workerAttach = async () => ({}); + client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); client.stepComplete = async (...args: unknown[]) => { completions.push(args); return {}; }; const worker = new AgentWorker(client as unknown as JournalClient, { workerId: 'memory', pins: { workspace: [], streams: [] } }); const errors: unknown[] = []; @@ -487,7 +489,7 @@ process.stdin.on('end', () => { client.emit('step.dispatch', { run_id: 'memory', step_id: 's', attempt: 2, step_type: 'agent', spec: { cli: wrapper, instruction: 'Use context' }, pins: { workspace: [], streams: [] }, - lease_id: 'lease', idempotency_key: 'effect', + lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, idempotency_key: 'effect', memory: { request: { scope: 'agent', query: 'lessons', budget: {} }, pack, budget: { tokens_in: 7, tokens_out: 0, dollars: '0.002' }, provider: 'stub' }, }); diff --git a/packages/sdk/tests/worker-lease.test.ts b/packages/sdk/tests/worker-lease.test.ts new file mode 100644 index 00000000..7c3d70d4 --- /dev/null +++ b/packages/sdk/tests/worker-lease.test.ts @@ -0,0 +1,105 @@ +import { EventEmitter } from 'node:events'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { AgentWorker } from '../src/worker.js'; +import { runAgentCli } from '../src/worker-cli.js'; + +vi.mock('../src/worker-cli.js', () => ({ runAgentCli: vi.fn() })); +afterEach(() => { vi.useRealTimers(); vi.resetAllMocks(); }); + +function setup() { + vi.useFakeTimers(); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({})), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'lease-test', pins: { workspace: [], streams: [] }, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + const dispatch: StepDispatchEvent = { + type: 'step.dispatch', run_id: 'run', step_id: 'agent', attempt: 1, + step_type: 'agent', spec: { cli: 'claude', instruction: 'hello' }, + lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, + idempotency_key: 'effect', pins: { workspace: [], streams: [] }, + }; + return { client, worker, errors, dispatch }; +} + +function runningCli(delay: number): AbortSignal[] { + const signals: AbortSignal[] = []; + vi.mocked(runAgentCli).mockImplementation(async (_cli, _instruction, _wake, _model, _limits, signal) => { + signals.push(signal!); + return new Promise(resolve => { + const timer = setTimeout(() => resolve({ exit_code: 0, stdout_tail: 'hello', stderr_tail: '' }), delay); + signal!.addEventListener('abort', () => { + clearTimeout(timer); + resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'aborted' }); + }, { once: true }); + }); + }); + return signals; +} + +describe('worker lease ownership', () => { + it('renews the same attempt through a long subprocess and drains before completing once', async () => { + const { client, worker, errors, dispatch } = setup(); + runningCli(35_000); + await worker.attach(); + client.emit('step.dispatch', dispatch); + const closing = worker.close(); + await vi.advanceTimersByTimeAsync(35_000); + await closing; + expect(errors).toEqual([]); + expect(runAgentCli).toHaveBeenCalledTimes(1); + expect(client.stepHeartbeat).toHaveBeenCalledTimes(4); + expect(client.stepHeartbeat.mock.calls).toEqual(Array(4).fill(['run', 'agent', 1, 'lease'])); + expect(client.stepComplete).toHaveBeenCalledTimes(1); + expect(client.stepComplete.mock.calls[0]).toEqual(expect.arrayContaining(['success'])); + await vi.advanceTimersByTimeAsync(60_000); + expect(client.stepHeartbeat).toHaveBeenCalledTimes(4); + }); + + it('aborts execution and never completes after a rejected heartbeat', async () => { + const { client, worker, errors, dispatch } = setup(); + const failure = new Error('lease rejected'); + client.stepHeartbeat.mockResolvedValueOnce({ lease_deadline_ms: Date.now() + 30_000 }) + .mockRejectedValueOnce(failure); + const signals = runningCli(60_000); + await worker.attach(); + client.emit('step.dispatch', dispatch); + await vi.advanceTimersByTimeAsync(10_001); + await worker.close(); + expect(signals[0]?.aborted).toBe(true); + expect(errors).toEqual([failure]); + expect(client.stepComplete).not.toHaveBeenCalled(); + }); + + it('expires locally when a renewal response never arrives, without stranding close', async () => { + const { client, worker, errors, dispatch } = setup(); + client.stepHeartbeat.mockResolvedValueOnce({ lease_deadline_ms: Date.now() + 30_000 }) + .mockImplementationOnce(() => new Promise(() => {})); + const signals = runningCli(60_000); + await worker.attach(); + client.emit('step.dispatch', dispatch); + await vi.advanceTimersByTimeAsync(30_001); + await worker.close(); + expect(signals[0]?.aborted).toBe(true); + expect(String(errors[0])).toContain('lease expired before renewal'); + expect(client.stepComplete).not.toHaveBeenCalled(); + }); + + it('does not spawn a process for an already-expired dispatch', async () => { + const { client, worker, errors, dispatch } = setup(); + dispatch.lease_deadline_ms = Date.now() - 1; + await worker.attach(); + client.emit('step.dispatch', dispatch); + await worker.close(); + expect(runAgentCli).not.toHaveBeenCalled(); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(String(errors[0])).toContain('already expired'); + }); +}); From d8fb664a5915fa85d24997ec3b99faf1da6340f5 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 22:11:50 +0200 Subject: [PATCH 10/16] fix: address local execution and evidence review findings Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/cold-clone.sh | 2 +- docs/evidence/ws13/cold-start.sh | 2 +- .../followup/default-budget/artifacts.json | 21 ++++++--- .../followup/default-budget/reports/claude.md | 4 +- .../followup/default-budget/reports/grok.md | 26 +++++------ .../default-budget/reports/synthesis.md | 24 +++++----- docs/evidence/ws13/followup/run-gallery.py | 3 ++ .../ws13/gallery-pr-review-pipeline.txt | 6 --- docs/evidence/ws13/gallery-research.txt | 5 -- docs/evidence/ws13/record.py | 10 +++- docs/evidence/ws13/research-typecheck.txt | 3 -- docs/evidence/ws13/review/build.txt | 6 +++ docs/evidence/ws13/review/cloud-failure.txt | 17 +++++++ docs/evidence/ws13/review/regressions.txt | 46 +++++++++++++++++++ packages/sdk/src/cli.ts | 4 +- packages/sdk/src/cli/direct-run.ts | 5 +- packages/sdk/src/progress.ts | 15 ++++-- packages/sdk/src/worker-cli.ts | 3 ++ packages/sdk/src/worker-lease.ts | 7 +++ packages/sdk/src/wrapper-session.ts | 3 ++ packages/sdk/tests/cli-progress-wait.test.ts | 28 +++++++++++ packages/sdk/tests/direct-run-failure.test.ts | 33 +++++++++++++ packages/sdk/tests/local-agent-live.test.ts | 18 ++++++-- packages/sdk/tests/local-dev-ux.test.ts | 17 ++++++- packages/sdk/tests/worker-lease.test.ts | 14 ++++++ packages/sdk/tests/worker-platform.test.ts | 16 +++++++ 26 files changed, 277 insertions(+), 61 deletions(-) delete mode 100644 docs/evidence/ws13/gallery-pr-review-pipeline.txt delete mode 100644 docs/evidence/ws13/gallery-research.txt delete mode 100644 docs/evidence/ws13/research-typecheck.txt create mode 100644 docs/evidence/ws13/review/build.txt create mode 100644 docs/evidence/ws13/review/cloud-failure.txt create mode 100644 docs/evidence/ws13/review/regressions.txt create mode 100644 packages/sdk/tests/cli-progress-wait.test.ts create mode 100644 packages/sdk/tests/direct-run-failure.test.ts create mode 100644 packages/sdk/tests/worker-platform.test.ts diff --git a/docs/evidence/ws13/cold-clone.sh b/docs/evidence/ws13/cold-clone.sh index 4b75ea3e..fe4aa17b 100644 --- a/docs/evidence/ws13/cold-clone.sh +++ b/docs/evidence/ws13/cold-clone.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -eu -export npm_config_registry="$1" +export npm_config_registry="${1:?Usage: cold-clone.sh CANDIDATE_REGISTRY_URL}" export npm_config_cache=/tmp/ws13-empty-cache export npm_config_audit=false export npm_config_fund=false diff --git a/docs/evidence/ws13/cold-start.sh b/docs/evidence/ws13/cold-start.sh index 89e38a45..dee954bc 100644 --- a/docs/evidence/ws13/cold-start.sh +++ b/docs/evidence/ws13/cold-start.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu # Candidate registry argument supplies this unpublished branch's packed npm artifacts. -export npm_config_registry="$1" +export npm_config_registry="${1:?Usage: cold-start.sh CANDIDATE_REGISTRY_URL}" export npm_config_cache=/tmp/ws13-empty-cache export npm_config_audit=false export npm_config_fund=false diff --git a/docs/evidence/ws13/followup/default-budget/artifacts.json b/docs/evidence/ws13/followup/default-budget/artifacts.json index 6a3e1378..ec8e1e1d 100644 --- a/docs/evidence/ws13/followup/default-budget/artifacts.json +++ b/docs/evidence/ws13/followup/default-budget/artifacts.json @@ -3,8 +3,11 @@ "name": "claude", "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/claude/report.md", "file": "docs/evidence/ws13/followup/default-budget/reports/claude.md", - "bytes": 12957, - "sha256": "a6ecb1f63eed56aa6d1cc79b813c864950dc298fa5a2b5fb7c93d92cd7453f63" + "bytes": 12887, + "sha256": "435dbb3907c4e41a488c493ab7706e1bb4a904f75dced36756f21e54474f8647", + "originalBytes": 12957, + "originalSha256": "a6ecb1f63eed56aa6d1cc79b813c864950dc298fa5a2b5fb7c93d92cd7453f63", + "normalization": "Removed temporary checkout prefix from repository citations; content otherwise unchanged. Paths are relative to repository root at ab1e3ff." }, { "name": "codex", @@ -17,14 +20,20 @@ "name": "grok", "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/grok/report.md", "file": "docs/evidence/ws13/followup/default-budget/reports/grok.md", - "bytes": 23777, - "sha256": "b733b7b6b0bb673dc5d8a2c4ba1b494d343585469f12fbdf507bdf733aa5d05e" + "bytes": 23322, + "sha256": "d0b6245e6e3641c2ae7906ab21e69b4687406b86b052d7e0526213ced037887b", + "originalBytes": 23777, + "originalSha256": "b733b7b6b0bb673dc5d8a2c4ba1b494d343585469f12fbdf507bdf733aa5d05e", + "normalization": "Removed temporary checkout prefix from repository citations; content otherwise unchanged. Paths are relative to repository root at ab1e3ff." }, { "name": "synthesis", "source": "/tmp/ws13-research-followup-runs/2026-09-08-ws13-default-budget/SYNTHESIS.md", "file": "docs/evidence/ws13/followup/default-budget/reports/synthesis.md", - "bytes": 24831, - "sha256": "ebf4fbfddf85693dcd0ac500d41333b7e987a3e8279e730643d1a0834fdba239" + "bytes": 24411, + "sha256": "7d8ed0a19996cd09c42f9a3949fe8e762120db7f6687cda0207ab5351f219574", + "originalBytes": 24831, + "originalSha256": "ebf4fbfddf85693dcd0ac500d41333b7e987a3e8279e730643d1a0834fdba239", + "normalization": "Removed temporary checkout prefix from repository citations; content otherwise unchanged. Paths are relative to repository root at ab1e3ff." } ] diff --git a/docs/evidence/ws13/followup/default-budget/reports/claude.md b/docs/evidence/ws13/followup/default-budget/reports/claude.md index 7d90300c..f8ca1d3e 100644 --- a/docs/evidence/ws13/followup/default-budget/reports/claude.md +++ b/docs/evidence/ws13/followup/default-budget/reports/claude.md @@ -55,5 +55,5 @@ Test both models by construction, not by hope: crash-inject at every step bounda 7. https://www.zenml.io/blog/no-journal-replay — argument against journal-replay for AI agents (Kitaru project) — verified (fetched) 8. https://docs.temporal.io/develop/go/workflows/versioning — Temporal GetVersion/Patch API for replay-safe code evolution — verified (fetched) 9. TigerBeetle deterministic simulation testing blog posts (tigerbeetle.com/blog) — unverified (reported by subagent, not independently fetched) -10. /private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md — local repo RFC documenting the step-journal-over-replay decision — verified (read directly, full text) -11. /private/tmp/ws13-gallery-followup/kernel/DESIGN.md — local kernel journal entry-type design — unverified in this pass (reported by subagent; not independently re-read by lead) +10. docs/RFC-0001-everything-is-a-relayflow.md — local repo RFC documenting the step-journal-over-replay decision — verified (read directly, full text) +11. kernel/DESIGN.md — local kernel journal entry-type design — unverified in this pass (reported by subagent; not independently re-read by lead) diff --git a/docs/evidence/ws13/followup/default-budget/reports/grok.md b/docs/evidence/ws13/followup/default-budget/reports/grok.md index 6ea23497..81205505 100644 --- a/docs/evidence/ws13/followup/default-budget/reports/grok.md +++ b/docs/evidence/ws13/followup/default-budget/reports/grok.md @@ -164,23 +164,23 @@ Control flow lives in the spec + kernel machine, not in user source that must re 7. https://docs.temporal.io/workflow-definition — Determinism constraints; Command/Event matching; non-deterministic errors; versioning. `verified` 8. https://docs.langchain.com/oss/python/langgraph/persistence — Checkpointers as graph-state snapshots; in-memory saver is not durable. `verified` 9. https://www.inngest.com/docs/learn/durable-agents — Calls memoization “deterministic replay”; dynamic agent loops. `verified` -10. `/private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md` — Decision #2 no deterministic replay; journal + memoization; Appendix A; epoch compaction. `verified` -11. `/private/tmp/ws13-gallery-followup/kernel/DESIGN.md` — Entry types, SQLite schema, memoized resume algorithm, protocol v0. `verified` -12. Same as [11] plus `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs`, `machine.rs`. `verified` +10. `docs/RFC-0001-everything-is-a-relayflow.md` — Decision #2 no deterministic replay; journal + memoization; Appendix A; epoch compaction. `verified` +11. `kernel/DESIGN.md` — Entry types, SQLite schema, memoized resume algorithm, protocol v0. `verified` +12. Same as [11] plus `kernel/relayflowd-core/src/state.rs`, `machine.rs`. `verified` 13. https://temporal.io/blog/of-course-you-can-build-dynamic-ai-agents-with-temporal — Agents via Activities. `unverified` (search snippet only) 14. https://docs.hatchet.run/v1/architecture-and-guarantees — Postgres state; **at-least-once**; tasks must be idempotent. `verified` 15. https://restate.dev/blog/solving-durable-executions-immutability-problem/ — Versioned deployments vs patching. `unverified` (parent did not fetch) 16. https://martinfowler.com/bliki/CQRS.html — CQRS. `unverified` (search only) -17. `/private/tmp/ws13-gallery-followup/packages/sdk/src/protocol.ts` — Verb set including `run.resume`, `effect.record`/`confirm`, `journal.read`. `verified` -18. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/append.rs` — Immediate transaction, fail-closed. `verified` -19. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine.rs` — `resume` / `resume_filtered`. `verified` -20. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs` — `RunState::fold`. `verified` -21. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/machine/recovery.rs` — Dead attempts → `crashed` / `lease_expired`. `verified` -22. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine/effects.rs` — Elect / confirm. `verified` -23. `/private/tmp/ws13-gallery-followup/kernel/DURABLE-CHANNELS.md` — Replay deliveries, do not re-execute receive. `verified` -24. `/private/tmp/ws13-gallery-followup/kernel/MEMORY.md` — Pack reused on resume; provider stub. `verified` -25. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/lib.rs` — `rollover_is_atomic_scaffolding_for_epoch_resume`. `verified` -26. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/tests/crash_resume.rs` — SIGKILL then resume; “completed effects must not be replayed as code”. `verified` +17. `packages/sdk/src/protocol.ts` — Verb set including `run.resume`, `effect.record`/`confirm`, `journal.read`. `verified` +18. `kernel/relayflowd-journal/src/append.rs` — Immediate transaction, fail-closed. `verified` +19. `kernel/relayflowd/src/engine.rs` — `resume` / `resume_filtered`. `verified` +20. `kernel/relayflowd-core/src/state.rs` — `RunState::fold`. `verified` +21. `kernel/relayflowd-core/src/machine/recovery.rs` — Dead attempts → `crashed` / `lease_expired`. `verified` +22. `kernel/relayflowd/src/engine/effects.rs` — Elect / confirm. `verified` +23. `kernel/DURABLE-CHANNELS.md` — Replay deliveries, do not re-execute receive. `verified` +24. `kernel/MEMORY.md` — Pack reused on resume; provider stub. `verified` +25. `kernel/relayflowd-journal/src/lib.rs` — `rollover_is_atomic_scaffolding_for_epoch_resume`. `verified` +26. `kernel/relayflowd/tests/crash_resume.rs` — SIGKILL then resume; “completed effects must not be replayed as code”. `verified` 27. https://github.com/humanlayer/12-factor-agents/blob/main/content/factor-05-unify-execution-state.md — Unify execution/business state; resume by loading the thread. `verified` 28. https://web.stanford.edu/class/cs345d-01/rl/aries.pdf — ARIES WAL. `unverified` (Landscape claimed fetch; parent did not) 29. https://cadenceworkflow.io/docs/go-client/workflow-replay-shadowing — Cadence replayer. `unverified` diff --git a/docs/evidence/ws13/followup/default-budget/reports/synthesis.md b/docs/evidence/ws13/followup/default-budget/reports/synthesis.md index 27abebdd..e81a0ec5 100644 --- a/docs/evidence/ws13/followup/default-budget/reports/synthesis.md +++ b/docs/evidence/ws13/followup/default-budget/reports/synthesis.md @@ -389,32 +389,32 @@ Restate Go durable-steps) except where I fetched them myself — see [9]. truth; external gateways must be gated on replay. (grok) 15. https://github.com/humanlayer/12-factor-agents/blob/main/content/factor-05-unify-execution-state.md — unify execution and business state; resume by loading the thread. (grok) -16. `/private/tmp/ws13-gallery-followup/docs/RFC-0001-everything-is-a-relayflow.md` +16. `docs/RFC-0001-everything-is-a-relayflow.md` — decision #2 (no deterministic replay), journal + memoization, Appendix A, epoch compaction, decisions #8/#14/#15. (claude, grok) -17. `/private/tmp/ws13-gallery-followup/kernel/DESIGN.md` — entry types, SQLite +17. `kernel/DESIGN.md` — entry types, SQLite schema, memoized resume algorithm, protocol v0, elect/perform/confirm. (codex, grok) -18. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/state.rs`, +18. `kernel/relayflowd-core/src/state.rs`, `machine.rs` — `RunState::fold`; pure state machine on a `Clock` trait. (grok) -19. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-core/src/machine/recovery.rs` +19. `kernel/relayflowd-core/src/machine/recovery.rs` — dead attempts resolved to `crashed` / `lease_expired`. (grok) -20. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine.rs` — +20. `kernel/relayflowd/src/engine.rs` — `resume` / `resume_filtered`. (grok) -21. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/src/engine/effects.rs` +21. `kernel/relayflowd/src/engine/effects.rs` — two-phase election and confirmation. (grok) -22. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/append.rs` +22. `kernel/relayflowd-journal/src/append.rs` — immediate transaction, fail-closed append. (grok) -23. `/private/tmp/ws13-gallery-followup/kernel/relayflowd-journal/src/segment.rs`, +23. `kernel/relayflowd-journal/src/segment.rs`, `lib.rs` — segment rollover; `rollover_is_atomic_scaffolding_for_epoch_resume`. (grok) -24. `/private/tmp/ws13-gallery-followup/kernel/relayflowd/tests/crash_resume.rs` +24. `kernel/relayflowd/tests/crash_resume.rs` — SIGKILL at step boundaries then `resume`; completed effects must not be replayed as code; `assert_exact_journal`. (grok) -25. `/private/tmp/ws13-gallery-followup/kernel/DURABLE-CHANNELS.md` — replay +25. `kernel/DURABLE-CHANNELS.md` — replay recorded deliveries; do not re-execute receive. (grok) -26. `/private/tmp/ws13-gallery-followup/kernel/MEMORY.md` — journaled pack +26. `kernel/MEMORY.md` — journaled pack reused on resume; `FixedMemoryProvider` is a stub. (grok) -27. `/private/tmp/ws13-gallery-followup/packages/sdk/src/protocol.ts` — verb set +27. `packages/sdk/src/protocol.ts` — verb set including `run.resume`, `effect.record` / `confirm`, `journal.read`. (grok) diff --git a/docs/evidence/ws13/followup/run-gallery.py b/docs/evidence/ws13/followup/run-gallery.py index f035e078..15bd8c42 100644 --- a/docs/evidence/ws13/followup/run-gallery.py +++ b/docs/evidence/ws13/followup/run-gallery.py @@ -18,6 +18,9 @@ node = shutil.which('node') if node is None: raise SystemExit('Node 22.18+ is required on PATH.') +version = subprocess.check_output([node, '-p', 'process.versions.node'], text=True, timeout=10).strip() +if tuple(map(int, version.split('.'))) < (22, 18, 0): + raise SystemExit(f'Node 22.18+ is required on PATH; found {version} at {node}.') cli = str(root / 'node_modules/relayflows/bin/flows.js') cases = [ ('dependency-upgrade-bot', 120, [node, cli, 'run', diff --git a/docs/evidence/ws13/gallery-pr-review-pipeline.txt b/docs/evidence/ws13/gallery-pr-review-pipeline.txt deleted file mode 100644 index 82d23d7d..00000000 --- a/docs/evidence/ws13/gallery-pr-review-pipeline.txt +++ /dev/null @@ -1,6 +0,0 @@ -$ cd /tmp/ws13-gallery -$ node /tmp/ws13-consumer/hello/node_modules/relayflows/bin/flows.js run examples/pr-review-pipeline/pr-review-pipeline.flow.ts --local-agent --input '{"diffRange": "origin/main...HEAD"}' --data-dir /tmp/ws13-gallery-pr-re -REFUSED [invalid_spec] unsupported_header: flow "pr-review-pipeline" uses unsupported header fields: budget - -EXIT_CODE=2 -ELAPSED_SECONDS=4.990 diff --git a/docs/evidence/ws13/gallery-research.txt b/docs/evidence/ws13/gallery-research.txt deleted file mode 100644 index 08a8b222..00000000 --- a/docs/evidence/ws13/gallery-research.txt +++ /dev/null @@ -1,5 +0,0 @@ -$ cd /tmp/ws13-gallery -$ node --experimental-strip-types examples/research/shims/run.ts --slug ws13-verification --question 'Compare durable step journals with deterministic replay. Keep every report under 200 words.' --timeout-minutes 1 -None -EXIT_CODE=124 -ELAPSED_SECONDS=150.067 diff --git a/docs/evidence/ws13/record.py b/docs/evidence/ws13/record.py index 1af73d36..5ec7c2b2 100644 --- a/docs/evidence/ws13/record.py +++ b/docs/evidence/ws13/record.py @@ -51,6 +51,11 @@ except ProcessLookupError: pass break + tail = decoder.decode(b'', final=True) + if tail: + cast.write(json.dumps([round(time.monotonic() - started, 6), 'o', tail]) + '\n') + transcript.write(tail.replace('\r\n', '\n')) + os.close(fd) _, status = os.waitpid(pid, 0) code = os.waitstatus_to_exitcode(status) elapsed = time.monotonic() - started @@ -58,4 +63,7 @@ transcript.write(ending) cast.write(json.dumps([round(elapsed, 6), 'o', ending.replace('\n', '\r\n')]) + '\n') print(ending) - sys.exit(code if code >= 0 else 128 - code) +# Normalize only the readable transcript; the cast retains terminal bytes. +path = Path(prefix + '.txt') +path.write_text('\n'.join(line.rstrip() for line in path.read_text().splitlines()) + '\n') +sys.exit(code if code >= 0 else 128 - code) diff --git a/docs/evidence/ws13/research-typecheck.txt b/docs/evidence/ws13/research-typecheck.txt deleted file mode 100644 index a6d5e50e..00000000 --- a/docs/evidence/ws13/research-typecheck.txt +++ /dev/null @@ -1,3 +0,0 @@ -$ npm --prefix examples/research run typecheck -npm notice run typecheck -npm notice run ../../packages/sdk/node_modules/.bin/tsc -p tsconfig.json diff --git a/docs/evidence/ws13/review/build.txt b/docs/evidence/ws13/review/build.txt new file mode 100644 index 00000000..4bce5a28 --- /dev/null +++ b/docs/evidence/ws13/review/build.txt @@ -0,0 +1,6 @@ +$ npm --prefix packages/sdk run build + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/cloud-failure.txt b/docs/evidence/ws13/review/cloud-failure.txt new file mode 100644 index 00000000..e2ddbf09 --- /dev/null +++ b/docs/evidence/ws13/review/cloud-failure.txt @@ -0,0 +1,17 @@ +$ gh run view 34267938676 --log | sed -n '498,513p' +review Wait for cloud swarm 2026-09-08T19:16:55.9809793Z ##[endgroup] +review Wait for cloud swarm 2026-09-08T19:19:31.1680377Z swarm failure reason: +review Wait for cloud swarm 2026-09-08T19:19:31.1682875Z relayfile ACL GET /.relayfile.acl failed with status 429 (correlationId=499e3981-c303-48e3-89be-595ac66ee3c4) +review Post verdict and transcripts 2026-09-08T19:19:31.1733020Z ##[group]Run ../gate-files/.github/workflows/scripts/swarm-post.sh "48b040cc-52e7-49ce-8dc3-f24df51b8687" "247" +review Post verdict and transcripts 2026-09-08T19:19:31.1733822Z ../gate-files/.github/workflows/scripts/swarm-post.sh "48b040cc-52e7-49ce-8dc3-f24df51b8687" "247" +review Post verdict and transcripts 2026-09-08T19:19:31.1754377Z shell: /usr/bin/bash -e {0} +review Post verdict and transcripts 2026-09-08T19:19:31.1754716Z env: +review Post verdict and transcripts 2026-09-08T19:19:31.1755052Z CLOUD_API_URL: https://agentrelay.com/cloud +review Post verdict and transcripts 2026-09-08T19:19:31.1755591Z CLOUD_API_KEY: *** +review Post verdict and transcripts 2026-09-08T19:19:31.1755986Z RELAY_WORKSPACE_KEY: *** +review Post verdict and transcripts 2026-09-08T19:19:31.1756314Z RELAY_API_KEY: *** +review Post verdict and transcripts 2026-09-08T19:19:31.1758969Z GH_TOKEN: *** +review Post verdict and transcripts 2026-09-08T19:19:31.1759281Z ##[endgroup] +review Post verdict and transcripts 2026-09-08T19:19:31.5616062Z Fetching patch for run 48b040cc-52e7-49ce-8dc3-f24df51b8687... +review Post verdict and transcripts 2026-09-08T19:19:33.3740051Z No changes to sync — the workflow did not modify any files. +review Post verdict and transcripts 2026-09-08T19:19:38.6752567Z ##[error]Process completed with exit code 1. diff --git a/docs/evidence/ws13/review/regressions.txt b/docs/evidence/ws13/review/regressions.txt new file mode 100644 index 00000000..f94d5dd9 --- /dev/null +++ b/docs/evidence/ws13/review/regressions.txt @@ -0,0 +1,46 @@ +$ node node_modules/vitest/vitest.mjs run tests/local-dev-ux.test.ts tests/worker-lease.test.ts tests/worker-platform.test.ts tests/cli-progress-wait.test.ts tests/direct-run-failure.test.ts tests/worker-cli-abort.test.ts tests/worker-cli.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > passes an explicit safe environment at identification and execution 1397ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 2845ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > bounds wrapper execution after acknowledgement 4958ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > bounds captured wrapper output 2942ms + ✓ tests/worker-cli.test.ts > custom wrapper execution identity > refuses a duplicate execute protocol frame 595ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2448ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 2217ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3256ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11304ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > accepts an execute token and an over-8KiB payload flushed in one write 303ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 907ms + ✓ tests/worker-cli.test.ts > custom wrapper execution bounds are reader-owned > still bounds an un-terminated handshake buffer and names the bound 525ms + ✓ tests/worker-cli.test.ts > delivers the journaled memory pack to the real wrapper and excludes its charge from completion usage 607ms + ✓ tests/worker-lease.test.ts > worker lease ownership > renews the same attempt through a long subprocess and drains before completing once + ✓ tests/worker-lease.test.ts > worker lease ownership > aborts execution and never completes after a rejected heartbeat + ✓ tests/worker-lease.test.ts > worker lease ownership > expires locally when a renewal response never arrives, without stranding close + ✓ tests/worker-lease.test.ts > worker lease ownership > does not spawn a process for an already-expired dispatch + ✓ tests/worker-lease.test.ts > refuses completion past the deadline even before the expiry timer runs + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > emits an agent starter, local-worker command and the chosen CLI + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > offers a credential-free deterministic starter + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > refuses an existing project without changing any files + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > validates names and template before writing + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > does not report completion before the journal operation resolves + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > propagates a journal failure without inventing a successful completion + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > renders time and strips terminal controls from step names + ✓ tests/local-dev-ux.test.ts > observer exceptions neither fail committed work nor mask the journal error + ✓ tests/worker-cli-abort.test.ts > stops claude and its process group when lease ownership is lost 1652ms + ✓ tests/worker-cli-abort.test.ts > stops wrapper.mjs and its process group when lease ownership is lost 1888ms + ✓ tests/direct-run-failure.test.ts > preserves authored agent_cli_unresolved classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > preserves authored agent_parked classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > preserves authored step_failed classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > uses the worker cause when the authored executor only saw a generic disconnect + ✓ tests/cli-progress-wait.test.ts > run starts the wait clock on its first observed lease + ✓ tests/cli-progress-wait.test.ts > resume starts the wait clock on its first observed lease + ✓ tests/worker-platform.test.ts > fails closed before spawning a lease-bound process on Windows + + Test Files 7 passed (7) + Tests 35 passed (35) + Start at 22:10:00 + Duration 74.22s (transform 4.07s, setup 0ms, collect 18.83s, tests 38.05s, environment 2ms, prepare 7.30s) + +EXIT_CODE=0 diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 35feebf1..1d8b86cf 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -146,8 +146,10 @@ export async function runCli( onProgress: showProgress, onWait: (progress: RunProgress) => { emitWait(progress, io); + const now = performance.now(); + if (!startedSteps.has(progress.stepId)) startedSteps.set(progress.stepId, now); showProgress({ type: 'step.running', stepId: progress.stepId, stepType: progress.stepType, - elapsedMs: performance.now() - (startedSteps.get(progress.stepId) ?? performance.now()) }); + elapsedMs: now - startedSteps.get(progress.stepId)! }); }, daemon: { spawn: parsed.spawn && spawnAllowedByEnv() }, }; diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 8348875b..57d2fe5d 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -75,7 +75,10 @@ export async function runDirectFlow( }, }; } catch (caught) { - const error = localAgent?.failure ?? caught; + // Preserve authored classifications/run IDs; use the worker's cause only + // when its connection teardown left a generic transport error. + const error = caught instanceof AuthoredFlowExecutionError || caught instanceof AuthoredFlowLoadError + ? caught : localAgent?.failure ?? caught; // `agent_cli_unresolved` and `unsupported_workspace_permission` are // preflight-shaped refusals, not protocol failures — `flows check` // returns exit 2 for the equivalent declarative-spec failures, and this diff --git a/packages/sdk/src/progress.ts b/packages/sdk/src/progress.ts index 27ce47c2..2839f21a 100644 --- a/packages/sdk/src/progress.ts +++ b/packages/sdk/src/progress.ts @@ -29,14 +29,23 @@ export async function observeStep( execute: () => Promise, emit?: (event: ProgressEvent) => void, ): Promise { + const publish = (event: ProgressEvent): void => { + try { emit?.(event); } catch { + // A projection failure must not turn a journaled success into a retry, + // or replace the executor's original failure. Surface it separately. + process.emitWarning(`Progress observer failed for ${event.type}.`, { + code: 'FLOWS_PROGRESS_OBSERVER_ERROR', + }); + } + }; const started = performance.now(); - emit?.({ type: 'step.started', stepId, stepType, elapsedMs: 0 }); + publish({ type: 'step.started', stepId, stepType, elapsedMs: 0 }); try { const result = await execute(); - emit?.({ type: 'step.completed', stepId, stepType, elapsedMs: performance.now() - started, completionReason: 'success' }); + publish({ type: 'step.completed', stepId, stepType, elapsedMs: performance.now() - started, completionReason: 'success' }); return result; } catch (error) { - emit?.({ type: 'step.failed', stepId, stepType, elapsedMs: performance.now() - started }); + publish({ type: 'step.failed', stepId, stepType, elapsedMs: performance.now() - started }); throw error; } } diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 77eb828e..083fcc99 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -35,6 +35,9 @@ export async function runAgentCli( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); + if (signal !== undefined && process.platform === 'win32') { + throw new Error('Lease-bound agent execution requires macOS or Linux process-group cancellation; Windows is unsupported.'); + } const kind = cliAdapterKind(cli); if (kind === 'relayflows-wrapper-v1') { diff --git a/packages/sdk/src/worker-lease.ts b/packages/sdk/src/worker-lease.ts index 6ce68561..dcf3488f 100644 --- a/packages/sdk/src/worker-lease.ts +++ b/packages/sdk/src/worker-lease.ts @@ -9,6 +9,7 @@ export async function withWorkerLease( ): Promise { const controller = new AbortController(); let stopped = false; + let latestDeadline = dispatch.lease_deadline_ms; let renewalTimer: NodeJS.Timeout | undefined; let expiryTimer: NodeJS.Timeout | undefined; let pending: Promise = Promise.resolve(); @@ -18,6 +19,7 @@ export async function withWorkerLease( if (!Number.isFinite(remaining) || remaining <= 0) { throw new Error(`Agent lease is already expired for ${dispatch.run_id}/${dispatch.step_id}.`); } + latestDeadline = deadline; if (expiryTimer !== undefined) clearTimeout(expiryTimer); expiryTimer = setTimeout(() => fail(new Error( `Agent lease expired before renewal for ${dispatch.run_id}/${dispatch.step_id}.`, @@ -47,6 +49,11 @@ export async function withWorkerLease( // racing after completion would otherwise report a spurious lease error. await pending; controller.signal.throwIfAborted(); + // Timer callbacks can be delayed behind a resolved subprocess promise. + // Check the clock itself before permitting step.complete. + if (Date.now() >= latestDeadline) { + throw new Error(`Agent lease expired before completion for ${dispatch.run_id}/${dispatch.step_id}.`); + } return result; } finally { stopped = true; diff --git a/packages/sdk/src/wrapper-session.ts b/packages/sdk/src/wrapper-session.ts index 82d5493d..bc0fe15c 100644 --- a/packages/sdk/src/wrapper-session.ts +++ b/packages/sdk/src/wrapper-session.ts @@ -51,6 +51,9 @@ export function runWrapperSession( signal?: AbortSignal, ): Promise { if (signal?.aborted) return Promise.reject(signal.reason); + if (signal !== undefined && process.platform === 'win32') { + return Promise.reject(new Error('Lease-bound wrapper execution requires macOS or Linux process-group cancellation; Windows is unsupported.')); + } const limits = sessionLimits(overrides); let request: string; try { diff --git a/packages/sdk/tests/cli-progress-wait.test.ts b/packages/sdk/tests/cli-progress-wait.test.ts new file mode 100644 index 00000000..3625ac3c --- /dev/null +++ b/packages/sdk/tests/cli-progress-wait.test.ts @@ -0,0 +1,28 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { runCli } from '../src/cli.js'; +import { emptyReport, runFlow, resumeFlow, type RunLifecycleOptions } from '../src/cli/run.js'; +vi.mock('../src/cli/run.js', async importOriginal => ({ + ...await importOriginal(), + runFlow: vi.fn(), resumeFlow: vi.fn(), +})); +afterEach(() => { vi.restoreAllMocks(); }); +it.each(['run', 'resume'])('%s starts the wait clock on its first observed lease', async command => { + const clock = vi.spyOn(performance, 'now'); + const output: string[] = []; + const execute = async (_value: string, _data: string, options?: RunLifecycleOptions) => { + const progress = { runId: 'run', stepId: 'agent', stepType: 'agent' as const, leaseDeadlineMs: Date.now() + 30_000 }; + clock.mockReturnValue(1000); + options?.onWait?.(progress); + clock.mockReturnValue(3500); + options?.onWait?.(progress); + return { exitCode: 0, report: emptyReport('run') }; + }; + vi.mocked(runFlow).mockImplementation(execute); + vi.mocked(resumeFlow).mockImplementation(execute); + await runCli([command, command === 'run' ? 'example.flow.yaml' : 'run'], { + stdout: () => {}, stderr: line => { output.push(line); }, + }); + expect(output.filter(line => line.startsWith('↻'))).toEqual([ + '↻ agent (agent) [agent: running] 0.00s', '↻ agent (agent) [agent: running] 2.50s', + ]); +}); diff --git a/packages/sdk/tests/direct-run-failure.test.ts b/packages/sdk/tests/direct-run-failure.test.ts new file mode 100644 index 00000000..e368ae73 --- /dev/null +++ b/packages/sdk/tests/direct-run-failure.test.ts @@ -0,0 +1,33 @@ +import { expect, it, vi } from 'vitest'; +import { runDirectFlow } from '../src/cli/direct-run.js'; +import { AuthoredFlowExecutionError, executeAuthoredFlow } from '../src/authored-flow-executor.js'; +vi.mock('../src/journal-client.js', () => ({ JournalClient: class { close() {} } })); +vi.mock('../src/cli/run.js', async importOriginal => ({ + ...await importOriginal(), connect: async () => undefined, +})); +vi.mock('../src/authored-flow-loader.js', async importOriginal => ({ + ...await importOriginal(), + loadAuthoredFlow: async () => ({ handle: {}, getDefinition: () => ({}) }), +})); +vi.mock('../src/authored-flow-executor.js', async importOriginal => ({ + ...await importOriginal(), executeAuthoredFlow: vi.fn(), +})); +vi.mock('../src/local-agent.js', () => ({ attachLocalAgent: async () => ({ + failure: new Error('worker transport closed'), stream: 'test', close: async () => {}, +}) })); +it.each([ + ['agent_cli_unresolved', 2], ['agent_parked', 3], ['step_failed', 1], +] as const)('preserves authored %s classification despite a worker failure', async (code, exitCode) => { + vi.mocked(executeAuthoredFlow).mockRejectedValueOnce(new AuthoredFlowExecutionError(code, 'authored cause', undefined, 'durable-run')); + const result = await runDirectFlow('flow.ts', '{}', '/tmp/unused', { localAgent: true }); + expect(result.exitCode).toBe(exitCode); + expect(JSON.stringify(result.report)).toContain('authored cause'); + expect(JSON.stringify(result.report)).not.toContain('worker transport closed'); + if (exitCode !== 2) expect(result.report.runId).toBe('durable-run'); +}); +it('uses the worker cause when the authored executor only saw a generic disconnect', async () => { + vi.mocked(executeAuthoredFlow).mockRejectedValueOnce(new Error('connection closed')); + const result = await runDirectFlow('flow.ts', '{}', '/tmp/unused', { localAgent: true }); + expect(result.exitCode).toBe(1); + expect(JSON.stringify(result.report)).toContain('worker transport closed'); +}); diff --git a/packages/sdk/tests/local-agent-live.test.ts b/packages/sdk/tests/local-agent-live.test.ts index 2f67dc2e..f818f163 100644 --- a/packages/sdk/tests/local-agent-live.test.ts +++ b/packages/sdk/tests/local-agent-live.test.ts @@ -11,11 +11,18 @@ const wrapperHelper = resolve('../../testdata/preflight/wrapper-session.mjs'); // Ask the existing build wrapper for its target directory. A temp fixture's // cwd cannot discover the checkout, and test:prep's child-shell exports do not // survive into vitest. Do not select another worktree's most recent binary. -const relayflowd = process.env['RELAYFLOWD_BIN'] ?? join(JSON.parse(execFileSync('sh', [ - resolve('../../ops/cargo.sh'), 'metadata', '--format-version=1', '--no-deps', '--locked', '--offline', -], { cwd: resolve('../../kernel'), encoding: 'utf8', - env: { ...process.env, RELAYFLOWS_NO_TOOLCHAIN_INSTALL: '1' }, -})).target_directory, 'debug', 'relayflowd'); +function resolveDaemon(): string { + if (process.env['RELAYFLOWD_BIN']) return process.env['RELAYFLOWD_BIN']; + try { + return join(JSON.parse(execFileSync('sh', [ + resolve('../../ops/cargo.sh'), 'metadata', '--format-version=1', '--no-deps', '--locked', '--offline', + ], { cwd: resolve('../../kernel'), encoding: 'utf8', + env: { ...process.env, RELAYFLOWS_NO_TOOLCHAIN_INSTALL: '1' }, + })).target_directory, 'debug', 'relayflowd'); + } catch (cause) { + throw new Error('Live CLI tests require npm run test:prep or an explicit RELAYFLOWD_BIN.', { cause }); + } +} afterEach(() => { for (const root of roots.splice(0)) { const connection = join(root, 'data/connection.json'); @@ -32,6 +39,7 @@ afterEach(() => { }); function fixture(exitCode = 0, workspace?: string, delayMs = 0) { + const relayflowd = resolveDaemon(); const root = mkdtempSync(join(tmpdir(), 'flows-local-agent-')); roots.push(root); symlinkSync(join(sdk, 'node_modules'), join(root, 'node_modules')); diff --git a/packages/sdk/tests/local-dev-ux.test.ts b/packages/sdk/tests/local-dev-ux.test.ts index 1c35beff..382b4264 100644 --- a/packages/sdk/tests/local-dev-ux.test.ts +++ b/packages/sdk/tests/local-dev-ux.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createFlow } from '../src/create-flow.js'; import { observeStep, renderProgress, type ProgressEvent } from '../src/progress.js'; @@ -70,3 +70,18 @@ describe('progress is an observation of execution', () => { .toEqual(['↻ ?[2Jagent (agent) [agent: running] 1.23s']); }); }); + +it('observer exceptions neither fail committed work nor mask the journal error', async () => { + const warning = vi.spyOn(process, 'emitWarning').mockImplementation(() => {}); + const observer = (): never => { throw new Error('display failed'); }; + try { + await expect(observeStep('ok', 'agent', async () => 42, observer)).resolves.toBe(42); + const journalError = new Error('journal write rejected'); + await expect(observeStep('bad', 'agent', async () => { throw journalError; }, observer)).rejects.toBe(journalError); + expect(warning).toHaveBeenCalledTimes(4); + expect(warning.mock.calls.map(call => call[0])).toEqual([ + 'Progress observer failed for step.started.', 'Progress observer failed for step.completed.', + 'Progress observer failed for step.started.', 'Progress observer failed for step.failed.', + ]); + } finally { warning.mockRestore(); } +}); diff --git a/packages/sdk/tests/worker-lease.test.ts b/packages/sdk/tests/worker-lease.test.ts index 7c3d70d4..ce2e09f1 100644 --- a/packages/sdk/tests/worker-lease.test.ts +++ b/packages/sdk/tests/worker-lease.test.ts @@ -103,3 +103,17 @@ describe('worker lease ownership', () => { expect(String(errors[0])).toContain('already expired'); }); }); + +it('refuses completion past the deadline even before the expiry timer runs', async () => { + const { client, worker, errors, dispatch } = setup(); + vi.mocked(runAgentCli).mockImplementation(async () => { + vi.setSystemTime(Date.now() + 30_001); // changes the clock WITHOUT running timers + return { exit_code: 0, stdout_tail: 'late', stderr_tail: '' }; + }); + await worker.attach(); + client.emit('step.dispatch', dispatch); + await worker.close(); + expect(runAgentCli).toHaveBeenCalledTimes(1); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(String(errors[0])).toContain('lease expired before completion'); +}); diff --git a/packages/sdk/tests/worker-platform.test.ts b/packages/sdk/tests/worker-platform.test.ts new file mode 100644 index 00000000..b792c91e --- /dev/null +++ b/packages/sdk/tests/worker-platform.test.ts @@ -0,0 +1,16 @@ +import { expect, it, vi } from 'vitest'; +import { spawn } from 'node:child_process'; +import { runAgentCli } from '../src/worker-cli.js'; +import { runWrapperSession } from '../src/wrapper-session.js'; +vi.mock('node:child_process', async importOriginal => ({ + ...await importOriginal(), spawn: vi.fn(), +})); +it('fails closed before spawning a lease-bound process on Windows', async () => { + const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); + const signal = new AbortController().signal; + try { + await expect(runAgentCli('claude', 'hello', undefined, undefined, undefined, signal)).rejects.toThrow('Windows is unsupported'); + await expect(runWrapperSession('wrapper', 'hello', undefined, undefined, {}, {}, signal)).rejects.toThrow('Windows is unsupported'); + expect(spawn).not.toHaveBeenCalled(); + } finally { platform.mockRestore(); } +}); From 552c2d63889a4bc5ec2b51cb391417b0512b06d4 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 22:19:14 +0200 Subject: [PATCH 11/16] docs: correct gallery evidence and record every review disposition Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/README.md | 6 +- docs/evidence/ws13/followup/README.md | 21 +- .../ws13/followup/final-sdk/README.md | 11 + docs/evidence/ws13/review/README.md | 66 ++++ .../gallery-dependency-upgrade-bot.txt | 12 + .../gallery/gallery-pr-review-pipeline.txt | 12 + .../ws13/review/gallery/gallery-results.json | 14 + docs/evidence/ws13/review/helpers.txt | 7 + docs/evidence/ws13/review/install.txt | 9 + .../installed-identity-commonjs-attempt.txt | 42 +++ .../ws13/review/installed-identity.txt | 9 + docs/evidence/ws13/review/live.txt | 16 + .../evidence/ws13/review/native-container.txt | 39 +++ docs/evidence/ws13/review/pack.txt | 319 ++++++++++++++++++ .../ws13/review/packed-launcher-live.txt | 12 + docs/evidence/ws13/review/threads.md | 35 ++ docs/evidence/ws13/review/typechecks.txt | 10 + docs/evidence/ws13/review/verify-helpers.py | 45 +++ docs/evidence/ws13/review/verify-installed.py | 23 ++ examples/README.md | 9 +- examples/dependency-upgrade-bot/README.md | 4 +- examples/pr-review-pipeline/README.md | 4 +- 22 files changed, 710 insertions(+), 15 deletions(-) create mode 100644 docs/evidence/ws13/followup/final-sdk/README.md create mode 100644 docs/evidence/ws13/review/README.md create mode 100644 docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt create mode 100644 docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt create mode 100644 docs/evidence/ws13/review/gallery/gallery-results.json create mode 100644 docs/evidence/ws13/review/helpers.txt create mode 100644 docs/evidence/ws13/review/install.txt create mode 100644 docs/evidence/ws13/review/installed-identity-commonjs-attempt.txt create mode 100644 docs/evidence/ws13/review/installed-identity.txt create mode 100644 docs/evidence/ws13/review/live.txt create mode 100644 docs/evidence/ws13/review/native-container.txt create mode 100644 docs/evidence/ws13/review/pack.txt create mode 100644 docs/evidence/ws13/review/packed-launcher-live.txt create mode 100644 docs/evidence/ws13/review/threads.md create mode 100644 docs/evidence/ws13/review/typechecks.txt create mode 100644 docs/evidence/ws13/review/verify-helpers.py create mode 100644 docs/evidence/ws13/review/verify-installed.py diff --git a/docs/evidence/ws13/README.md b/docs/evidence/ws13/README.md index afc75082..23c30d8c 100644 --- a/docs/evidence/ws13/README.md +++ b/docs/evidence/ws13/README.md @@ -18,7 +18,9 @@ Cloud run-publication API is part of this follow-up. The [current three-entry gallery](../../../examples/README.md) is **1 PASS, 2 BLOCKED** and supersedes the initial invocation results below. Research completed with the default budget in [690.935s](followup/default-budget/gallery-research.txt); -the SDK flows still refuse unsupported budget headers. Research now reports each provider probe +the SDK flows still refuse unsupported budget headers in the +[corrected, verified launcher runs](review/README.md). The prior 0.138s/0.143s +captures were stale-launcher invocation refusals and had been misclassified. Research now reports each provider probe and timeout on stderr. [Research regression tests](followup/research-tests.txt) and [typecheck](followup/research-typecheck.txt) contain the commands/output. @@ -34,7 +36,7 @@ and [typecheck](followup/research-typecheck.txt) contain the commands/output. | First live-worker test attempt | 3 process timeouts, 1 passed | [Command and output](local-agent-tests-first-attempt.txt) | | Real Claude invocation in the generated project | Completed, 132.637s for the command; existing authenticated macOS host | [Transcript](agent-run.txt), [asciicast v2 recording](agent-run.cast) | | Earlier recording attempts | Auth probe timeout; then a broken host Node shared-library dependency | [Auth timeout](agent-probe-timeout.txt), [host failure](agent-host-node-failure.txt) | -| Empty-cache install + deterministic run in fresh Debian Trixie container | Completed in 43.374s; deterministic template, no source clone or agent | [Command and output](cold-trixie.txt) | +| Empty-cache install + deterministic run in fresh Debian Trixie container | Completed in 43.374s; Node/image provisioning excluded, empty npm cache, deterministic template, no source clone or agent | [Command and output](cold-trixie.txt) | | Empty-cache install + deterministic run in fresh Debian Bookworm container | Refused: published Linux daemon requires GLIBC_2.39; 55.223s | [Command and output](cold-container.txt) | | Linux container test runner | esbuild Go runtime crashed under amd64 emulation before collecting tests | [Command, script and full output](container-tests.txt) | | Research typecheck after correcting its compiler path | Superseded by the complete follow-up capture | [Command and output](followup/research-typecheck.txt) | diff --git a/docs/evidence/ws13/followup/README.md b/docs/evidence/ws13/followup/README.md index b7584304..88c563ac 100644 --- a/docs/evidence/ws13/followup/README.md +++ b/docs/evidence/ws13/followup/README.md @@ -8,8 +8,8 @@ The three requested gallery entries have individual, explicit outcomes: | Entry | Result | Elapsed | Command and output | |---|---|---:|---| -| dependency-upgrade-bot | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 0.138s | [Final SDK capture](final-sdk/gallery-dependency-upgrade-bot.txt) | -| pr-review-pipeline | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 0.143s | [Final SDK capture](final-sdk/gallery-pr-review-pipeline.txt) | +| dependency-upgrade-bot | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 5.138s | [Verified launcher capture](../review/gallery/gallery-dependency-upgrade-bot.txt) | +| pr-review-pipeline | BLOCKED: `unsupported_header` for `budget`, exit 2 before the body | 3.539s | [Verified launcher capture](../review/gallery/gallery-pr-review-pipeline.txt) | | research | PASS: three lane reports and synthesis, `completionReason: synthesized`, exit 0 | 690.935s | [Default-budget capture](default-budget/gallery-research.txt) | The SDK/kernel capability owner must supply the two blocked flows' budget @@ -17,9 +17,16 @@ headers, postfix artifact gates and declared workspace behavior. Those requirements were not removed or weakened. Research uses its documented source shim; the SDK examples use installed candidate npm artifacts. These are runs on an authenticated development host in a separate clone, not cold benchmarks. -The final SDK artifact's [hash](final-sdk/artifact.json) identifies the package -used for the last two invocations; it includes the lease fix below. Research's -shim does not import AgentWorker and was unchanged by that fix. +**Correction:** the earlier `final-sdk/` invocations returned `invalid_invocation` +from a stale public launcher. The 0.138s / 0.143s values were incorrectly labeled +as budget refusals. Those captures are retained as failed packaging evidence, +not gallery capability evidence. Installing only a candidate SDK let npm +re-resolve the launcher from public npm. The current table uses a fresh install +with both launcher and SDK pinned to explicit candidate tarballs, every installed +file compared against its tarball and ESM resolution checked from the launcher. +See [installation/provenance and corrected results](../review/README.md). +Research's source shim does not import AgentWorker and was unchanged by the +worker fixes. Research now prints each preflight probe and its timeout on stderr, leaving stdout for the structured result. The first follow-up used a shorter three-minute @@ -53,8 +60,8 @@ No full-suite green or mutation verification is claimed. No package was published. The release-gate owner must register `create-flow` for packaging and publishing. The review-swarm/CI owner must obtain fresh maintainability, history and structure transcripts; the old missing transcripts and new review comments -are not approving signoff at the final head. Lower-priority review comments -remain for review; this report does not claim every comment is resolved. +are not approving signoff at the final head. The [review response ledger](../review/threads.md) records each original thread +and its disposition; none of these responses constitute independent signoff. `run-gallery.py` takes a clone and a fresh evidence directory. Use `research-default` for the documented-budget research run or `sdk-only` for the diff --git a/docs/evidence/ws13/followup/final-sdk/README.md b/docs/evidence/ws13/followup/final-sdk/README.md new file mode 100644 index 00000000..a8e2f665 --- /dev/null +++ b/docs/evidence/ws13/followup/final-sdk/README.md @@ -0,0 +1,11 @@ +# Superseded: stale launcher captures + +Both gallery captures here returned `invalid_invocation`, not `unsupported_header`. +The original report misclassified them. Installing only the SDK tarball allowed +npm to re-resolve the launcher from public npm; the direct packed-SDK test did +not exercise that launcher. `artifact.json` identifies the installed SDK, but +it does not establish which SDK the stale launcher executed. + +Use the [corrected gallery](../../review/README.md), where the launcher and SDK +are pinned together and every installed candidate file is verified. These +original failures are preserved, not rewritten into passing evidence. diff --git a/docs/evidence/ws13/review/README.md b/docs/evidence/ws13/review/README.md new file mode 100644 index 00000000..d4625d00 --- /dev/null +++ b/docs/evidence/ws13/review/README.md @@ -0,0 +1,66 @@ +# WS-13 review corrections + +The original 29 threads are tracked individually in [the response ledger](threads.md). +The SDK/code fixes are in `e3f756c`; no kernel or judging gate was changed. + +## Corrected gallery: 1 pass, 2 blocked + +| Entry | Result | Elapsed | Literal command and output | +|---|---|---:|---| +| dependency-upgrade-bot | **BLOCKED:** `unsupported_header` for `budget`, exit 2 before body execution | 5.138s | [Capture](gallery/gallery-dependency-upgrade-bot.txt) | +| pr-review-pipeline | **BLOCKED:** `unsupported_header` for `budget`, exit 2 before body execution | 3.539s | [Capture](gallery/gallery-pr-review-pipeline.txt) | +| research | **PASS:** three reports and synthesis, exit 0, `completionReason: synthesized` | 690.935s | [Existing default-budget capture](../followup/default-budget/gallery-research.txt) | + +The SDK/kernel capability owner must implement the budgets, postfix gates and +workspace semantics of the two blocked examples. They have not been weakened. +Research was not rerun: its unchanged shim's successful authenticated run is +retained. Report copies now remove temporary checkout prefixes from citations; +[the manifest](../followup/default-budget/artifacts.json) records both original +and normalized hashes. Source citations are repository-relative at `ab1e3ff`. + +**Correction of a false report:** the old 0.138s/0.143s captures in +`followup/final-sdk/` returned `invalid_invocation` from a stale public launcher. +They were incorrectly reported as budget refusals. The direct packed SDK test +bypassed the launcher and did not validate those invocations. This attempt +installs both candidate SDK and launcher via explicit tarball dependencies in a +new clone. [Install output](install.txt), [all installed files and ESM resolution](installed-identity.txt), +and [pack output with three distinct SDK SHA-256 hashes](pack.txt) establish provenance. +The same 2.0.8 filename represents different candidate revisions, not identical +artifacts. The native daemon is the previously packed published 2.0.8 binary; +no Rust build is claimed here. The first identity helper used CommonJS resolution +for an import-only export and failed; [that helper failure](installed-identity-commonjs-attempt.txt) +is retained. The corrected helper uses the launcher's ESM resolution conditions. + +## Verification + +| Check | Result | Literal command and captured output | +|---|---|---| +| Build | Exit 0 | [Output](build.txt) | +| SDK/API/existing test-source typechecks | Exit 0 | [Output](typechecks.txt) | +| Observer isolation, wait elapsed, error precedence, lease deadline, Windows refusal and subprocess cancellation | 35 passed | [Output](regressions.txt) | +| Native ARM64 Linux container, focused regressions | 20 passed; no kernel in this selection | [Output](native-container.txt) | +| Built CLI + real macOS daemon | 5 passed, including a 35-second single invocation | [Output](live.txt) | +| Packed launcher + real daemon | Long-lease case selected | [Output](packed-launcher-live.txt) | +| Recorder EOF/whitespace and harness refusal paths | Exit 0 | [Output](helpers.txt), [driver](verify-helpers.py) | + +Lease-bound agent/wrapper execution now refuses Windows before spawning: the +shipped macOS/Linux implementation relies on POSIX process groups. The mocked +Windows guard is tested; no native Windows process-tree cancellation is claimed. +The older amd64-emulation esbuild crash and broader kernel-suite timeout remain +failed attempts; this is not a claim that those runs passed or that the full +repository test suite is green. Timing remains accepted by Khaliq: 49.975s for +the cold deterministic loop, 132.637s for the existing-host real Claude command. +No new timing benchmark was run. + +## External handoffs + +**Review-swarm/Cloud relayfile owner:** the failed review job +[34267938676](https://github.com/AgentWorkforce/flows/actions/runs/34267938676) +returned `relayfile ACL GET /.relayfile.acl failed with status 429`, correlation +`499e3981-c303-48e3-89be-595ac66ee3c4`. [Captured failure](cloud-failure.txt). +All three fresh review transcripts are missing; this is **not review signoff**. +The gate is unchanged. See the PR for the retry status at the current head. + +**Release-gate owner:** register and publish `create-flow`; publishing remains +outside this lane. Route both owner assignments through session-thread-rollout. +The PR is out of draft. Do not merge. diff --git a/docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt b/docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt new file mode 100644 index 00000000..4676b86e --- /dev/null +++ b/docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt @@ -0,0 +1,12 @@ +$ cd /private/tmp/ws13-gallery-review +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-review/node_modules/relayflows/bin/flows.js run examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts --local-agent --input '{}' --data-dir /tmp/ws13-followup-upgrade-daemon +OUTER_TIMEOUT_SECONDS=120 +(node:6783) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///private/tmp/ws13-gallery-review/examples/dependency-upgrade-bot/dependency-upgrade-bot.flow.ts is not specified and it doesn't parse as CommonJS. +Reparsing as ES module because module syntax was detected. This incurs a performance overhead. +To eliminate this warning, add "type": "module" to /private/tmp/ws13-gallery-review/package.json. +(Use `node --trace-warnings ...` to show where the warning was created) +REFUSED [invalid_spec] unsupported_header: flow "dependency-upgrade-bot" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=5.138 +TIMED_OUT=False diff --git a/docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt b/docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt new file mode 100644 index 00000000..84a3db9f --- /dev/null +++ b/docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt @@ -0,0 +1,12 @@ +$ cd /private/tmp/ws13-gallery-review +$ /tmp/ws13-toolchain/node /private/tmp/ws13-gallery-review/node_modules/relayflows/bin/flows.js run examples/pr-review-pipeline/pr-review-pipeline.flow.ts --local-agent --input '{"diffRange":"origin/main...HEAD"}' --data-dir /tmp/ws13-followup-review-daemon +OUTER_TIMEOUT_SECONDS=120 +(node:6985) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///private/tmp/ws13-gallery-review/examples/pr-review-pipeline/pr-review-pipeline.flow.ts is not specified and it doesn't parse as CommonJS. +Reparsing as ES module because module syntax was detected. This incurs a performance overhead. +To eliminate this warning, add "type": "module" to /private/tmp/ws13-gallery-review/package.json. +(Use `node --trace-warnings ...` to show where the warning was created) +REFUSED [invalid_spec] unsupported_header: flow "pr-review-pipeline" uses unsupported header fields: budget + +EXIT_CODE=2 +ELAPSED_SECONDS=3.539 +TIMED_OUT=False diff --git a/docs/evidence/ws13/review/gallery/gallery-results.json b/docs/evidence/ws13/review/gallery/gallery-results.json new file mode 100644 index 00000000..54207258 --- /dev/null +++ b/docs/evidence/ws13/review/gallery/gallery-results.json @@ -0,0 +1,14 @@ +[ + { + "example": "dependency-upgrade-bot", + "exitCode": 2, + "elapsedSeconds": 5.138, + "timedOut": false + }, + { + "example": "pr-review-pipeline", + "exitCode": 2, + "elapsedSeconds": 3.539, + "timedOut": false + } +] diff --git a/docs/evidence/ws13/review/helpers.txt b/docs/evidence/ws13/review/helpers.txt new file mode 100644 index 00000000..26260ab3 --- /dev/null +++ b/docs/evidence/ws13/review/helpers.txt @@ -0,0 +1,7 @@ +$ python3 docs/evidence/ws13/review/verify-helpers.py +PASS: EOF UTF-8 replacement is captured in cast and text; text trims trailing spaces; cast preserves them. +PASS: cold-clone.sh without registry refuses with usage: /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/docs/evidence/ws13/cold-clone.sh: line 3: 1: Usage: cold-clone.sh CANDIDATE_REGISTRY_URL +PASS: cold-start.sh without registry refuses with usage: /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/docs/evidence/ws13/cold-start.sh: line 4: 1: Usage: cold-start.sh CANDIDATE_REGISTRY_URL +PASS: older Node refuses before any gallery command: Node 22.18+ is required on PATH; found 20.19.0 at /var/folders/6d/0x5fkt8d01gfmmjdzkxqzwnh0000gn/T/ws13-helper-check-9rfsyemg/node. +PASS: existing gallery capture preserved; overwrite refused. +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/install.txt b/docs/evidence/ws13/review/install.txt new file mode 100644 index 00000000..a23417dc --- /dev/null +++ b/docs/evidence/ws13/review/install.txt @@ -0,0 +1,9 @@ +$ git clone --no-hardlinks --single-branch --branch feat/flows-local-dev-ux /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local /tmp/ws13-gallery-review +Cloning into '/tmp/ws13-gallery-review'... +done. +EXIT_CODE=0 +$ cd /tmp/ws13-gallery-review +$ npm install --ignore-scripts --no-audit --no-fund /tmp/ws13-review-artifacts/relayflows-sdk-2.0.8.tgz /tmp/ws13-review-artifacts/relayflows-2.0.8.tgz /tmp/ws13-artifacts/relayflows-runtime-darwin-arm64-2.0.8.tgz @relayflows/surface@2.0.8 + +added 14 packages in 10s +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/installed-identity-commonjs-attempt.txt b/docs/evidence/ws13/review/installed-identity-commonjs-attempt.txt new file mode 100644 index 00000000..604b8371 --- /dev/null +++ b/docs/evidence/ws13/review/installed-identity-commonjs-attempt.txt @@ -0,0 +1,42 @@ +$ python3 docs/evidence/ws13/review/verify-installed.py +node:internal/modules/esm/resolve:314 + return new ERR_PACKAGE_PATH_NOT_EXPORTED( + ^ + +Error [ERR_PACKAGE_PATH_NOT_EXPORTED]: Package subpath './cli' is not defined by "exports" in /private/tmp/ws13-gallery-review/node_modules/@relayflows/sdk/package.json + at exportsNotFound (node:internal/modules/esm/resolve:314:10) + at packageExportsResolve (node:internal/modules/esm/resolve:604:13) + at resolveExports (node:internal/modules/cjs/loader:650:36) + at Function._findPath (node:internal/modules/cjs/loader:717:31) + at Function._resolveFilename (node:internal/modules/cjs/loader:1369:27) + at Function.resolve (node:internal/modules/helpers:157:19) + at file:///private/tmp/ws13-gallery-review/[eval1]:1:125 + at ModuleJob.run (node:internal/modules/esm/module_job:343:25) + at async onImport.tracePromise.__proto__ (node:internal/modules/esm/loader:272:26) + at async ModuleLoader.executeModuleJob (node:internal/modules/esm/loader:268:20) { + code: 'ERR_PACKAGE_PATH_NOT_EXPORTED' +} + +Node.js v22.22.2 +PASS: @relayflows/sdk: all 271 installed files match /tmp/ws13-review-artifacts/relayflows-sdk-2.0.8.tgz +SHA256=b7d0cc50aa4bd76fe577d6a07bf865bb37b7944e0e1f277d0d8743f281e5bbda +PASS: relayflows: all 3 installed files match /tmp/ws13-review-artifacts/relayflows-2.0.8.tgz +SHA256=ae555a4aa2a65b15fe934286d158d2b5477ad7a1eba8fbf738889416475ec7c0 +PASS: @relayflows/runtime-darwin-arm64: all 4 installed files match /tmp/ws13-artifacts/relayflows-runtime-darwin-arm64-2.0.8.tgz +SHA256=6e2d749b641abd66812c5e2633f37937b941854533b65993cb571ddfbb1c0744 +Traceback (most recent call last): + File "/Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/docs/evidence/ws13/review/verify-installed.py", line 20, in + resolved = subprocess.check_output(['node', '--input-type=module', '-e', + ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + "import {createRequire} from 'node:module'; console.log(createRequire(process.cwd()+'/node_modules/relayflows/bin/flows.js').resolve('@relayflows/sdk/cli'));"], cwd=root, text=True).strip() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/homebrew/Cellar/python@3.14/3.14.3_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/subprocess.py", line 472, in check_output + return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, + ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + **kwargs).stdout + ^^^^^^^^^ + File "/opt/homebrew/Cellar/python@3.14/3.14.3_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/subprocess.py", line 577, in run + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) +subprocess.CalledProcessError: Command '['node', '--input-type=module', '-e', "import {createRequire} from 'node:module'; console.log(createRequire(process.cwd()+'/node_modules/relayflows/bin/flows.js').resolve('@relayflows/sdk/cli'));"]' returned non-zero exit status 1. +EXIT_CODE=1 diff --git a/docs/evidence/ws13/review/installed-identity.txt b/docs/evidence/ws13/review/installed-identity.txt new file mode 100644 index 00000000..989ba485 --- /dev/null +++ b/docs/evidence/ws13/review/installed-identity.txt @@ -0,0 +1,9 @@ +$ python3 docs/evidence/ws13/review/verify-installed.py +PASS: @relayflows/sdk: all 271 installed files match /tmp/ws13-review-artifacts/relayflows-sdk-2.0.8.tgz +SHA256=b7d0cc50aa4bd76fe577d6a07bf865bb37b7944e0e1f277d0d8743f281e5bbda +PASS: relayflows: all 3 installed files match /tmp/ws13-review-artifacts/relayflows-2.0.8.tgz +SHA256=ae555a4aa2a65b15fe934286d158d2b5477ad7a1eba8fbf738889416475ec7c0 +PASS: @relayflows/runtime-darwin-arm64: all 4 installed files match /tmp/ws13-artifacts/relayflows-runtime-darwin-arm64-2.0.8.tgz +SHA256=6e2d749b641abd66812c5e2633f37937b941854533b65993cb571ddfbb1c0744 +PASS: launcher resolves the verified candidate SDK: /private/tmp/ws13-gallery-review/node_modules/@relayflows/sdk/dist/cli.js +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/live.txt b/docs/evidence/ws13/review/live.txt new file mode 100644 index 00000000..a838d301 --- /dev/null +++ b/docs/evidence/ws13/review/live.txt @@ -0,0 +1,16 @@ +$ RELAYFLOWD_BIN=/tmp/ws13-consumer/hello/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 2034ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 37796ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > renders actual agent completion in text output 3675ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > returns a failed run when the agent process fails 1311ms + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 667ms + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 22:12:50 + Duration 46.42s (transform 150ms, setup 0ms, collect 114ms, tests 45.49s, environment 0ms, prepare 207ms) + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/native-container.txt b/docs/evidence/ws13/review/native-container.txt new file mode 100644 index 00000000..e6e1b6f7 --- /dev/null +++ b/docs/evidence/ws13/review/native-container.txt @@ -0,0 +1,39 @@ +$ docker run --rm --platform linux/arm64 -v /tmp/ws13-native-check:/sdk -w /sdk node:22-trixie-slim sh -c 'node -p process.arch; npm ci --ignore-scripts --no-audit --no-fund && node node_modules/vitest/vitest.mjs run tests/local-dev-ux.test.ts tests/worker-lease.test.ts tests/worker-platform.test.ts tests/cli-progress-wait.test.ts tests/direct-run-failure.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1' +arm64 + +added 58 packages in 3s +npm notice +npm notice New major version of npm available! 10.9.8 -> 12.0.2 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v12.0.2 +npm notice To update run: npm install -g npm@12.0.2 +npm notice + + RUN v2.1.9 /sdk + + ✓ tests/worker-lease.test.ts > worker lease ownership > renews the same attempt through a long subprocess and drains before completing once + ✓ tests/worker-lease.test.ts > worker lease ownership > aborts execution and never completes after a rejected heartbeat + ✓ tests/worker-lease.test.ts > worker lease ownership > expires locally when a renewal response never arrives, without stranding close + ✓ tests/worker-lease.test.ts > worker lease ownership > does not spawn a process for an already-expired dispatch + ✓ tests/worker-lease.test.ts > refuses completion past the deadline even before the expiry timer runs + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > emits an agent starter, local-worker command and the chosen CLI + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > offers a credential-free deterministic starter + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > refuses an existing project without changing any files + ✓ tests/local-dev-ux.test.ts > SDK project scaffolder > validates names and template before writing + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > does not report completion before the journal operation resolves + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > propagates a journal failure without inventing a successful completion + ✓ tests/local-dev-ux.test.ts > progress is an observation of execution > renders time and strips terminal controls from step names + ✓ tests/local-dev-ux.test.ts > observer exceptions neither fail committed work nor mask the journal error + ✓ tests/direct-run-failure.test.ts > preserves authored agent_cli_unresolved classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > preserves authored agent_parked classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > preserves authored step_failed classification despite a worker failure + ✓ tests/direct-run-failure.test.ts > uses the worker cause when the authored executor only saw a generic disconnect + ✓ tests/cli-progress-wait.test.ts > run starts the wait clock on its first observed lease + ✓ tests/cli-progress-wait.test.ts > resume starts the wait clock on its first observed lease + ✓ tests/worker-platform.test.ts > fails closed before spawning a lease-bound process on Windows + + Test Files 5 passed (5) + Tests 20 passed (20) + Start at 20:13:32 + Duration 2.43s (transform 499ms, setup 0ms, collect 1.28s, tests 41ms, environment 1ms, prepare 514ms) + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/pack.txt b/docs/evidence/ws13/review/pack.txt new file mode 100644 index 00000000..26a9668b --- /dev/null +++ b/docs/evidence/ws13/review/pack.txt @@ -0,0 +1,319 @@ +$ git rev-parse HEAD +e3f756c456180e9708335d9aa493103466aa6a3e +$ npm pack --ignore-scripts --pack-destination /tmp/ws13-review-artifacts ./packages/sdk ./packages/relayflows + +> @relayflows/sdk@2.0.8 prepare +> npm run build + + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + +npm notice +npm notice 📦 @relayflows/sdk@2.0.8 +npm notice Tarball Contents +npm notice 1.1kB dist/authored-flow-error.d.ts +npm notice 723B dist/authored-flow-error.d.ts.map +npm notice 404B dist/authored-flow-error.js +npm notice 418B dist/authored-flow-error.js.map +npm notice 3.2kB dist/authored-flow-executor.d.ts +npm notice 1.4kB dist/authored-flow-executor.d.ts.map +npm notice 17.7kB dist/authored-flow-executor.js +npm notice 11.7kB dist/authored-flow-executor.js.map +npm notice 2.8kB dist/authored-flow-lifecycle.d.ts +npm notice 1.4kB dist/authored-flow-lifecycle.d.ts.map +npm notice 12.0kB dist/authored-flow-lifecycle.js +npm notice 8.7kB dist/authored-flow-lifecycle.js.map +npm notice 1.1kB dist/authored-flow-loader.d.ts +npm notice 618B dist/authored-flow-loader.d.ts.map +npm notice 3.9kB dist/authored-flow-loader.js +npm notice 2.1kB dist/authored-flow-loader.js.map +npm notice 1.9kB dist/authored-flow-operation.d.ts +npm notice 1.7kB dist/authored-flow-operation.d.ts.map +npm notice 7.1kB dist/authored-flow-operation.js +npm notice 6.8kB dist/authored-flow-operation.js.map +npm notice 681B dist/authored-flow.d.ts +npm notice 338B dist/authored-flow.d.ts.map +npm notice 574B dist/authored-flow.js +npm notice 262B dist/authored-flow.js.map +npm notice 3.4kB dist/authored-promise-graph.d.ts +npm notice 1.2kB dist/authored-promise-graph.d.ts.map +npm notice 8.8kB dist/authored-promise-graph.js +npm notice 6.0kB dist/authored-promise-graph.js.map +npm notice 2.5kB dist/backlog-picker.d.ts +npm notice 1.0kB dist/backlog-picker.d.ts.map +npm notice 5.3kB dist/backlog-picker.js +npm notice 3.4kB dist/backlog-picker.js.map +npm notice 919B dist/canonical.d.ts +npm notice 248B dist/canonical.d.ts.map +npm notice 2.5kB dist/canonical.js +npm notice 1.3kB dist/canonical.js.map +npm notice 1.6kB dist/cli-adapter.d.ts +npm notice 958B dist/cli-adapter.d.ts.map +npm notice 3.4kB dist/cli-adapter.js +npm notice 2.9kB dist/cli-adapter.js.map +npm notice 75B dist/cli-executable.d.ts +npm notice 122B dist/cli-executable.d.ts.map +npm notice 333B dist/cli-executable.js +npm notice 531B dist/cli-executable.js.map +npm notice 342B dist/cli.d.ts +npm notice 439B dist/cli.d.ts.map +npm notice 16.1kB dist/cli.js +npm notice 14.1kB dist/cli.js.map +npm notice 1.2kB dist/cli/check.d.ts +npm notice 1.0kB dist/cli/check.d.ts.map +npm notice 13.2kB dist/cli/check.js +npm notice 12.7kB dist/cli/check.js.map +npm notice 563B dist/cli/daemon-refusal.d.ts +npm notice 382B dist/cli/daemon-refusal.d.ts.map +npm notice 2.1kB dist/cli/daemon-refusal.js +npm notice 1.0kB dist/cli/daemon-refusal.js.map +npm notice 271B dist/cli/direct-run.d.ts +npm notice 329B dist/cli/direct-run.d.ts.map +npm notice 5.2kB dist/cli/direct-run.js +npm notice 3.7kB dist/cli/direct-run.js.map +npm notice 4.6kB dist/cli/hn-monitor.d.ts +npm notice 1.6kB dist/cli/hn-monitor.d.ts.map +npm notice 9.2kB dist/cli/hn-monitor.js +npm notice 5.8kB dist/cli/hn-monitor.js.map +npm notice 735B dist/cli/interruptible-sleep.d.ts +npm notice 264B dist/cli/interruptible-sleep.d.ts.map +npm notice 1.2kB dist/cli/interruptible-sleep.js +npm notice 730B dist/cli/interruptible-sleep.js.map +npm notice 3.8kB dist/cli/run.d.ts +npm notice 2.8kB dist/cli/run.d.ts.map +npm notice 15.4kB dist/cli/run.js +npm notice 11.7kB dist/cli/run.js.map +npm notice 7.0kB dist/cli/tick-runner.d.ts +npm notice 1.9kB dist/cli/tick-runner.d.ts.map +npm notice 12.1kB dist/cli/tick-runner.js +npm notice 8.0kB dist/cli/tick-runner.js.map +npm notice 1.9kB dist/compile.d.ts +npm notice 888B dist/compile.d.ts.map +npm notice 25.6kB dist/compile.js +npm notice 22.4kB dist/compile.js.map +npm notice 618B dist/create-flow.d.ts +npm notice 535B dist/create-flow.d.ts.map +npm notice 4.1kB dist/create-flow.js +npm notice 3.4kB dist/create-flow.js.map +npm notice 5.8kB dist/daemon-connection.d.ts +npm notice 2.5kB dist/daemon-connection.d.ts.map +npm notice 10.6kB dist/daemon-connection.js +npm notice 6.6kB dist/daemon-connection.js.map +npm notice 1.5kB dist/daemon-lifecycle.d.ts +npm notice 594B dist/daemon-lifecycle.d.ts.map +npm notice 6.7kB dist/daemon-lifecycle.js +npm notice 4.3kB dist/daemon-lifecycle.js.map +npm notice 56B dist/demo-hn-monitor.d.ts +npm notice 124B dist/demo-hn-monitor.d.ts.map +npm notice 6.4kB dist/demo-hn-monitor.js +npm notice 4.5kB dist/demo-hn-monitor.js.map +npm notice 2.9kB dist/dir-watcher-poller.d.ts +npm notice 918B dist/dir-watcher-poller.d.ts.map +npm notice 3.2kB dist/dir-watcher-poller.js +npm notice 1.8kB dist/dir-watcher-poller.js.map +npm notice 540B dist/direct-input.d.ts +npm notice 445B dist/direct-input.d.ts.map +npm notice 2.3kB dist/direct-input.js +npm notice 2.1kB dist/direct-input.js.map +npm notice 3.3kB dist/failure-kinds.d.ts +npm notice 877B dist/failure-kinds.d.ts.map +npm notice 3.2kB dist/failure-kinds.js +npm notice 1.3kB dist/failure-kinds.js.map +npm notice 1.1kB dist/gate-contract.d.ts +npm notice 651B dist/gate-contract.d.ts.map +npm notice 1.5kB dist/gate-contract.js +npm notice 1.4kB dist/gate-contract.js.map +npm notice 2.3kB dist/hn-poller.d.ts +npm notice 812B dist/hn-poller.d.ts.map +npm notice 3.3kB dist/hn-poller.js +npm notice 1.8kB dist/hn-poller.js.map +npm notice 4.0kB dist/index.d.ts +npm notice 3.0kB dist/index.d.ts.map +npm notice 2.2kB dist/index.js +npm notice 1.3kB dist/index.js.map +npm notice 7.3kB dist/journal-client.d.ts +npm notice 3.9kB dist/journal-client.d.ts.map +npm notice 12.2kB dist/journal-client.js +npm notice 8.8kB dist/journal-client.js.map +npm notice 351B dist/json-schema-bound.d.ts +npm notice 242B dist/json-schema-bound.d.ts.map +npm notice 13.8kB dist/json-schema-bound.js +npm notice 13.0kB dist/json-schema-bound.js.map +npm notice 379B dist/json-schema.d.ts +npm notice 386B dist/json-schema.d.ts.map +npm notice 3.6kB dist/json-schema.js +npm notice 2.3kB dist/json-schema.js.map +npm notice 294B dist/json-value.d.ts +npm notice 333B dist/json-value.d.ts.map +npm notice 3.8kB dist/json-value.js +npm notice 3.8kB dist/json-value.js.map +npm notice 336B dist/local-agent.d.ts +npm notice 349B dist/local-agent.d.ts.map +npm notice 893B dist/local-agent.js +npm notice 863B dist/local-agent.js.map +npm notice 349B dist/model-name.d.ts +npm notice 189B dist/model-name.d.ts.map +npm notice 731B dist/model-name.js +npm notice 570B dist/model-name.js.map +npm notice 372B dist/output-schema.d.ts +npm notice 350B dist/output-schema.d.ts.map +npm notice 1.2kB dist/output-schema.js +npm notice 933B dist/output-schema.js.map +npm notice 3.0kB dist/preflight.d.ts +npm notice 2.2kB dist/preflight.d.ts.map +npm notice 15.7kB dist/preflight.js +npm notice 11.6kB dist/preflight.js.map +npm notice 761B dist/progress.d.ts +npm notice 690B dist/progress.d.ts.map +npm notice 1.9kB dist/progress.js +npm notice 1.9kB dist/progress.js.map +npm notice 11.3kB dist/protocol.d.ts +npm notice 7.7kB dist/protocol.d.ts.map +npm notice 868B dist/protocol.js +npm notice 344B dist/protocol.js.map +npm notice 1.8kB dist/relayflowd-path.d.ts +npm notice 1.0kB dist/relayflowd-path.d.ts.map +npm notice 6.3kB dist/relayflowd-path.js +npm notice 4.9kB dist/relayflowd-path.js.map +npm notice 12.4kB dist/spec.d.ts +npm notice 6.0kB dist/spec.d.ts.map +npm notice 764B dist/spec.js +npm notice 279B dist/spec.js.map +npm notice 161B dist/step-dependencies.d.ts +npm notice 235B dist/step-dependencies.d.ts.map +npm notice 3.6kB dist/step-dependencies.js +npm notice 3.4kB dist/step-dependencies.js.map +npm notice 1.1kB dist/step-fields.d.ts +npm notice 303B dist/step-fields.d.ts.map +npm notice 1.1kB dist/step-fields.js +npm notice 708B dist/step-fields.js.map +npm notice 9.9kB dist/tick-source.d.ts +npm notice 2.0kB dist/tick-source.d.ts.map +npm notice 11.7kB dist/tick-source.js +npm notice 4.4kB dist/tick-source.js.map +npm notice 373B dist/unknown-keys.d.ts +npm notice 267B dist/unknown-keys.d.ts.map +npm notice 1.9kB dist/unknown-keys.js +npm notice 2.2kB dist/unknown-keys.js.map +npm notice 262B dist/validate.d.ts +npm notice 263B dist/validate.d.ts.map +npm notice 21.5kB dist/validate.js +npm notice 19.0kB dist/validate.js.map +npm notice 1.6kB dist/work-package-consumer.d.ts +npm notice 741B dist/work-package-consumer.d.ts.map +npm notice 2.1kB dist/work-package-consumer.js +npm notice 1.6kB dist/work-package-consumer.js.map +npm notice 731B dist/work-package-validator.d.ts +npm notice 489B dist/work-package-validator.d.ts.map +npm notice 3.4kB dist/work-package-validator.js +npm notice 2.9kB dist/work-package-validator.js.map +npm notice 859B dist/worker-cli.d.ts +npm notice 578B dist/worker-cli.d.ts.map +npm notice 4.2kB dist/worker-cli.js +npm notice 3.8kB dist/worker-cli.js.map +npm notice 391B dist/worker-lease.d.ts +npm notice 390B dist/worker-lease.d.ts.map +npm notice 3.2kB dist/worker-lease.js +npm notice 3.0kB dist/worker-lease.js.map +npm notice 2.5kB dist/worker.d.ts +npm notice 901B dist/worker.d.ts.map +npm notice 5.8kB dist/worker.js +npm notice 3.7kB dist/worker.js.map +npm notice 619B dist/wrapper-runtime.d.ts +npm notice 498B dist/wrapper-runtime.d.ts.map +npm notice 2.6kB dist/wrapper-runtime.js +npm notice 2.4kB dist/wrapper-runtime.js.map +npm notice 707B dist/wrapper-session.d.ts +npm notice 604B dist/wrapper-session.d.ts.map +npm notice 12.3kB dist/wrapper-session.js +npm notice 9.3kB dist/wrapper-session.js.map +npm notice 1.7kB package.json +npm notice 979B src/authored-flow-error.ts +npm notice 22.7kB src/authored-flow-executor.ts +npm notice 13.2kB src/authored-flow-lifecycle.ts +npm notice 4.7kB src/authored-flow-loader.ts +npm notice 8.8kB src/authored-flow-operation.ts +npm notice 709B src/authored-flow.ts +npm notice 8.4kB src/authored-promise-graph.ts +npm notice 6.0kB src/backlog-picker.ts +npm notice 2.5kB src/canonical.ts +npm notice 3.7kB src/cli-adapter.ts +npm notice 312B src/cli-executable.ts +npm notice 16.0kB src/cli.ts +npm notice 13.9kB src/cli/check.ts +npm notice 2.1kB src/cli/daemon-refusal.ts +npm notice 4.8kB src/cli/direct-run.ts +npm notice 11.5kB src/cli/hn-monitor.ts +npm notice 1.1kB src/cli/interruptible-sleep.ts +npm notice 17.0kB src/cli/run.ts +npm notice 15.3kB src/cli/tick-runner.ts +npm notice 25.4kB src/compile.ts +npm notice 4.3kB src/create-flow.ts +npm notice 12.8kB src/daemon-connection.ts +npm notice 7.1kB src/daemon-lifecycle.ts +npm notice 6.1kB src/demo-hn-monitor.ts +npm notice 4.2kB src/dir-watcher-poller.ts +npm notice 2.3kB src/direct-input.ts +npm notice 3.6kB src/failure-kinds.ts +npm notice 2.2kB src/gate-contract.ts +npm notice 3.8kB src/hn-poller.ts +npm notice 5.0kB src/index.ts +npm notice 14.4kB src/journal-client.ts +npm notice 13.9kB src/json-schema-bound.ts +npm notice 3.4kB src/json-schema.ts +npm notice 4.1kB src/json-value.ts +npm notice 970B src/local-agent.ts +npm notice 682B src/model-name.ts +npm notice 1.4kB src/output-schema.ts +npm notice 18.0kB src/preflight.ts +npm notice 2.2kB src/progress.ts +npm notice 11.3kB src/protocol.ts +npm notice 6.9kB src/relayflowd-path.ts +npm notice 13.3kB src/spec.ts +npm notice 3.5kB src/step-dependencies.ts +npm notice 1.1kB src/step-fields.ts +npm notice 14.9kB src/tick-source.ts +npm notice 1.9kB src/unknown-keys.ts +npm notice 21.2kB src/validate.ts +npm notice 2.9kB src/work-package-consumer.ts +npm notice 3.8kB src/work-package-validator.ts +npm notice 4.2kB src/worker-cli.ts +npm notice 3.2kB src/worker-lease.ts +npm notice 6.1kB src/worker.ts +npm notice 2.6kB src/wrapper-runtime.ts +npm notice 11.8kB src/wrapper-session.ts +npm notice Tarball Details +npm notice name: @relayflows/sdk +npm notice version: 2.0.8 +npm notice filename: relayflows-sdk-2.0.8.tgz +npm notice package size: 285.1 kB +npm notice unpacked size: 1.2 MB +npm notice shasum: 2d140a2076f1d37939b0353cac1feb31d239d1af +npm notice integrity: sha512-Nu8WzQUIGHHp8[...]oncmJE/y0Omig== +npm notice total files: 271 +npm notice +relayflows-sdk-2.0.8.tgz +npm notice +npm notice 📦 relayflows@2.0.8 +npm notice Tarball Contents +npm notice 1.5kB README.md +npm notice 123B bin/flows.js +npm notice 712B package.json +npm notice Tarball Details +npm notice name: relayflows +npm notice version: 2.0.8 +npm notice filename: relayflows-2.0.8.tgz +npm notice package size: 1.3 kB +npm notice unpacked size: 2.4 kB +npm notice shasum: 8ddf134b9982fc446c5729c69715b548fe599991 +npm notice integrity: sha512-O2IgcbeQtIw0F[...]jXbNze+a2jSDw== +npm notice total files: 3 +npm notice +relayflows-2.0.8.tgz +EXIT_CODE=0 +$ shasum -a 256 /tmp/ws13-artifacts/relayflows-sdk-2.0.8.tgz /tmp/ws13-followup-artifacts/relayflows-sdk-2.0.8.tgz /tmp/ws13-review-artifacts/*.tgz +6c1986cb526f7e348190be83b4665dd2094e5434bd8429b2bfab7e0cf077b0f6 /tmp/ws13-artifacts/relayflows-sdk-2.0.8.tgz +ba58cee2b966299e3c224c20097d8f336848aebbca26cf092a02aade6ea41bac /tmp/ws13-followup-artifacts/relayflows-sdk-2.0.8.tgz +ae555a4aa2a65b15fe934286d158d2b5477ad7a1eba8fbf738889416475ec7c0 /tmp/ws13-review-artifacts/relayflows-2.0.8.tgz +b7d0cc50aa4bd76fe577d6a07bf865bb37b7944e0e1f277d0d8743f281e5bbda /tmp/ws13-review-artifacts/relayflows-sdk-2.0.8.tgz diff --git a/docs/evidence/ws13/review/packed-launcher-live.txt b/docs/evidence/ws13/review/packed-launcher-live.txt new file mode 100644 index 00000000..0c7f72a1 --- /dev/null +++ b/docs/evidence/ws13/review/packed-launcher-live.txt @@ -0,0 +1,12 @@ +$ FLOWS_TEST_CLI=/tmp/ws13-gallery-review/node_modules/relayflows/bin/flows.js RELAYFLOWD_BIN=/tmp/ws13-gallery-review/node_modules/@relayflows/runtime-darwin-arm64/bin/relayflowd node node_modules/vitest/vitest.mjs run tests/local-agent-live.test.ts -t "runs beyond the initial 30-second lease" --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/local-agent-live.test.ts > built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 42068ms + + Test Files 1 passed (1) + Tests 1 passed | 4 skipped (5) + Start at 22:15:11 + Duration 44.64s (transform 383ms, setup 0ms, collect 98ms, tests 42.07s, environment 0ms, prepare 1.18s) + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/threads.md b/docs/evidence/ws13/review/threads.md new file mode 100644 index 00000000..cf7e1e57 --- /dev/null +++ b/docs/evidence/ws13/review/threads.md @@ -0,0 +1,35 @@ +# Disposition of the 29 original review threads + +Code fixes: `e3f756c`. Links point to each original review comment. Declines are explicit; this ledger is not independent approval. + +| # | Thread | Disposition | +|---|---|---| +| 1 | [packages/sdk/src/local-agent.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961238121) | Fixed | +| 2 | [packages/sdk/src/local-agent.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242667) | Fixed | +| 3 | [packages/sdk/src/progress.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242675) | Fixed | +| 4 | [docs/evidence/ws13/stage-registry.mjs](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242679) | Fixed earlier | +| 5 | [docs/evidence/ws13/stage-registry.mjs](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242693) | Fixed earlier | +| 6 | [docs/evidence/ws13/record.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242695) | Fixed | +| 7 | [docs/evidence/ws13/record.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242699) | Fixed | +| 8 | [docs/evidence/ws13/gallery-research.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242703) | Superseded | +| 9 | [docs/evidence/ws13/followup/run-gallery.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242706) | Fixed earlier | +| 10 | [docs/evidence/ws13/followup/run-gallery.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242710) | Fixed earlier | +| 11 | [docs/evidence/ws13/container-tests.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242712) | Verified narrower scope | +| 12 | [packages/sdk/src/cli.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242720) | Fixed | +| 13 | [docs/evidence/ws13/cold-clone-direct.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242741) | Declined timing rerun; clarified | +| 14 | [packages/sdk/src/cli/direct-run.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242750) | Fixed | +| 15 | [packages/sdk/tests/local-agent-live.test.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242756) | Fixed lazy discovery; declined silent skip | +| 16 | [docs/evidence/ws13/research-typecheck.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242763) | Superseded | +| 17 | [docs/evidence/ws13/local-agent-tests-final.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242779) | Declined rewriting captured output | +| 18 | [docs/evidence/ws13/cold-clone.sh](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242784) | Declined source-build reinterpretation | +| 19 | [docs/evidence/ws13/gallery-pr-review-pipeline.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242805) | Superseded | +| 20 | [docs/evidence/ws13/cold-start.sh](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242809) | Fixed | +| 21 | [docs/evidence/ws13/cold-trixie.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961242817) | Clarified | +| 22 | [docs/evidence/ws13/followup/default-budget/reports/synthesis.md](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433476) | Fixed | +| 23 | [packages/sdk/src/worker-lease.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433484) | Fixed | +| 24 | [docs/evidence/ws13/followup/final-sdk/artifact.json](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433491) | Fixed provenance; distinct revisions | +| 25 | [packages/sdk/src/worker-cli.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433497) | Fixed by failing closed; Windows tree implementation declined | +| 26 | [docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433511) | Corrected report and verified real launcher | +| 27 | [docs/evidence/ws13/followup/run-gallery.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433523) | Fixed | +| 28 | [docs/evidence/ws13/followup/default-budget/reports/claude.md](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433529) | Fixed | +| 29 | [docs/evidence/ws13/followup/default-budget/reports/grok.md](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433543) | Fixed | diff --git a/docs/evidence/ws13/review/typechecks.txt b/docs/evidence/ws13/review/typechecks.txt new file mode 100644 index 00000000..9831caa8 --- /dev/null +++ b/docs/evidence/ws13/review/typechecks.txt @@ -0,0 +1,10 @@ +$ npm run typecheck && npm run typecheck:tests + +> @relayflows/sdk@2.0.8 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.8 typecheck:tests +> tsc -p tsconfig.tests.json + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/verify-helpers.py b/docs/evidence/ws13/review/verify-helpers.py new file mode 100644 index 00000000..db683ada --- /dev/null +++ b/docs/evidence/ws13/review/verify-helpers.py @@ -0,0 +1,45 @@ +"""Exercise recorder EOF/normalization and harness refusal paths without agents.""" +from pathlib import Path +import json +import os +import subprocess +import sys +import tempfile + +base = Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix='ws13-helper-check-') as directory: + root = Path(directory) + prefix = root / 'terminal' + argv = [sys.executable, str(base / 'record.py'), str(prefix), directory, + sys.executable, '-c', "import os; os.write(1, b'trailing \\n\\xe2\\x82')"] + result = subprocess.run(argv, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + transcript = prefix.with_suffix('.txt').read_text() + frames = [json.loads(line) for line in prefix.with_suffix('.cast').read_text().splitlines()] + terminal = ''.join(frame[2] for frame in frames[1:]) + assert '\ufffd' in terminal and '\ufffd' in transcript + assert 'trailing \r\n' in terminal + assert '\ntrailing\n' in transcript + assert all(line == line.rstrip() for line in transcript.splitlines()) + print('PASS: EOF UTF-8 replacement is captured in cast and text; text trims trailing spaces; cast preserves them.') + for name in ['cold-clone.sh', 'cold-start.sh']: + result = subprocess.run(['bash', str(base / name)], text=True, capture_output=True) + assert result.returncode != 0 and 'Usage:' in result.stderr, result + print(f'PASS: {name} without registry refuses with usage: {result.stderr.strip()}') + older = root / 'node' + older.write_text('#!/bin/sh\nprintf "20.19.0\\n"\n') + older.chmod(0o755) + result = subprocess.run([sys.executable, str(base / 'followup/run-gallery.py'), directory, + str(root / 'older-evidence'), 'sdk-only'], + env={**os.environ, 'PATH': directory}, text=True, capture_output=True) + assert result.returncode != 0 and 'found 20.19.0' in result.stderr, result + print('PASS: older Node refuses before any gallery command: ' + result.stderr.strip()) + occupied = root / 'occupied' + occupied.mkdir() + capture = occupied / 'gallery-existing.txt' + capture.write_text('original evidence') + result = subprocess.run([sys.executable, str(base / 'followup/run-gallery.py'), directory, + str(occupied), 'sdk-only'], text=True, capture_output=True) + assert result.returncode != 0 and 'will not be overwritten' in result.stderr + assert capture.read_text() == 'original evidence' + print('PASS: existing gallery capture preserved; overwrite refused.') diff --git a/docs/evidence/ws13/review/verify-installed.py b/docs/evidence/ws13/review/verify-installed.py new file mode 100644 index 00000000..f161142f --- /dev/null +++ b/docs/evidence/ws13/review/verify-installed.py @@ -0,0 +1,23 @@ +"""Verify every installed candidate file and SDK resolution from the launcher.""" +from pathlib import Path +import hashlib +import subprocess +import tarfile + +root = Path('/tmp/ws13-gallery-review') +for package, archive in [ + ('@relayflows/sdk', '/tmp/ws13-review-artifacts/relayflows-sdk-2.0.8.tgz'), + ('relayflows', '/tmp/ws13-review-artifacts/relayflows-2.0.8.tgz'), + ('@relayflows/runtime-darwin-arm64', '/tmp/ws13-artifacts/relayflows-runtime-darwin-arm64-2.0.8.tgz'), +]: + with tarfile.open(archive) as tar: + members = [member for member in tar if member.isfile()] + for member in members: + installed = root / 'node_modules' / package / member.name.removeprefix('package/') + assert installed.read_bytes() == tar.extractfile(member).read(), str(installed) + print(f'PASS: {package}: all {len(members)} installed files match {archive}') + print(f'SHA256={hashlib.sha256(Path(archive).read_bytes()).hexdigest()}') +resolved = subprocess.check_output(['node', '--experimental-import-meta-resolve', '--input-type=module', '-e', + "import {pathToFileURL,fileURLToPath} from 'node:url'; console.log(fileURLToPath(import.meta.resolve('@relayflows/sdk/cli', pathToFileURL(process.cwd()+'/node_modules/relayflows/bin/flows.js'))));"], cwd=root, text=True).strip() +assert Path(resolved).resolve() == (root / 'node_modules/@relayflows/sdk/dist/cli.js').resolve(), resolved +print('PASS: launcher resolves the verified candidate SDK: ' + resolved) diff --git a/examples/README.md b/examples/README.md index bf73b77a..606dad24 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,10 +8,15 @@ For a working local starting point, use the [small agent starter](../README.md) | Example | Status | Observed result | Elapsed | |---|---|---|---:| -| [dependency-upgrade-bot](dependency-upgrade-bot/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.138s](../docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt) | -| [pr-review-pipeline](pr-review-pipeline/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [0.143s](../docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt) | +| [dependency-upgrade-bot](dependency-upgrade-bot/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [5.138s](../docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt) | +| [pr-review-pipeline](pr-review-pipeline/) | **BLOCKED** | SDK refuses unsupported `budget` header before entering the body; exit 2 | [3.539s](../docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt) | | [research](research/) | **PASS** | All model probes passed; three lane reports and synthesis produced; exit 0, `completionReason: synthesized` | [690.935s](../docs/evidence/ws13/followup/default-budget/gallery-research.txt) | +**Correction:** the previously listed 0.138s and 0.143s captures used a stale +launcher and returned `invalid_invocation`. They did not establish budget +refusals. The current values above come from a fresh, +[verified candidate install](../docs/evidence/ws13/review/installed-identity.txt). + Each link contains the literal command, captured output, exit code and timing. These are individual runs from a separate clone on an authenticated macOS host, against the packed candidate CLI. Research uses its documented source diff --git a/examples/dependency-upgrade-bot/README.md b/examples/dependency-upgrade-bot/README.md index 7d60217d..79a311b2 100644 --- a/examples/dependency-upgrade-bot/README.md +++ b/examples/dependency-upgrade-bot/README.md @@ -1,8 +1,8 @@ # dependency-upgrade-bot **BLOCKED — not runnable on the current authored executor.** The candidate -CLI refuses the `budget` header before any step runs (exit 2, 0.138s). -[Exact command and captured output](../../docs/evidence/ws13/followup/final-sdk/gallery-dependency-upgrade-bot.txt). +CLI refuses the `budget` header before any step runs (exit 2, 5.138s). +[Exact command and captured output](../../docs/evidence/ws13/review/gallery/gallery-dependency-upgrade-bot.txt). The SDK/kernel capability owner must supply budget-header support, postfix artifact gates, and the declared workspace behavior before this example can be advertised as working. Its existing requirements remain intact. diff --git a/examples/pr-review-pipeline/README.md b/examples/pr-review-pipeline/README.md index 82eca89a..4e3f11c9 100644 --- a/examples/pr-review-pipeline/README.md +++ b/examples/pr-review-pipeline/README.md @@ -1,8 +1,8 @@ # pr-review-pipeline **BLOCKED — not runnable on the current authored executor.** The candidate -CLI refuses the `budget` header before any step runs (exit 2, 0.143s). -[Exact command and captured output](../../docs/evidence/ws13/followup/final-sdk/gallery-pr-review-pipeline.txt). +CLI refuses the `budget` header before any step runs (exit 2, 3.539s). +[Exact command and captured output](../../docs/evidence/ws13/review/gallery/gallery-pr-review-pipeline.txt). The SDK/kernel capability owner must supply budget-header support, postfix artifact gates, and the declared workspace behavior before this example can be advertised as working. Its existing requirements remain intact. From 8abb4db0cc736b865c627759f96ab23cbb12614e Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 22:23:04 +0200 Subject: [PATCH 12/16] fix: encode terminal controls in review evidence for safe diff preparation Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/review/README.md | 4 +++ docs/evidence/ws13/review/cloud-failure.txt | 3 +- docs/evidence/ws13/review/prepare-failure.txt | 31 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 docs/evidence/ws13/review/prepare-failure.txt diff --git a/docs/evidence/ws13/review/README.md b/docs/evidence/ws13/review/README.md index d4625d00..2045a4d9 100644 --- a/docs/evidence/ws13/review/README.md +++ b/docs/evidence/ws13/review/README.md @@ -59,6 +59,10 @@ No new timing benchmark was run. returned `relayfile ACL GET /.relayfile.acl failed with status 429`, correlation `499e3981-c303-48e3-89be-595ac66ee3c4`. [Captured failure](cloud-failure.txt). All three fresh review transcripts are missing; this is **not review signoff**. +The first retry at `14cb174` failed sooner: `gh pr diff` refused two literal +ANSI escape bytes in this newly captured Cloud log. That was an evidence-format +mistake in this PR. The log now encodes ESC as visible `\u001b` text, preserving +the captured content without terminal controls. [Failed preparation](prepare-failure.txt). The gate is unchanged. See the PR for the retry status at the current head. **Release-gate owner:** register and publish `create-flow`; publishing remains diff --git a/docs/evidence/ws13/review/cloud-failure.txt b/docs/evidence/ws13/review/cloud-failure.txt index e2ddbf09..98a82daa 100644 --- a/docs/evidence/ws13/review/cloud-failure.txt +++ b/docs/evidence/ws13/review/cloud-failure.txt @@ -1,9 +1,10 @@ +# Rendering: literal ANSI ESC bytes are encoded as \u001b for a safe text diff. $ gh run view 34267938676 --log | sed -n '498,513p' review Wait for cloud swarm 2026-09-08T19:16:55.9809793Z ##[endgroup] review Wait for cloud swarm 2026-09-08T19:19:31.1680377Z swarm failure reason: review Wait for cloud swarm 2026-09-08T19:19:31.1682875Z relayfile ACL GET /.relayfile.acl failed with status 429 (correlationId=499e3981-c303-48e3-89be-595ac66ee3c4) review Post verdict and transcripts 2026-09-08T19:19:31.1733020Z ##[group]Run ../gate-files/.github/workflows/scripts/swarm-post.sh "48b040cc-52e7-49ce-8dc3-f24df51b8687" "247" -review Post verdict and transcripts 2026-09-08T19:19:31.1733822Z ../gate-files/.github/workflows/scripts/swarm-post.sh "48b040cc-52e7-49ce-8dc3-f24df51b8687" "247" +review Post verdict and transcripts 2026-09-08T19:19:31.1733822Z \u001b[36;1m../gate-files/.github/workflows/scripts/swarm-post.sh "48b040cc-52e7-49ce-8dc3-f24df51b8687" "247"\u001b[0m review Post verdict and transcripts 2026-09-08T19:19:31.1754377Z shell: /usr/bin/bash -e {0} review Post verdict and transcripts 2026-09-08T19:19:31.1754716Z env: review Post verdict and transcripts 2026-09-08T19:19:31.1755052Z CLOUD_API_URL: https://agentrelay.com/cloud diff --git a/docs/evidence/ws13/review/prepare-failure.txt b/docs/evidence/ws13/review/prepare-failure.txt new file mode 100644 index 00000000..44188830 --- /dev/null +++ b/docs/evidence/ws13/review/prepare-failure.txt @@ -0,0 +1,31 @@ +# Rendering: literal ANSI ESC bytes are encoded as \u001b for a safe text diff. +$ gh run view 34274116824 --log-failed +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2009627Z ##[group]Run ../gate-files/.github/workflows/scripts/swarm-prepare.sh \ +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2010102Z \u001b[36;1m../gate-files/.github/workflows/scripts/swarm-prepare.sh \\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2010393Z \u001b[36;1m "247"\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2010601Z \u001b[36;1mmkdir -p .github/workflows/scripts\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2010912Z \u001b[36;1mcp ../gate-files/.github/workflows/scripts/swarm-verdict.sh \\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2011237Z \u001b[36;1m .github/workflows/scripts/swarm-verdict.sh\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2011539Z \u001b[36;1mgit add -f .github/workflows/scripts/swarm-verdict.sh\u001b[0m +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2043163Z shell: /usr/bin/bash -e {0} +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2043380Z env: +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2043599Z CLOUD_API_URL: https://agentrelay.com/cloud +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2044021Z CLOUD_API_KEY: *** +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2044271Z RELAY_WORKSPACE_KEY: *** +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2044521Z RELAY_API_KEY: *** +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2046374Z GH_TOKEN: *** +review Prepare review input on GitHub runner 2026-09-08T20:20:46.2046556Z ##[endgroup] +review Prepare review input on GitHub runner 2026-09-08T20:20:47.4373935Z the diff contains terminal escape sequences; pass --allow-escape-sequences to output it anyway +review Prepare review input on GitHub runner 2026-09-08T20:20:47.4406557Z ##[error]Process completed with exit code 1. +review Enforce swarm result 2026-09-08T20:20:47.4493262Z ##[group]Run echo "Review swarm did not complete successfully: " >&2 +review Enforce swarm result 2026-09-08T20:20:47.4493651Z \u001b[36;1mecho "Review swarm did not complete successfully: " >&2\u001b[0m +review Enforce swarm result 2026-09-08T20:20:47.4493935Z \u001b[36;1mexit 1\u001b[0m +review Enforce swarm result 2026-09-08T20:20:47.4525582Z shell: /usr/bin/bash -e {0} +review Enforce swarm result 2026-09-08T20:20:47.4525791Z env: +review Enforce swarm result 2026-09-08T20:20:47.4526002Z CLOUD_API_URL: https://agentrelay.com/cloud +review Enforce swarm result 2026-09-08T20:20:47.4526411Z CLOUD_API_KEY: *** +review Enforce swarm result 2026-09-08T20:20:47.4526674Z RELAY_WORKSPACE_KEY: *** +review Enforce swarm result 2026-09-08T20:20:47.4526928Z RELAY_API_KEY: *** +review Enforce swarm result 2026-09-08T20:20:47.4527103Z ##[endgroup] +review Enforce swarm result 2026-09-08T20:20:47.4571247Z Review swarm did not complete successfully: +review Enforce swarm result 2026-09-08T20:20:47.4573991Z ##[error]Process completed with exit code 1. From 015e8a3db91cbba8c6e85a65c2f401dd37a6c696 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Tue, 8 Sep 2026 22:34:10 +0200 Subject: [PATCH 13/16] fix: reject late lease renewals and preserve transcript line boundaries Session-Id: 01a08114-4273-7951-9e4f-2d9fdaba25fb Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- docs/evidence/ws13/record.py | 2 +- docs/evidence/ws13/review/README.md | 40 ++++++++++++++----- docs/evidence/ws13/review/infra-final.txt | 5 +++ .../evidence/ws13/review/last-two-threads.txt | 31 ++++++++++++++ docs/evidence/ws13/review/threads.md | 6 ++- docs/evidence/ws13/review/verify-helpers.py | 7 ++-- packages/sdk/src/worker-lease.ts | 5 +++ packages/sdk/tests/worker-lease.test.ts | 32 +++++++++++++++ 8 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 docs/evidence/ws13/review/infra-final.txt create mode 100644 docs/evidence/ws13/review/last-two-threads.txt diff --git a/docs/evidence/ws13/record.py b/docs/evidence/ws13/record.py index 5ec7c2b2..b6f21e55 100644 --- a/docs/evidence/ws13/record.py +++ b/docs/evidence/ws13/record.py @@ -65,5 +65,5 @@ print(ending) # Normalize only the readable transcript; the cast retains terminal bytes. path = Path(prefix + '.txt') -path.write_text('\n'.join(line.rstrip() for line in path.read_text().splitlines()) + '\n') +path.write_text('\n'.join(line.rstrip() for line in path.read_text().removesuffix('\n').split('\n')) + '\n') sys.exit(code if code >= 0 else 128 - code) diff --git a/docs/evidence/ws13/review/README.md b/docs/evidence/ws13/review/README.md index 2045a4d9..189be9bd 100644 --- a/docs/evidence/ws13/review/README.md +++ b/docs/evidence/ws13/review/README.md @@ -1,6 +1,6 @@ # WS-13 review corrections -The original 29 threads are tracked individually in [the response ledger](threads.md). +The original 29 threads and two follow-up threads are tracked individually in [the response ledger](threads.md). The SDK/code fixes are in `e3f756c`; no kernel or judging gate was changed. ## Corrected gallery: 1 pass, 2 blocked @@ -52,18 +52,36 @@ repository test suite is green. Timing remains accepted by Khaliq: 49.975s for the cold deterministic loop, 132.637s for the existing-host real Claude command. No new timing benchmark was run. +## Two follow-up findings + +A heartbeat response handled after the prior deadline is now rejected before it +can replace the deadline snapshot. Initial and periodic late-response tests both +leave timer callbacks queued; no expired lease can spawn/complete work. The +recorder now splits only on LF, preserving embedded vertical-tab/form-feed bytes. +[Seven lease tests, recorder checks and SDK build passed](last-two-threads.txt). +The gallery captures above identify the earlier `e3f756c` candidate; these final +lease/recorder changes do not implement either missing gallery budget capability. +No new gallery or cold-timing execution is claimed for this follow-up. + ## External handoffs -**Review-swarm/Cloud relayfile owner:** the failed review job -[34267938676](https://github.com/AgentWorkforce/flows/actions/runs/34267938676) -returned `relayfile ACL GET /.relayfile.acl failed with status 429`, correlation -`499e3981-c303-48e3-89be-595ac66ee3c4`. [Captured failure](cloud-failure.txt). -All three fresh review transcripts are missing; this is **not review signoff**. -The first retry at `14cb174` failed sooner: `gh pr diff` refused two literal -ANSI escape bytes in this newly captured Cloud log. That was an evidence-format -mistake in this PR. The log now encodes ESC as visible `\u001b` text, preserving -the captured content without terminal controls. [Failed preparation](prepare-failure.txt). -The gate is unchanged. See the PR for the retry status at the current head. +**Review-swarm / Cloud + Relaycast service owner: INFRA-FAILED.** Per the +user's ruling, this check is infrastructure-owned and is not being repaired in +this lane. At `1aad66a`, preparation and launch passed, then Cloud run +`7202379b-3bcb-4706-b6e3-a7e59495c4e2` failed during Relaycast workspace-key repair +with HTTP 503, database temporarily overloaded. Post-verdict then reported: + +> No changes to sync — the workflow did not modify any files. + +No fresh maintainability, history or structure transcripts were produced. This +is not a verdict on the PR code and **not independent review signoff**. +[Exact command and captured failure](infra-final.txt), +[job](https://github.com/AgentWorkforce/flows/actions/runs/34274491229/job/102224017363). +The earlier [ACL HTTP 429](cloud-failure.txt) and [ANSI preparation failure](prepare-failure.txt) +remain captured. The latter was fixed in this PR by visibly encoding ESC bytes; +no judging gate was edited. One retry had already been queued before the user +ruled this out of scope; no further manual retries or infrastructure work follow. +The PR carries the current handoff; remain out of draft. **Release-gate owner:** register and publish `create-flow`; publishing remains outside this lane. Route both owner assignments through session-thread-rollout. diff --git a/docs/evidence/ws13/review/infra-final.txt b/docs/evidence/ws13/review/infra-final.txt new file mode 100644 index 00000000..2ec87afb --- /dev/null +++ b/docs/evidence/ws13/review/infra-final.txt @@ -0,0 +1,5 @@ +$ gh run view 34274491229 --attempt 1 --log | rg 'Z (swarm failure reason:| Relaycast workspace key repair|Fetching patch for run|No changes to sync)' +review Wait for cloud swarm 2026-09-08T20:28:37.0904334Z swarm failure reason: +review Wait for cloud swarm 2026-09-08T20:28:37.0908009Z Relaycast workspace key repair failed: 503 The database is temporarily overloaded. Retry after the interval in the Retry-After header. +review Post verdict and transcripts 2026-09-08T20:28:37.4838611Z Fetching patch for run 7202379b-3bcb-4706-b6e3-a7e59495c4e2... +review Post verdict and transcripts 2026-09-08T20:28:39.3397078Z No changes to sync — the workflow did not modify any files. diff --git a/docs/evidence/ws13/review/last-two-threads.txt b/docs/evidence/ws13/review/last-two-threads.txt new file mode 100644 index 00000000..2a07e6ba --- /dev/null +++ b/docs/evidence/ws13/review/last-two-threads.txt @@ -0,0 +1,31 @@ +$ node node_modules/vitest/vitest.mjs run tests/worker-lease.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/packages/sdk + + ✓ tests/worker-lease.test.ts > worker lease ownership > renews the same attempt through a long subprocess and drains before completing once + ✓ tests/worker-lease.test.ts > worker lease ownership > aborts execution and never completes after a rejected heartbeat + ✓ tests/worker-lease.test.ts > worker lease ownership > expires locally when a renewal response never arrives, without stranding close + ✓ tests/worker-lease.test.ts > worker lease ownership > does not spawn a process for an already-expired dispatch + ✓ tests/worker-lease.test.ts > refuses completion past the deadline even before the expiry timer runs + ✓ tests/worker-lease.test.ts > does not revive ownership when the initial heartbeat response is handled late + ✓ tests/worker-lease.test.ts > aborts the CLI when a later heartbeat response would revive an expired lease + + Test Files 1 passed (1) + Tests 7 passed (7) + Start at 22:32:44 + Duration 187ms (transform 49ms, setup 0ms, collect 47ms, tests 6ms, environment 0ms, prepare 38ms) + +EXIT_CODE=0 +$ python3 docs/evidence/ws13/review/verify-helpers.py +PASS: EOF UTF-8 replacement is captured in cast and text; text trims trailing spaces; cast preserves them; embedded VT/FF remain on the same line. +PASS: cold-clone.sh without registry refuses with usage: /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/docs/evidence/ws13/cold-clone.sh: line 3: 1: Usage: cold-clone.sh CANDIDATE_REGISTRY_URL +PASS: cold-start.sh without registry refuses with usage: /Users/khaliqgant/Projects/AgentWorkforce/.worktrees/ws13-flows-local/docs/evidence/ws13/cold-start.sh: line 4: 1: Usage: cold-start.sh CANDIDATE_REGISTRY_URL +PASS: older Node refuses before any gallery command: Node 22.18+ is required on PATH; found 20.19.0 at /var/folders/6d/0x5fkt8d01gfmmjdzkxqzwnh0000gn/T/ws13-helper-check-sc5m2_yh/node. +PASS: existing gallery capture preserved; overwrite refused. +EXIT_CODE=0 +$ npm --prefix packages/sdk run build + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + +EXIT_CODE=0 diff --git a/docs/evidence/ws13/review/threads.md b/docs/evidence/ws13/review/threads.md index cf7e1e57..52a76b17 100644 --- a/docs/evidence/ws13/review/threads.md +++ b/docs/evidence/ws13/review/threads.md @@ -1,6 +1,6 @@ -# Disposition of the 29 original review threads +# Disposition of 31 review threads -Code fixes: `e3f756c`. Links point to each original review comment. Declines are explicit; this ledger is not independent approval. +Initial 29-thread code fixes: `e3f756c`. The two follow-up findings are covered by [the additional captured verification](last-two-threads.txt). Links point to each original review comment. Declines are explicit; this ledger is not independent approval. | # | Thread | Disposition | |---|---|---| @@ -33,3 +33,5 @@ Code fixes: `e3f756c`. Links point to each original review comment. Declines are | 27 | [docs/evidence/ws13/followup/run-gallery.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433523) | Fixed | | 28 | [docs/evidence/ws13/followup/default-budget/reports/claude.md](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433529) | Fixed | | 29 | [docs/evidence/ws13/followup/default-budget/reports/grok.md](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961433543) | Fixed | +| 30 | [worker-lease.ts](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961970551) | Fixed: a late heartbeat cannot revive an expired lease | +| 31 | [record.py](https://github.com/AgentWorkforce/flows/pull/247#discussion_r3961970578) | Fixed: normalize only actual newline boundaries | diff --git a/docs/evidence/ws13/review/verify-helpers.py b/docs/evidence/ws13/review/verify-helpers.py index db683ada..35bc5e95 100644 --- a/docs/evidence/ws13/review/verify-helpers.py +++ b/docs/evidence/ws13/review/verify-helpers.py @@ -11,7 +11,7 @@ root = Path(directory) prefix = root / 'terminal' argv = [sys.executable, str(base / 'record.py'), str(prefix), directory, - sys.executable, '-c', "import os; os.write(1, b'trailing \\n\\xe2\\x82')"] + sys.executable, '-c', "import os; os.write(1, b'trailing \\nvertical\\x0btab form\\x0cfeed\\n\\xe2\\x82')"] result = subprocess.run(argv, text=True, capture_output=True) assert result.returncode == 0, result.stderr transcript = prefix.with_suffix('.txt').read_text() @@ -20,8 +20,9 @@ assert '\ufffd' in terminal and '\ufffd' in transcript assert 'trailing \r\n' in terminal assert '\ntrailing\n' in transcript - assert all(line == line.rstrip() for line in transcript.splitlines()) - print('PASS: EOF UTF-8 replacement is captured in cast and text; text trims trailing spaces; cast preserves them.') + assert '\nvertical\x0btab form\x0cfeed\n' in transcript + assert all(line == line.rstrip() for line in transcript.removesuffix('\n').split('\n')) + print('PASS: EOF UTF-8 replacement is captured in cast and text; text trims trailing spaces; cast preserves them; embedded VT/FF remain on the same line.') for name in ['cold-clone.sh', 'cold-start.sh']: result = subprocess.run(['bash', str(base / name)], text=True, capture_output=True) assert result.returncode != 0 and 'Usage:' in result.stderr, result diff --git a/packages/sdk/src/worker-lease.ts b/packages/sdk/src/worker-lease.ts index dcf3488f..5424b7e3 100644 --- a/packages/sdk/src/worker-lease.ts +++ b/packages/sdk/src/worker-lease.ts @@ -31,6 +31,11 @@ export async function withWorkerLease( dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.lease_id, ), controller.signal); controller.signal.throwIfAborted(); + // A response handled after local expiry cannot revive ownership, even + // if its future deadline was issued before this event loop stalled. + if (Date.now() >= latestDeadline) { + throw new Error(`Agent lease expired before renewal for ${dispatch.run_id}/${dispatch.step_id}.`); + } const remaining = armExpiry(result.lease_deadline_ms); if (!stopped) { renewalTimer = setTimeout(() => { diff --git a/packages/sdk/tests/worker-lease.test.ts b/packages/sdk/tests/worker-lease.test.ts index ce2e09f1..ef4c2f93 100644 --- a/packages/sdk/tests/worker-lease.test.ts +++ b/packages/sdk/tests/worker-lease.test.ts @@ -117,3 +117,35 @@ it('refuses completion past the deadline even before the expiry timer runs', asy expect(client.stepComplete).not.toHaveBeenCalled(); expect(String(errors[0])).toContain('lease expired before completion'); }); + +it('does not revive ownership when the initial heartbeat response is handled late', async () => { + const { client, worker, errors, dispatch } = setup(); + client.stepHeartbeat.mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 30_001); // leave timer callbacks queued + return { lease_deadline_ms: Date.now() + 30_000 }; + }); + await worker.attach(); + client.emit('step.dispatch', dispatch); + await worker.close(); + expect(runAgentCli).not.toHaveBeenCalled(); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(String(errors[0])).toContain('lease expired before renewal'); +}); + +it('aborts the CLI when a later heartbeat response would revive an expired lease', async () => { + const { client, worker, errors, dispatch } = setup(); + client.stepHeartbeat.mockResolvedValueOnce({ lease_deadline_ms: Date.now() + 30_000 }) + .mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 20_001); // renewal starts at t=10s + return { lease_deadline_ms: Date.now() + 30_000 }; + }); + const signals = runningCli(60_000); + await worker.attach(); + client.emit('step.dispatch', dispatch); + await vi.advanceTimersByTimeAsync(10_000); + await worker.close(); + expect(client.stepHeartbeat).toHaveBeenCalledTimes(2); + expect(signals[0]?.aborted).toBe(true); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(String(errors[0])).toContain('lease expired before renewal'); +}); From b61f81facb174dbdd332bbaef00a111abc4e8e5a Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Wed, 9 Sep 2026 14:43:04 +0200 Subject: [PATCH 14/16] test: pin the rebase resolution of --cloud against --local-agent The rebase onto main put two independently-authored `flows run` flags in one parser for the first time: `--cloud` (#246), which hands the spec to Cloud, and `--local-agent` (#247), which attaches a wrapper process on this machine. Neither branch could have known about the other, so nothing decided what the pair means. `--cloud` already refuses every other flag that only describes a local run -- `--input`, `--data-dir`, `--no-spawn` -- because on a Cloud run they describe nothing. `--local-agent` is the same kind of flag, so it gets the same answer rather than being accepted and silently dropped, and this case joins the table that states it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SSL8So4wFQqJkLa4kRZDEc Session-Id: 49bfa1cd-6bfe-47c2-af22-6cd0529ce49f --- packages/sdk/tests/cloud-run.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/sdk/tests/cloud-run.test.ts b/packages/sdk/tests/cloud-run.test.ts index 66e0032e..abeba4de 100644 --- a/packages/sdk/tests/cloud-run.test.ts +++ b/packages/sdk/tests/cloud-run.test.ts @@ -169,6 +169,9 @@ describe('thin cloud CLI', () => { ['check', '--cloud', 'flow.yaml'], ['run', '--cloud', '--no-spawn', 'flow.yaml'], ['run', '--cloud', '--cloud', 'flow.yaml'], ['run', '--cloud', '--input', '{}', 'flow.yaml'], ['run', '--cloud', '--data-dir', 'x', 'flow.yaml'], + // `--local-agent` describes a local wrapper process, so it says nothing + // about a run Cloud executes: refused rather than silently dropped. + ['run', '--cloud', '--local-agent', 'flow.yaml'], ])('refuses incompatible argv %j', async (...args) => { const fetch = vi.spyOn(globalThis, 'fetch'); expect(await runCli(args, { stdout: () => {}, stderr: () => {} })).toBe(2); From b69189e7c0b303ec5ddb2df95b84f43a47753fd9 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Wed, 9 Sep 2026 15:32:23 +0200 Subject: [PATCH 15/16] fix: route every agent stop through one process-group kill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only abort killed the process group. Timeout and protocol `terminate` still signalled the direct child, so a wrapper's grandchildren survived holding the stdio they inherited: the step Promise settled, but `'close'` never fired and the `flows run` event loop stayed referenced by those pipe handles. The defect was the duplicated decision, not the two missing guards. `spawn`'s `detached` flag and the kill strategy that depends on it now come from one place, `child-stop.ts`, bound once at each spawn site. Call sites choose only the force of a stop — `kill()` or `terminate()` — never its reach, so a fourth stop path cannot be added with the old behaviour by omission. `terminate()` also escalates to `SIGKILL` on an unref'd timer, which the raw CLI timeout path never did at all; `wrapper-session` drops its own copy of that escalation. tests/stop-process-group.test.ts asserts on process exit rather than on a settled Promise: a real node process drives the built SDK to a stop and must then die on its own, once after an execution-timeout stop and once after a protocol `terminate`. Abort stays covered by worker-cli-abort.test.ts. The raw CLI timeout call site is unreachable through `runAgentCli` today (`agentExecution` pins `timeoutMs: 0`), so its reach is asserted directly on the helper it now delegates to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013dsGogLaoDWpnStiUSCWdX Session-Id: 94be54e8-1714-490d-be52-69b10133c16d --- packages/sdk/src/child-stop.ts | 82 +++++++ packages/sdk/src/worker-cli.ts | 14 +- packages/sdk/src/wrapper-session.ts | 24 +- packages/sdk/tests/stop-process-group.test.ts | 214 ++++++++++++++++++ 4 files changed, 319 insertions(+), 15 deletions(-) create mode 100644 packages/sdk/src/child-stop.ts create mode 100644 packages/sdk/tests/stop-process-group.test.ts diff --git a/packages/sdk/src/child-stop.ts b/packages/sdk/src/child-stop.ts new file mode 100644 index 00000000..b0f2d5d5 --- /dev/null +++ b/packages/sdk/src/child-stop.ts @@ -0,0 +1,82 @@ +import type { ChildProcess } from 'node:child_process'; + +/** + * Grace between the graceful signal and the force kill. A tree that ignores + * `SIGTERM` still has to go: it is holding the inherited stdio pipes that keep + * the `flows run` event loop alive long after the step itself has settled. + */ +export const FORCE_KILL_DELAY_MS = 1_000; + +/** + * The one stop path for a spawned agent process. + * + * There are three stops — lease abort, execution timeout, and protocol + * `terminate` — and the decision they share is whether a signal must address + * the process GROUP or the direct child. That decision used to be made at each + * call site, which meant it was made once and omitted twice: only abort killed + * the group, so a timeout or a protocol violation left the grandchildren alive + * holding the pipes they inherited. The step Promise settled; `flows run` never + * exited, because those pipe handles stay open and referenced. + * + * So the decision lives here instead, bound once at the spawn site. A call site + * chooses only the FORCE of the stop, never its REACH. + */ +export interface ChildStop { + /** Stop the tree now, unconditionally. */ + kill(): void; + /** Ask the tree to stop, then force whatever is still alive. */ + terminate(): void; + /** Drop a pending force kill once the tree is known to be gone. */ + cancel(): void; +} + +/** + * Whether a stop can reach descendants at all. It can only when the spawn asked + * for a process group of its own, which needs POSIX and is only worth the + * detach on the lease-bound path. Callers pass this same value to `spawn`'s + * `detached` and to {@link childStop}, so the spawn flag and the stop strategy + * cannot drift apart. + */ +export function ownsProcessGroup(signal: AbortSignal | undefined): boolean { + return signal !== undefined && process.platform !== 'win32'; +} + +export function childStop( + child: ChildProcess, + ownsGroup: boolean, + forceKillDelayMs: number = FORCE_KILL_DELAY_MS, +): ChildStop { + let forceTimer: NodeJS.Timeout | undefined; + const cancel = (): void => { + if (forceTimer !== undefined) clearTimeout(forceTimer); + forceTimer = undefined; + }; + const signalTree = (name: NodeJS.Signals): void => { + if (ownsGroup && child.pid !== undefined) { + // `-pid` addresses the group this detached child leads, which is every + // descendant that has not left it. It throws only once the whole group + // is gone — the outcome we were asking for — so fall through and let the + // direct-child signal report on a child that never became a leader. + try { + process.kill(-child.pid, name); + return; + } catch { /* the group is gone, or we never led one */ } + } + child.kill(name); + }; + return { + kill: (): void => { + cancel(); + signalTree('SIGKILL'); + }, + terminate: (): void => { + cancel(); + signalTree('SIGTERM'); + forceTimer = setTimeout(() => signalTree('SIGKILL'), forceKillDelayMs); + // Never a reason on its own to hold the loop open: if no leaked pipe is + // keeping this process alive, the tree we would force is already gone. + forceTimer.unref(); + }, + cancel, + }; +} diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 083fcc99..dc928af1 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { childStop, ownsProcessGroup } from './child-stop.js'; import { agentExecution, cliAdapterKind, @@ -80,10 +81,12 @@ function spawnInvocation( signal?: AbortSignal, ): Promise { return new Promise((resolve) => { + const ownsGroup = ownsProcessGroup(signal); const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env, - detached: signal !== undefined && process.platform !== 'win32', + detached: ownsGroup, }); + const stop = childStop(child, ownsGroup); const stdout: Buffer[] = []; const stderr: Buffer[] = []; let settled = false; @@ -96,9 +99,7 @@ function spawnInvocation( resolve(result); }; const onAbort = (): void => { - if (child.pid !== undefined && process.platform !== 'win32') { - try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } - } else child.kill('SIGKILL'); + stop.kill(); finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' }); }; signal?.addEventListener('abort', onAbort, { once: true }); @@ -117,7 +118,10 @@ function spawnInvocation( })); if (invocation.timeoutMs > 0) { timer = setTimeout(() => { - child.kill('SIGTERM'); + // The stop outlives this settle on purpose: `finish` resolves the step, + // but only the forced group kill releases the pipes a leaked descendant + // is holding, and until they are released `flows run` cannot exit. + stop.terminate(); finish({ exit_code: null, stdout_tail: Buffer.concat(stdout).toString('utf8'), diff --git a/packages/sdk/src/wrapper-session.ts b/packages/sdk/src/wrapper-session.ts index bc0fe15c..a5c893da 100644 --- a/packages/sdk/src/wrapper-session.ts +++ b/packages/sdk/src/wrapper-session.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { FORCE_KILL_DELAY_MS, childStop, ownsProcessGroup } from './child-stop.js'; import { WRAPPER_EXECUTE_TOKEN, WRAPPER_IDENTIFY_ARG, @@ -28,7 +29,6 @@ const DEFAULT_LIMITS: WrapperSessionLimits = { maxOutputBytes: 1_048_576, }; const HANDSHAKE_OUTPUT_LIMIT = 8_192; -const FORCE_KILL_DELAY_MS = 1_000; /** * Grace after `SIGKILL` before the reader settles on its own. Node emits * `'close'` only once every inherited stdio pipe is closed, which any @@ -89,11 +89,13 @@ function executePinnedWrapper( signal?: AbortSignal, ): Promise { return new Promise((resolve) => { + const ownsGroup = ownsProcessGroup(signal); const child = spawn(identity.executable, [WRAPPER_IDENTIFY_ARG], { stdio: ['pipe', 'pipe', 'pipe'], env, - detached: signal !== undefined && process.platform !== 'win32', + detached: ownsGroup, }); + const stop = childStop(child, ownsGroup); const stdout: string[] = []; const stderr: Buffer[] = []; let handshakePending = ''; @@ -103,13 +105,16 @@ function executePinnedWrapper( let protocolError: string | undefined; let settled = false; let lifecycleTimer: NodeJS.Timeout | undefined; - let killTimer: NodeJS.Timeout | undefined; let settleTimer: NodeJS.Timeout | undefined; const clearTimers = (): void => { if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); - if (killTimer !== undefined) clearTimeout(killTimer); if (settleTimer !== undefined) clearTimeout(settleTimer); + // Only reachable once the child has actually closed, or once the forced + // kill has already fired: `terminate` settles on its own deadline, never + // on `'close'`. Dropping the escalation here therefore cannot spare a + // surviving descendant. + stop.cancel(); }; const finish = (result: WrapperSessionResult): void => { if (settled) return; @@ -119,9 +124,7 @@ function executePinnedWrapper( resolve(result); }; const onAbort = (): void => { - if (child.pid !== undefined && process.platform !== 'win32') { - try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } - } else child.kill('SIGKILL'); + stop.kill(); finish(failure('Agent execution aborted: lease ownership lost.')); }; signal?.addEventListener('abort', onAbort, { once: true }); @@ -130,9 +133,10 @@ function executePinnedWrapper( if (protocolError !== undefined) return; protocolError = message; if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); - child.kill('SIGTERM'); - killTimer = setTimeout(() => child.kill('SIGKILL'), FORCE_KILL_DELAY_MS); - killTimer.unref(); + // Same reach as an abort, only gentler first: this stop must find the + // whole group, or a descendant outlives the session still holding the + // stdio it inherited. + stop.terminate(); // The reader owns the bound. `'close'` is emitted only after every // inherited stdio pipe closes, so a wrapper that leaves a descendant // holding one withholds it forever and strands the step with no diff --git a/packages/sdk/tests/stop-process-group.test.ts b/packages/sdk/tests/stop-process-group.test.ts new file mode 100644 index 00000000..14b0b32e --- /dev/null +++ b/packages/sdk/tests/stop-process-group.test.ts @@ -0,0 +1,214 @@ +import { spawn } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { childStop } from '../src/child-stop.js'; + +/** + * A stop that settles the step Promise is not a stop that lets `flows run` + * exit. `'close'` waits on every inherited stdio pipe, and a grandchild that + * survived the stop holds one forever — the run's own event loop stays + * referenced by that pipe handle long after the step has been journaled. + * + * So these assert on PROCESS EXIT, not on a resolved Promise: a real node + * process drives the built SDK to a stop and then has to die on its own. The + * step-settles side is already covered in `worker-cli.test.ts`; what is proved + * here is the reach of the stop, across every path that has one. Abort is + * covered by `worker-cli-abort.test.ts`; the two wrapper stops are covered + * end-to-end below; the raw-CLI timeout call site, which `agentExecution` + * currently pins to `timeoutMs: 0` and so cannot be reached through + * `runAgentCli`, is covered at the shared helper it now delegates to. + */ +const SDK = join(dirname(fileURLToPath(import.meta.url)), '..'); +const BUILT_WORKER_CLI = join(SDK, 'dist', 'worker-cli.js'); +const WRAPPER_HELPER = resolve(SDK, '..', '..', 'testdata', 'preflight', 'wrapper-session.mjs'); +const directories: string[] = []; + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function makeDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'flows-stop-group-')); + directories.push(directory); + return directory; +} + +/** + * A conforming wrapper that leaves a grandchild behind holding the stdio it + * inherited, then does whatever `tail` asks for to trigger a stop. + */ +function writeLeakyWrapper(directory: string, name: string, tail: string): { + wrapper: string; + wrapperPid: string; + grandchildPid: string; +} { + const wrapper = join(directory, name); + const wrapperPid = join(directory, `${name}.wrapper-pid`); + const grandchildPid = join(directory, `${name}.grandchild-pid`); + const grandchildSource = `require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`; + writeFileSync(wrapper, `#!/usr/bin/env node +import { existsSync, writeFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { receiveWrapperRequest } from ${JSON.stringify(WRAPPER_HELPER)}; +await receiveWrapperRequest(); +writeFileSync(${JSON.stringify(wrapperPid)}, String(process.pid)); +spawn(process.execPath, ['-e', ${JSON.stringify(grandchildSource)}], { stdio: 'inherit' }); +// Block until the grandchild has announced itself, so a stop that arrives on +// the very next line still has a pid on disk to be judged against. A sync wait +// is the point: the wrapper's own loop must not advance past this. +const idle = new Int32Array(new SharedArrayBuffer(4)); +for (let waited = 0; waited < 5_000 && !existsSync(${JSON.stringify(grandchildPid)}); waited += 10) { + Atomics.wait(idle, 0, 0, 10); +} +${tail} +`); + chmodSync(wrapper, 0o755); + return { wrapper, wrapperPid, grandchildPid }; +} + +/** + * Drive one wrapper session to a stop inside a real node process, exactly the + * way `flows run` does, and report how long that process took to exit. Nothing + * calls `process.exit()`: the harness ends only when its own event loop drains. + */ +async function runUntilExit( + directory: string, + wrapper: string, + executionTimeoutMs: number, +): Promise<{ exitedWithinMs: number; code: number | null; stderrTail: string }> { + expect( + existsSync(BUILT_WORKER_CLI), + `${BUILT_WORKER_CLI} is missing; run \`npm run build\` (\`npm test\` does) before this test`, + ).toBe(true); + const harness = join(directory, 'harness.mjs'); + writeFileSync(harness, ` +import { runAgentCli } from ${JSON.stringify(BUILT_WORKER_CLI)}; +// A never-aborted signal is what a lease-bound run holds for its whole life; +// it is also what asks the spawn for a process group of its own. +const controller = new AbortController(); +const result = await runAgentCli( + ${JSON.stringify(wrapper)}, + 'instruction', + undefined, + undefined, + { handshakeTimeoutMs: 5_000, executionTimeoutMs: ${executionTimeoutMs}, maxOutputBytes: 100_000 }, + controller.signal, +); +process.stdout.write(JSON.stringify({ stderr_tail: result.stderr_tail }) + '\\n'); +`); + const started = Date.now(); + const child = spawn(process.execPath, [harness], { stdio: ['ignore', 'pipe', 'inherit'] }); + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + const code = await new Promise((resolveExit, rejectExit) => { + const bound = setTimeout(() => { + child.kill('SIGKILL'); + rejectExit(new Error('the run process never exited after the stop')); + }, 15_000); + child.once('error', rejectExit); + child.once('exit', exitCode => { clearTimeout(bound); resolveExit(exitCode); }); + }); + const settled: unknown = JSON.parse(stdout.trim() === '' ? '{}' : stdout.trim()); + return { + exitedWithinMs: Date.now() - started, + code, + stderrTail: String((settled as { stderr_tail?: unknown }).stderr_tail ?? ''), + }; +} + +async function expectReaped(pidFile: string): Promise { + expect(existsSync(pidFile)).toBe(true); + const pid = Number(readFileSync(pidFile, 'utf8')); + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + try { + process.kill(pid, 0); + } catch { + return; + } + await new Promise(wait => setTimeout(wait, 25)); + } + expect(() => process.kill(pid, 0)).toThrow(); +} + +describe('every stop reaches the process group, not just the direct child', () => { + it('exits the run after an execution-timeout stop', async () => { + const directory = makeDirectory(); + const leaky = writeLeakyWrapper(directory, 'timeout-wrapper.mjs', 'setInterval(() => {}, 1000);'); + + const run = await runUntilExit(directory, leaky.wrapper, 400); + + expect(run.stderrTail).toMatch(/execution timed out after 400ms/i); + expect(run.code).toBe(0); + expect(run.exitedWithinMs).toBeLessThan(10_000); + await expectReaped(leaky.wrapperPid); + await expectReaped(leaky.grandchildPid); + }, 40_000); + + it('exits the run after a protocol terminate stop', async () => { + const directory = makeDirectory(); + const leaky = writeLeakyWrapper( + directory, + 'protocol-wrapper.mjs', + // A second execute frame is a protocol violation, so the session + // terminates on the spot rather than on any clock. + `process.stdout.write('relayflows-agent-cli-v1-execute\\n');\nsetInterval(() => {}, 1000);`, + ); + + const run = await runUntilExit(directory, leaky.wrapper, 30_000); + + expect(run.stderrTail).toMatch(/duplicate execute protocol frame/i); + expect(run.code).toBe(0); + expect(run.exitedWithinMs).toBeLessThan(10_000); + await expectReaped(leaky.wrapperPid); + await expectReaped(leaky.grandchildPid); + }, 40_000); + + /** + * The raw-CLI timeout call site in `worker-cli.ts` cannot be reached through + * `runAgentCli` today — `agentExecution` pins agent invocations to + * `timeoutMs: 0` — so its reach is asserted on the helper it now delegates + * to, which is the same object the two wrapper stops above go through. + */ + it('terminate() forces a group that outlives SIGTERM', async () => { + const directory = makeDirectory(); + const grandchildPid = join(directory, 'grandchild-pid'); + const source = ` +const { spawn } = require('node:child_process'); +process.on('SIGTERM', () => {}); +spawn(process.execPath, ['-e', ${JSON.stringify(`process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`)}], { stdio: 'inherit' }); +setInterval(() => {}, 1000); +`; + const child = spawn(process.execPath, ['-e', source], { + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + const stop = childStop(child, true, 200); + try { + const deadline = Date.now() + 5_000; + while (!existsSync(grandchildPid) && Date.now() < deadline) { + await new Promise(wait => setTimeout(wait, 10)); + } + expect(existsSync(grandchildPid)).toBe(true); + + stop.terminate(); + await new Promise(resolveClose => child.once('close', () => { resolveClose(); })); + await expectReaped(grandchildPid); + } finally { + stop.kill(); + } + }, 30_000); +}); From 033685d230a503bd9124772ec899ea04c25e9d25 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Wed, 9 Sep 2026 16:17:11 +0200 Subject: [PATCH 16/16] fix: never let child-level evidence settle over a live process group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The invariant, stated once and then satisfied: a session may not settle until either the process group is confirmed dead or the escalation has actually run. `'close'` and `'error'` are evidence about the DIRECT CHILD and nothing more. A descendant that ignores SIGTERM and inherited none of the wrapper's stdio emits exactly those events while it is still running, so `finish` -> `clearTimers` -> `stop.cancel()` dropped the pending group SIGKILL on the strength of an event that proved nothing about the group. Unifying the three signal paths moved this race rather than closing it. So `cancel()` leaves the ChildStop interface entirely — no call site can now drop an escalation. In its place `maySettleOnChildExit()` asks the GROUP (`kill(-pgid, 0)`; only ESRCH counts as empty) and answers the one question a child-level event can raise. It refuses when an escalation is armed over a group that still answers, and the escalation's own deadline settles instead with a byte-identical result. Both settle sites go through it, not only the `'close'` one cursor named: - wrapper-session `'close'` and `'error'` - worker-cli `'close'` and `'error'` (which never cancelled, but now also refunds a pointless escalation instead of holding the loop for it) Enumerated and left alone: `onAbort` cancels inside `kill()`, which is the escalation, and `terminate`'s settle deadline runs after it. The escalation timer is also now referenced rather than unref'd. The survivor it exists for holds none of our stdio, so nothing of ours keeps the loop alive and the drain dropped the timer in precisely the case it was armed for. The pid is pinned at spawn for the same reason: every use of it happens after the child has been reaped. Tests: two wrapper stops (protocol terminate, execution timeout) against a SIGTERM-deaf grandchild that holds no stdio, plus a harness that calls terminate() and is then left alone to die, which is the only way to see the unref. Mutation-checked both ways. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SzSdtnqJqC1RyfbYJ5Q7Fh Session-Id: ab2a44a3-2846-442c-9ba2-b91ec18622ee --- packages/sdk/src/child-stop.ts | 76 +++++++++-- packages/sdk/src/worker-cli.ts | 16 ++- packages/sdk/src/wrapper-session.ts | 41 ++++-- packages/sdk/tests/stop-process-group.test.ts | 127 +++++++++++++++++- 4 files changed, 230 insertions(+), 30 deletions(-) diff --git a/packages/sdk/src/child-stop.ts b/packages/sdk/src/child-stop.ts index b0f2d5d5..71b642c7 100644 --- a/packages/sdk/src/child-stop.ts +++ b/packages/sdk/src/child-stop.ts @@ -19,15 +19,35 @@ export const FORCE_KILL_DELAY_MS = 1_000; * exited, because those pipe handles stay open and referenced. * * So the decision lives here instead, bound once at the spawn site. A call site - * chooses only the FORCE of the stop, never its REACH. + * chooses only the FORCE of the stop, never its REACH — and, per + * {@link ChildStop.maySettleOnChildExit}, never gets to decide on its own that + * a stop is finished. */ export interface ChildStop { /** Stop the tree now, unconditionally. */ kill(): void; /** Ask the tree to stop, then force whatever is still alive. */ terminate(): void; - /** Drop a pending force kill once the tree is known to be gone. */ - cancel(): void; + /** + * Answer the only question a child-level event can raise: the direct child is + * gone — may the session settle? + * + * INVARIANT: a session may not settle until either the process group is + * confirmed dead or the escalation has actually run. + * + * `'close'` and `'error'` are evidence about the CHILD, never about the + * group. A descendant that ignores `SIGTERM` and inherited none of the + * wrapper's stdio emits exactly those events while it is still running, so no + * call site may read one as a dead tree. This is therefore the only place a + * pending escalation may be dropped for any reason other than a forced kill, + * and it drops one only after asking the group whether it is empty. + * + * Returns true when settling is safe: no escalation is armed, or the group is + * confirmed gone and the now-pointless escalation has been dropped here. + * Returns false when an escalation is armed over a group that still answers — + * the caller must then leave the settle to that escalation's own deadline. + */ + maySettleOnChildExit(): boolean; } /** @@ -46,19 +66,43 @@ export function childStop( ownsGroup: boolean, forceKillDelayMs: number = FORCE_KILL_DELAY_MS, ): ChildStop { + // Pinned at the spawn rather than read per signal. Every interesting use of + // this id happens AFTER the direct child has been reaped — the escalation + // fires a second later, and the group probe runs on `'close'` — so reading + // `child.pid` there would be reading a field the runtime owns and is free to + // clear. The group keeps this id for as long as any member of it is alive, + // which is exactly the window both of those need to address. + const pid = child.pid; let forceTimer: NodeJS.Timeout | undefined; const cancel = (): void => { if (forceTimer !== undefined) clearTimeout(forceTimer); forceTimer = undefined; }; + /** Whether anything is still in the group. See `maySettleOnChildExit`. */ + const groupAnswers = (): boolean => { + // With no group of our own a stop never reached past the direct child, so + // that child's exit IS the whole of our reach and there is nothing left to + // ask about. + if (!ownsGroup || pid === undefined) return false; + try { + process.kill(-pid, 0); + return true; + } catch (error) { + // Only `ESRCH` proves the group is empty. `EPERM` proves the opposite — + // something is in there that we may not signal — and any other errno + // proves nothing at all, so both must read as alive: an unproven group is + // not a reason to spare a survivor. + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } + }; const signalTree = (name: NodeJS.Signals): void => { - if (ownsGroup && child.pid !== undefined) { + if (ownsGroup && pid !== undefined) { // `-pid` addresses the group this detached child leads, which is every // descendant that has not left it. It throws only once the whole group // is gone — the outcome we were asking for — so fall through and let the // direct-child signal report on a child that never became a leader. try { - process.kill(-child.pid, name); + process.kill(-pid, name); return; } catch { /* the group is gone, or we never led one */ } } @@ -72,11 +116,23 @@ export function childStop( terminate: (): void => { cancel(); signalTree('SIGTERM'); - forceTimer = setTimeout(() => signalTree('SIGKILL'), forceKillDelayMs); - // Never a reason on its own to hold the loop open: if no leaked pipe is - // keeping this process alive, the tree we would force is already gone. - forceTimer.unref(); + forceTimer = setTimeout(() => { + forceTimer = undefined; + signalTree('SIGKILL'); + }, forceKillDelayMs); + // Deliberately REFERENCED, unlike every other timer we arm. The survivor + // this escalation exists for is the one that ignored `SIGTERM` and holds + // none of our stdio: nothing it does keeps our loop alive, so an unref'd + // escalation would be dropped by the drain in precisely the case it was + // armed for. The cost is bounded by `forceKillDelayMs`, is paid only + // after a stop was actually issued, and is refunded the moment + // `maySettleOnChildExit` confirms the group is empty. + }, + maySettleOnChildExit: (): boolean => { + if (forceTimer === undefined) return true; + if (groupAnswers()) return false; + cancel(); + return true; }, - cancel, }; } diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index dc928af1..37967181 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -102,16 +102,28 @@ function spawnInvocation( stop.kill(); finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' }); }; + /** + * Same invariant as `wrapper-session.ts`: `'close'` and `'error'` are + * evidence about the DIRECT CHILD, so they may not settle over a pending + * escalation, and only `maySettleOnChildExit` may drop one. This settle + * carries no deadline of its own because it needs none — the timeout below + * settles on the spot and lets its escalation outlive that, so refusing + * here can only defer to a `'close'` we are still going to get. + */ + const finishOnChildExit = (result: WorkerCliResult): void => { + if (!stop.maySettleOnChildExit()) return; + finish(result); + }; signal?.addEventListener('abort', onAbort, { once: true }); if (signal?.aborted) onAbort(); child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); - child.once('error', (error) => finish({ + child.once('error', (error) => finishOnChildExit({ exit_code: null, stdout_tail: Buffer.concat(stdout).toString('utf8'), stderr_tail: error.message, })); - child.once('close', (code) => finish({ + child.once('close', (code) => finishOnChildExit({ exit_code: code, stdout_tail: Buffer.concat(stdout).toString('utf8'), stderr_tail: Buffer.concat(stderr).toString('utf8'), diff --git a/packages/sdk/src/wrapper-session.ts b/packages/sdk/src/wrapper-session.ts index a5c893da..bff6ad01 100644 --- a/packages/sdk/src/wrapper-session.ts +++ b/packages/sdk/src/wrapper-session.ts @@ -107,22 +107,37 @@ function executePinnedWrapper( let lifecycleTimer: NodeJS.Timeout | undefined; let settleTimer: NodeJS.Timeout | undefined; - const clearTimers = (): void => { - if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); - if (settleTimer !== undefined) clearTimeout(settleTimer); - // Only reachable once the child has actually closed, or once the forced - // kill has already fired: `terminate` settles on its own deadline, never - // on `'close'`. Dropping the escalation here therefore cannot spare a - // surviving descendant. - stop.cancel(); - }; const finish = (result: WrapperSessionResult): void => { if (settled) return; settled = true; - clearTimers(); + if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); + if (settleTimer !== undefined) clearTimeout(settleTimer); signal?.removeEventListener('abort', onAbort); resolve(result); }; + /** + * INVARIANT: a session may not settle until either the process group is + * confirmed dead or the escalation has actually run. + * + * `'close'` and `'error'` are evidence about the DIRECT CHILD and nothing + * more. A descendant that ignores `SIGTERM` and inherited none of the + * wrapper's stdio emits exactly those events while it is still running, so + * neither may drop a pending escalation and neither may settle ahead of + * one. Every child-level settle therefore goes through + * `maySettleOnChildExit`, which is the one place that asks the GROUP. + * + * When it says no, `terminate`'s own deadline settles instead, with a + * byte-identical `failure(protocolError)` result. That deadline is armed + * whenever an escalation is — both come from the single `terminate` below — + * so refusing here can defer a settle but can never strand one. + */ + const finishOnChildExit = (result: WrapperSessionResult): void => { + // Asked before `finish`, and asked even once we have already settled: + // this is also the only place a pointless escalation is refunded, and a + // session that settled on `terminate`'s deadline still owes that refund. + if (!stop.maySettleOnChildExit()) return; + finish(result); + }; const onAbort = (): void => { stop.kill(); finish(failure('Agent execution aborted: lease ownership lost.')); @@ -261,7 +276,7 @@ function executePinnedWrapper( child.stdin.on('error', () => { // A child that closes stdin before acknowledgement is classified on close. }); - child.once('error', (error) => finish(failure(error.message))); + child.once('error', (error) => finishOnChildExit(failure(error.message))); child.once('close', (code) => { if (protocolError === undefined && phase === 'execute' && executionPending.length > 0) { if (normalizeLine(executionPending) === WRAPPER_EXECUTE_TOKEN) { @@ -271,13 +286,13 @@ function executePinnedWrapper( } } if (protocolError !== undefined || phase !== 'execute') { - finish(failure( + finishOnChildExit(failure( protocolError ?? `CLI ${JSON.stringify(cli)} exited before completing the ${WRAPPER_IDENTIFY_TOKEN} same-process handshake.`, )); return; } - finish({ + finishOnChildExit({ exit_code: code, stdout_tail: stdout.join(''), stderr_tail: Buffer.concat(stderr).toString('utf8'), diff --git a/packages/sdk/tests/stop-process-group.test.ts b/packages/sdk/tests/stop-process-group.test.ts index 14b0b32e..d90b0a5e 100644 --- a/packages/sdk/tests/stop-process-group.test.ts +++ b/packages/sdk/tests/stop-process-group.test.ts @@ -30,6 +30,7 @@ import { childStop } from '../src/child-stop.js'; */ const SDK = join(dirname(fileURLToPath(import.meta.url)), '..'); const BUILT_WORKER_CLI = join(SDK, 'dist', 'worker-cli.js'); +const BUILT_CHILD_STOP = join(SDK, 'dist', 'child-stop.js'); const WRAPPER_HELPER = resolve(SDK, '..', '..', 'testdata', 'preflight', 'wrapper-session.mjs'); const directories: string[] = []; @@ -46,10 +47,31 @@ function makeDirectory(): string { } /** - * A conforming wrapper that leaves a grandchild behind holding the stdio it - * inherited, then does whatever `tail` asks for to trigger a stop. + * The two shapes a surviving grandchild can take, which are the two different + * ways a stop can be cut short: + * + * - `holds-stdio` keeps the pipes it inherited. `'close'` is emitted only once + * every one of them is closed, so this grandchild withholds the event + * forever and keeps `flows run`'s loop referenced by those handles. The + * settle deadline is what bounds that. + * - `deaf-to-sigterm` ignores `SIGTERM` and inherits none of our stdio. So + * `'close'` fires promptly on the DIRECT CHILD while the grandchild is still + * running — a child-level event that says nothing whatever about the group. + * Only the escalation reaches this one, and only if nothing cancelled it on + * the strength of that event. + */ +type Survivor = 'holds-stdio' | 'deaf-to-sigterm'; + +/** + * A conforming wrapper that leaves a grandchild of the given shape behind, then + * does whatever `tail` asks for to trigger a stop. */ -function writeLeakyWrapper(directory: string, name: string, tail: string): { +function writeLeakyWrapper( + directory: string, + name: string, + tail: string, + survivor: Survivor = 'holds-stdio', +): { wrapper: string; wrapperPid: string; grandchildPid: string; @@ -57,14 +79,15 @@ function writeLeakyWrapper(directory: string, name: string, tail: string): { const wrapper = join(directory, name); const wrapperPid = join(directory, `${name}.wrapper-pid`); const grandchildPid = join(directory, `${name}.grandchild-pid`); - const grandchildSource = `require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`; + const deaf = survivor === 'deaf-to-sigterm'; + const grandchildSource = `${deaf ? `process.on('SIGTERM', () => {}); ` : ''}require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`; writeFileSync(wrapper, `#!/usr/bin/env node import { existsSync, writeFileSync } from 'node:fs'; import { spawn } from 'node:child_process'; import { receiveWrapperRequest } from ${JSON.stringify(WRAPPER_HELPER)}; await receiveWrapperRequest(); writeFileSync(${JSON.stringify(wrapperPid)}, String(process.pid)); -spawn(process.execPath, ['-e', ${JSON.stringify(grandchildSource)}], { stdio: 'inherit' }); +spawn(process.execPath, ['-e', ${JSON.stringify(grandchildSource)}], { stdio: ${JSON.stringify(deaf ? 'ignore' : 'inherit')} }); // Block until the grandchild has announced itself, so a stop that arrives on // the very next line still has a pid on disk to be judged against. A sync wait // is the point: the wrapper's own loop must not advance past this. @@ -177,6 +200,100 @@ describe('every stop reaches the process group, not just the direct child', () = await expectReaped(leaky.grandchildPid); }, 40_000); + /** + * The same two stops again, against the survivor that the unified group kill + * did NOT cover: one that ignores `SIGTERM` and holds none of our stdio, so + * `'close'` fires on the direct child while it is still alive. Every earlier + * defect of this family was a stop path settling over a live descendant; this + * is that path reading a child-level event as proof of a dead group. Nothing + * but the escalation reaches this grandchild, so if a settle is allowed to + * cancel the escalation, it survives the run. + */ + it('kills a SIGTERM-deaf grandchild after a protocol terminate stop', async () => { + const directory = makeDirectory(); + const leaky = writeLeakyWrapper( + directory, + 'deaf-protocol-wrapper.mjs', + `process.stdout.write('relayflows-agent-cli-v1-execute\\n');\nsetInterval(() => {}, 1000);`, + 'deaf-to-sigterm', + ); + + const run = await runUntilExit(directory, leaky.wrapper, 30_000); + + expect(run.stderrTail).toMatch(/duplicate execute protocol frame/i); + expect(run.code).toBe(0); + expect(run.exitedWithinMs).toBeLessThan(10_000); + await expectReaped(leaky.wrapperPid); + await expectReaped(leaky.grandchildPid); + }, 40_000); + + it('kills a SIGTERM-deaf grandchild after an execution-timeout stop', async () => { + const directory = makeDirectory(); + const leaky = writeLeakyWrapper( + directory, + 'deaf-timeout-wrapper.mjs', + 'setInterval(() => {}, 1000);', + 'deaf-to-sigterm', + ); + + const run = await runUntilExit(directory, leaky.wrapper, 400); + + expect(run.stderrTail).toMatch(/execution timed out after 400ms/i); + expect(run.code).toBe(0); + expect(run.exitedWithinMs).toBeLessThan(10_000); + await expectReaped(leaky.wrapperPid); + await expectReaped(leaky.grandchildPid); + }, 40_000); + + /** + * The other half of the invariant: not just that nothing CANCELS the + * escalation, but that the escalation actually RUNS. A survivor that ignores + * `SIGTERM` and holds none of our stdio leaves nothing of ours referencing + * the loop, so an unref'd escalation would be dropped by the drain in exactly + * the case it exists for. Nothing here settles a Promise or arms a deadline — + * the harness calls `terminate()` and is then left alone to die, and the only + * thing that can hold it open long enough to force the group is the + * escalation's own handle. + */ + it('holds the loop open long enough for the escalation to run', async () => { + const directory = makeDirectory(); + const grandchildPid = join(directory, 'unheld-grandchild-pid'); + const grandchildSource = `process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(${JSON.stringify(grandchildPid)}, String(process.pid)); setInterval(() => {}, 1000);`; + const childSource = `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(grandchildSource)}], { stdio: 'ignore' }); setInterval(() => {}, 1000);`; + const harness = join(directory, 'escalation-harness.mjs'); + writeFileSync(harness, ` +import { existsSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { childStop } from ${JSON.stringify(BUILT_CHILD_STOP)}; +// No stdio of ours for anything in the tree to hold, so the child process +// handle is the only thing referencing this loop, and it goes on SIGTERM. +const child = spawn(process.execPath, ['-e', ${JSON.stringify(childSource)}], { + stdio: 'ignore', detached: true, +}); +for (let waited = 0; waited < 5_000 && !existsSync(${JSON.stringify(grandchildPid)}); waited += 10) { + await new Promise(wait => setTimeout(wait, 10)); +} +childStop(child, true).terminate(); +`); + const started = Date.now(); + const code = await new Promise((resolveExit, rejectExit) => { + const process_ = spawn(globalThis.process.execPath, [harness], { stdio: 'inherit' }); + const bound = setTimeout(() => { + process_.kill('SIGKILL'); + rejectExit(new Error('the harness never exited after terminate()')); + }, 15_000); + process_.once('error', rejectExit); + process_.once('exit', exitCode => { clearTimeout(bound); resolveExit(exitCode); }); + }); + + expect(code).toBe(0); + // It has to have waited for the escalation, and it has to have stopped + // waiting once that fired: a bound on both sides, not just the reap. + expect(Date.now() - started).toBeGreaterThanOrEqual(1_000); + expect(Date.now() - started).toBeLessThan(10_000); + await expectReaped(grandchildPid); + }, 40_000); + /** * The raw-CLI timeout call site in `worker-cli.ts` cannot be reached through * `runAgentCli` today — `agentExecution` pins agent invocations to