diff --git a/ops/DRIVE-LOCAL.md b/ops/DRIVE-LOCAL.md index 48ff2967c..8576a11ee 100644 --- a/ops/DRIVE-LOCAL.md +++ b/ops/DRIVE-LOCAL.md @@ -13,7 +13,7 @@ submits commands through the existing local launcher. The daemon journals those pins before implementation starts. Running the YAML template directly refuses the snapshot step because its pinned inputs are missing. -`gate-snapshot` runs first. It extracts the package helper, verifier and SDK +`gate-snapshot` runs first. It extracts the package helper, verifier, acceptance helper and SDK picker source with `git --no-replace-objects show :`, then compiles that picker in a temporary directory outside the checkout. Its own extraction script also comes from that Git ref. Neither working-tree helper files nor @@ -38,7 +38,44 @@ executed at the repository root with a two-minute bound. For example: Multiple `Verify:` lines mean all commands must pass. Commands come from the backlog before implementation; the agent cannot substitute its own acceptance -checks in package.json. Entries with no executable checks are skipped with +checks in package.json. Existing inline Node assertions (`node -e`, with an +optional `--input-type=module`) remain supported: their code is part of the +pinned backlog. Script checks must declare their code inputs explicitly: + +```text +- **Fix the value** Update `src/value.txt` to contain exactly "fixed". + Verify: {"argv":["node","checks/value.cjs"],"inputs":[{"path":"checks/value.cjs","ref":"HEAD"}]} +``` + +`HEAD` is resolved once to the launcher's full commit ID. An explicit full +commit ID is also accepted; branch names and missing Git objects are refused. +Git inputs must name regular files at repository-relative paths. Verification +extracts their Git bytes into a fresh temporary directory outside the checkout, +preserving relative paths, and replaces matching argv elements with extracted +paths. Declare assertion dependencies in the same `inputs` array so they are +extracted together. The command still runs with the implementation checkout as +its working directory, so assertions can read changed source and artifacts. + +Alternatively, use an absolute path without `ref` for an externally owned +script, for example `{"argv":["node","/opt/acceptance/value.cjs"],"inputs":[{"path":"/opt/acceptance/value.cjs"}]}`. +Such files must exist outside the implementation's declared write scope. +Validation checks both real paths and symlink aliases beneath writable +directories; hard-linked external inputs are refused because their ownership +cannot be established from a path. A relative path without a pin, an undeclared script, an unavailable +pin, or an external file inside write scope fails with +`ACCEPTANCE_IMMUTABILITY_VIOLATION` before any check executes. Node script checks +use the launcher's Node binary; other entry points must be declared scripts +with a shebang (for example `checks/value.sh`). Declaring `/bin/sh` does not +authorize an arbitrary `-c` command. Ambient `NODE_OPTIONS` and `NODE_PATH` +cannot preload checkout code. + +The selected package retains `verificationCommands` and adds +`verification: {ref, checks: [{argv, inputs}]}`. Every scope, verification and +report operation reconstructs both fields from the original pinned backlog. +The initial scope step therefore refuses an invalid contract before handing +the package to implementation. + +Entries with no executable checks are skipped with `missing_executable_checks`, alongside the existing unbounded/stale scope reasons. The old F8b entry now carries a source assertion for its declared rename. That assertion also allows an already-completed package to pass @@ -60,7 +97,8 @@ Verification reconstructs the selected work from the unchanged backlog and compares its scope and commands to package.json. Each package check must pass before the SDK regression suite runs. An unchanged implementation whose DoD is unmet fails. HEAD/branch changes fail, and out-of-scope changes produced by -an acceptance command fail too. A failed scope or verification step prevents +an acceptance command fail too. `PACKAGE_VERIFIED` is emitted only after that +post-check validation succeeds. Refusals include the package's DoD. A failed scope or verification step prevents the dependent report step from running. The scope command runs again after the SDK build and suite, before reporting. @@ -74,14 +112,17 @@ The execution contract has one owner for each kind of data: | HEAD, branch, backlog hash | Preparing launcher pins the original commit/branch and the hash of its committed backlog in submitted commands. The package records the commit as `head`; no adjacent ref or checksum file is authority. | | Package metadata | Private atomic JSON artifact for the agent; scope, verify and report reconstruct its fields from the picker built from the pinned ref and the pinned backlog. It cannot redefine the original HEAD. | | Allowed paths and protected paths | `local-work-verification.mjs` extracted from the pinned ref; index and working tree checked separately against the original HEAD, including non-ignored untracked files. | -| Acceptance argv | Parsed from the pinned backlog; every command must succeed, with scope rechecked after execution. | -| Acceptance scripts and dependencies | **Open blocker:** arbitrary argv can load mutable source from implementation scope. | +| Acceptance argv | Parsed from the pinned backlog and reconstructed as the package's `verification` contract; every command must succeed, with scope rechecked before emitting `PACKAGE_VERIFIED`. | +| Acceptance scripts and declared dependencies | Explicit `inputs` load regular files from full Git commit IDs or absolute external paths validated outside implementation write scope. Undeclared script entry points are refused. | **A same-user agent can still write to the temporary execution directory.** This meets the narrower bar that gate inputs come from a pinned Git ref rather than files implementation edits. It does not make extracted runtime files immutable, isolate processes, or prevent arbitrary Git-storage tampering. -The remaining acceptance-input decision is documented in -`runtime-evidence/drive-threads-0909-decisions.md`. This flow is not ready for -unattended use until those inputs have an explicit enforced ownership contract. +Acceptance authors still own the assertion program and its dependency +declarations. This is an input ownership contract, not analysis or isolation of +arbitrary programs: trusted checks must not delegate assertions to undeclared +mutable code via inline evaluation, subprocesses, or dynamically computed paths. +The historical acceptance-input decision is documented in +`runtime-evidence/drive-threads-0909-decisions.md`. diff --git a/ops/drive-local-flow.test.mjs b/ops/drive-local-flow.test.mjs index c39ed7fe5..c220e0c79 100644 --- a/ops/drive-local-flow.test.mjs +++ b/ops/drive-local-flow.test.mjs @@ -3,12 +3,21 @@ import { spawnSync } from 'node:child_process'; import { chmodSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import test from 'node:test'; -import { fixture, packagePath } from './local-work-test-fixture.mjs'; +import { socketPathFor } from '../packages/sdk/dist/daemon-connection.js'; +import { entry, fixture, packagePath } from './local-work-test-fixture.mjs'; const quote = text => "'" + text.replaceAll("'", "'\\''") + "'"; -for (const scenario of ['outside edit', 'verifier edit', 'unchanged package', 'HEAD repin', 'forged snapshot']) { +for (const scenario of ['outside edit', 'verifier edit', 'unchanged package', 'HEAD repin', 'forged snapshot', + 'edited pinned acceptance', 'undeclared acceptance']) { test(`drive-local journals failure and blocks reporting for ${scenario}`, t => { - const f = fixture(t); + const acceptance = scenario.includes('acceptance'); + const argv = ['node', 'src/check.cjs']; + const check = scenario === 'undeclared acceptance' ? argv : { + argv, inputs: [{ path: 'src/check.cjs', ref: 'HEAD' }], + }; + const f = acceptance ? fixture(t, entry('Fix value', 'src/', [check]), { + 'src/check.cjs': "require('node:assert/strict').equal(require('node:fs').readFileSync('src/value.txt','utf8'),'fixed');", + }) : fixture(t); const wrapper = resolve('testdata/preflight/wrapper-session.mjs'); const cli = join(f.root, '.relayflow/agent.mjs'); f.put('.relayflow/agent.mjs', `#!/usr/bin/env node @@ -19,6 +28,7 @@ const request = await receiveWrapperRequest(); if (request) { ${scenario === 'outside edit' ? `writeFileSync(${JSON.stringify(join(f.root, 'outside.txt'))}, 'changed');` : ''} ${scenario === 'verifier edit' ? `writeFileSync(${JSON.stringify(join(f.root, 'ops/local-work-package.mjs'))}, 'process.exit(0)');` : ''} + ${scenario === 'edited pinned acceptance' ? `writeFileSync(${JSON.stringify(join(f.root, 'src/check.cjs'))}, 'process.exit(0)');` : ''} ${scenario === 'HEAD repin' ? ` const git = (...args) => execFileSync('git', args, {cwd: ${JSON.stringify(f.root)}, encoding: 'utf8'}).trim(); writeFileSync(${JSON.stringify(join(f.root, 'outside.txt'))}, 'changed'); @@ -56,12 +66,16 @@ if (request) { assert(dataDir, result.stderr); t.after(() => rmSync(dataDir, { recursive: true, force: true })); const journal = readFileSync(join(dataDir, 'journal.jsonl'), 'utf8').trim().split('\n').map(JSON.parse); - const failedStep = ['unchanged package', 'forged snapshot'].includes(scenario) ? 'verify' : 'scope'; + const failedStep = scenario === 'undeclared acceptance' ? 'initial-scope' : + ['unchanged package', 'forged snapshot', 'edited pinned acceptance'].includes(scenario) ? 'verify' : 'scope'; const completion = journal.find(e => e.entry_type === 'step.completed' && e.step_id === failedStep); assert.equal(completion?.payload.verification.verdict, 'fail', JSON.stringify(journal)); assert.equal(completion.payload.completionReason, 'retries_exhausted'); assert.equal(completion.payload.verification.detail, 'exit code was 1'); assert(!journal.some(e => e.entry_type === 'step.attempt.started' && e.step_id === 'report')); - assert(!existsSync(join(dataDir, 'relayflowd.sock'))); + if (scenario === 'undeclared acceptance') { + assert(!journal.some(e => e.entry_type === 'step.attempt.started' && e.step_id === 'implement')); + } + assert(!existsSync(socketPathFor(dataDir))); }); } diff --git a/ops/local-work-acceptance.mjs b/ops/local-work-acceptance.mjs new file mode 100644 index 000000000..567a516a9 --- /dev/null +++ b/ops/local-work-acceptance.mjs @@ -0,0 +1,154 @@ +// Acceptance code has a separate ownership contract from implementation data. +// Git inputs are extracted afresh; external inputs must be outside write scope. +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, isAbsolute, join, resolve, sep } from 'node:path'; +import { tmpdir } from 'node:os'; + +const violation = detail => `ACCEPTANCE_IMMUTABILITY_VIOLATION: ${detail}`; +const within = (path, root) => path === root || path.startsWith(`${root}${sep}`); +const git = (...args) => execFileSync('git', ['--no-replace-objects', ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + +export function acceptanceContract(body, ref) { + const checks = body.split('\n').filter(line => /^\s*Verify:/.test(line)).map(line => { + let declaration; + try { declaration = JSON.parse(line.replace(/^\s*Verify:\s*/, '')); } + catch { throw new Error('INVALID_EXECUTABLE_CHECK: Verify must contain JSON argv or {argv, inputs}'); } + assert(Array.isArray(declaration) || (declaration && typeof declaration === 'object' && + Object.keys(declaration).every(key => key === 'argv' || key === 'inputs')), + 'INVALID_EXECUTABLE_CHECK: expected argv or {argv, inputs}'); + const { argv, inputs = [] } = Array.isArray(declaration) ? { argv: declaration } : declaration ?? {}; + assert(Array.isArray(argv) && argv.length > 0 && + argv.every(arg => typeof arg === 'string' && !arg.includes('\0')) && argv[0].trim(), + 'INVALID_EXECUTABLE_CHECK: expected nonempty command argv'); + assert(Array.isArray(inputs), violation('inputs must be an array')); + return { argv, inputs: inputs.map(input => { + assert(input && typeof input.path === 'string' && input.path && !input.path.includes('\0'), + violation('each input needs a path')); + assert(Object.keys(input).every(key => key === 'path' || key === 'ref'), violation('unknown input field')); + return { ...input, ...(input.ref === 'HEAD' ? { ref } : {}) }; + }) }; + }); + return { ref, checks }; +} + +// Include symlink aliases beneath a directory scope: src/link -> checks grants +// implementation a write route to checks even if checks is lexically outside src. +function writablePaths(scopes) { + const paths = []; + const visited = new Set(); + function visit(path) { + paths.push(resolve(path)); + let target; + try { target = realpathSync(path); } + catch (error) { + if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return; + throw error; + } + paths.push(target); + if (visited.has(target)) return; + visited.add(target); + if (lstatSync(target).isDirectory()) { + for (const entry of readdirSync(target)) visit(join(target, entry)); + } + } + for (const scope of scopes) visit(scope); + return paths; +} + +function validateInput(input, writable) { + if (input.ref !== undefined) { + assert(/^[a-f0-9]{40}$/.test(input.ref), violation(`missing pinned Git ref for ${input.path}`)); + assert(!isAbsolute(input.path) && input.path.split('/').every(part => part && part !== '.' && part !== '..'), + violation(`Git input must be a repository-relative path: ${input.path}`)); + try { + assert.equal(git('cat-file', '-t', input.ref).toString().trim(), 'commit'); + // Do not materialize a symlink blob as executable source. + const mode = git('ls-tree', input.ref, '--', input.path).toString().split(' ')[0]; + assert(mode === '100644' || mode === '100755'); + return git('show', `${input.ref}:${input.path}`); + } catch (cause) { + throw new Error(violation(`pinned Git input unavailable: ${input.ref}:${input.path}`), { cause }); + } + } + assert(isAbsolute(input.path), violation(`missing pinned Git ref for ${input.path}; external inputs require absolute paths`)); + let target; + try { target = realpathSync(input.path); } + catch (cause) { throw new Error(violation(`external input unavailable: ${input.path}`), { cause }); } + const stat = lstatSync(target); + assert(stat.isFile(), violation(`external input must be a file: ${input.path}`)); + assert(stat.nlink === 1, violation(`external input has writable hard-link aliases: ${input.path}`)); + assert(!writable.some(scope => within(resolve(input.path), scope) || within(target, scope)), + violation(`acceptance input is implementation-writable: ${input.path}`)); + return null; +} + +export function validateAcceptance(pkg) { + const contract = pkg.verification; + assert(contract && /^[a-f0-9]{40}$/.test(contract.ref) && contract.ref === pkg.head, + violation('missing or changed verification pinned Git ref')); + assert(contract.checks?.length > 0, 'MISSING_EXECUTABLE_CHECKS'); + assert.deepEqual(contract.checks.map(check => check.argv), pkg.verificationCommands, + violation('argv does not match the verification contract')); + const writable = writablePaths(pkg.filesInScope); + return contract.checks.map(check => { + const paths = new Set(); + const sources = check.inputs.map(input => { + assert(!paths.has(input.path), violation(`duplicate acceptance input: ${input.path}`)); + paths.add(input.path); + return validateInput(input, writable); + }); + const argv = check.argv; + // Legacy inline Node assertions are themselves code pinned in the backlog. + // Runtime options that could preload checkout code are deliberately excluded. + const node = argv[0] === 'node' || argv[0] === process.execPath; + const inline = node && ((argv[1] === '-e' && argv.length === 3) || + (argv[1] === '--input-type=module' && argv[2] === '-e' && argv.length === 4)); + const script = node ? argv[1] : argv[0]; + if (!inline) { + assert(script && !script.startsWith('-'), violation(`unsupported acceptance invocation: ${JSON.stringify(argv)}`)); + const input = check.inputs.find(input => input.path === script); + if (!input) { + assert(!writable.some(scope => within(resolve(script), scope)), + violation(`acceptance script is implementation-writable: ${script}`)); + throw new Error(violation(`undeclared acceptance script: ${script}; declare a pinned Git input or external absolute path`)); + } + if (!node) { + const source = sources[check.inputs.indexOf(input)] ?? readFileSync(input.path); + assert(source.subarray(0, 2).toString() === '#!', + violation(`declare an acceptance script with a shebang, not a runtime executable: ${script}`)); + } + } + return { ...check, sources, node }; + }); +} + +export function runAcceptance(pkg) { + // Validate every input before executing any command, including later checks. + const checks = validateAcceptance(pkg); + for (const check of checks) { + const directory = mkdtempSync(join(tmpdir(), 'drive-acceptance-')); + try { + const replacements = new Map(); + check.inputs.forEach((input, index) => { + if (check.sources[index] === null) return; + const path = join(directory, input.path); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, check.sources[index], { flag: 'wx', mode: 0o600 }); + if (check.argv[0] === input.path) chmodSync(path, 0o700); + replacements.set(input.path, path); + }); + const argv = check.argv.map(arg => replacements.get(arg) ?? arg); + if (check.node) argv[0] = process.execPath; + console.log(`CHECK ${JSON.stringify(check.argv)}`); + // Do not let ambient Node preload or search-path settings add mutable code. + const env = { ...process.env }; + delete env.NODE_OPTIONS; + delete env.NODE_PATH; + const result = spawnSync(argv[0], argv.slice(1), { stdio: 'inherit', timeout: 120000, env }); + assert(!result.error && result.status === 0, + `PACKAGE_CHECK_FAILED: ${JSON.stringify(check.argv)} (${result.error?.message ?? result.signal ?? result.status}); DoD: ${pkg.definitionOfDone.join('; ')}`); + } finally { rmSync(directory, { recursive: true, force: true }); } + } +} diff --git a/ops/local-work-acceptance.test.mjs b/ops/local-work-acceptance.test.mjs new file mode 100644 index 000000000..8f3b49aa9 --- /dev/null +++ b/ops/local-work-acceptance.test.mjs @@ -0,0 +1,173 @@ +import assert from 'node:assert/strict'; +import { linkSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import test from 'node:test'; +import { entry, fixture, packagePath, pass, fail } from './local-work-test-fixture.mjs'; + +const assertion = "require('node:assert/strict').equal(require('node:fs').readFileSync('src/value.txt','utf8'),'fixed');\n"; +const commit = f => { + f.git('add', '.'); + f.git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.test', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'acceptance input'); +}; +const pinned = (ref = 'HEAD') => ({ argv: ['node', 'src/check.cjs'], inputs: [{ path: 'src/check.cjs', ref }] }); + +test('edited in-scope acceptance argv refuses while implementation stays broken (#284)', t => { + const f = fixture(t, entry('Fix value', 'src/', [['node', 'src/check.cjs']])); + f.put('src/check.cjs', assertion); + commit(f); + pass(f.run('select')); + f.put('src/check.cjs', 'process.exit(0);\n'); + const result = f.run('verify'); + fail(result, /ACCEPTANCE_IMMUTABILITY_VIOLATION: acceptance script is implementation-writable: src\/check.cjs/); + assert.match(result.stderr, /DoD:/); + assert.doesNotMatch(result.stdout, /PACKAGE_VERIFIED|CHECK /); + assert.equal(readFileSync(join(f.root, 'src/value.txt'), 'utf8'), 'broken'); +}); + +test('pinned acceptance executes Git bytes despite an edited checkout script', t => { + const f = fixture(t, entry('Fix value', 'src/', [pinned()])); + f.put('src/check.cjs', assertion); + commit(f); + pass(f.run('select')); + const pkg = JSON.parse(readFileSync(join(f.root, packagePath))); + assert.equal(pkg.verification.ref, pkg.head); + assert.equal(pkg.verification.checks[0].inputs[0].ref, pkg.head); + f.put('src/check.cjs', 'process.exit(0);\n'); + fail(f.run('verify'), /PACKAGE_CHECK_FAILED.*DoD:/); + f.put('src/value.txt', 'fixed'); + pass(f.run('verify')); +}); + +for (const ref of [undefined, '', 'main', 'f'.repeat(40)]) { + test(`missing or unavailable script pin refuses: ${String(ref)}`, t => { + const check = pinned(); + check.inputs[0].ref = ref; + const f = fixture(t, entry('Fix value', 'src/', [check])); + f.put('src/check.cjs', 'process.exit(0);'); + commit(f); + pass(f.run('select')); + fail(f.run('verify'), /ACCEPTANCE_IMMUTABILITY_VIOLATION:.*(pinned Git ref|pinned Git input unavailable)/); + }); +} + +test('all inputs are validated before any acceptance command executes', t => { + const f = fixture(t, entry('Fix value', 'src/', [ + ['node', '-e', "require('node:fs').writeFileSync('src/ran.txt','ran')"], pinned(), + ])); + pass(f.run('select')); + const result = f.run('verify'); + fail(result, /ACCEPTANCE_IMMUTABILITY_VIOLATION/); + assert.doesNotMatch(result.stdout, /CHECK /); +}); + +test('external absolute acceptance reads the changed implementation', t => { + const directory = mkdtempSync(join(tmpdir(), 'external-acceptance-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const path = join(directory, 'check.cjs'); + writeFileSync(path, assertion); + const f = fixture(t, entry('Fix value', 'src/value.txt', [{ argv: ['node', path], inputs: [{ path }] }])); + pass(f.run('select')); + fail(f.run('verify'), /PACKAGE_CHECK_FAILED/); + f.put('src/value.txt', 'fixed'); + pass(f.run('verify')); +}); + +for (const alias of [false, true]) { + test(`an absolute acceptance path inside write scope refuses${alias ? ' through a symlink' : ''}`, t => { + const f = fixture(t); + f.put('src/check.cjs', 'process.exit(0);'); + const path = join(f.root, alias ? 'alias.cjs' : 'src/check.cjs'); + if (alias) symlinkSync('src/check.cjs', path); + f.put('ops/BACKLOG.md', entry('Fix value', 'src/', [{ argv: ['node', path], inputs: [{ path }] }])); + commit(f); + pass(f.run('select')); + fail(f.run('verify'), /ACCEPTANCE_IMMUTABILITY_VIOLATION: acceptance input is implementation-writable/); + }); +} + +test('a symlink beneath writable scope cannot grant access to an external acceptance file', t => { + const f = fixture(t); + f.put('checks/check.cjs', 'process.exit(0);'); + symlinkSync('../checks', join(f.root, 'src/checks')); + const path = join(f.root, 'checks/check.cjs'); + f.put('ops/BACKLOG.md', entry('Fix value', 'src/', [{ argv: ['node', path], inputs: [{ path }] }])); + commit(f); + pass(f.run('select')); + fail(f.run('verify'), /ACCEPTANCE_IMMUTABILITY_VIOLATION: acceptance input is implementation-writable/); +}); + +test('pinned relative dependencies also come from Git', t => { + const check = pinned(); + check.inputs.push({ path: 'src/assertion.cjs', ref: 'HEAD' }); + const f = fixture(t, entry('Fix value', 'src/', [check])); + f.put('src/check.cjs', "require('./assertion.cjs');\n"); + f.put('src/assertion.cjs', assertion); + commit(f); + pass(f.run('select')); + f.put('src/assertion.cjs', 'process.exit(0);'); + fail(f.run('verify'), /PACKAGE_CHECK_FAILED/); + f.put('src/value.txt', 'fixed'); + pass(f.run('verify')); +}); + +test('an external input with an in-scope hard link is refused', t => { + const f = fixture(t); + f.put('checks/check.cjs', 'process.exit(0);'); + const path = join(f.root, 'checks/check.cjs'); + linkSync(path, join(f.root, 'src/check.cjs')); + f.put('ops/BACKLOG.md', entry('Fix value', 'src/', [{ argv: ['node', path], inputs: [{ path }] }])); + commit(f); + pass(f.run('select')); + fail(f.run('verify'), /ACCEPTANCE_IMMUTABILITY_VIOLATION: external input has writable hard-link aliases/); +}); + +test('package metadata cannot delete or repin the verification block', t => { + const f = fixture(t); + pass(f.run('select')); + const pkg = JSON.parse(readFileSync(join(f.root, packagePath))); + for (const verification of [undefined, { ...pkg.verification, ref: 'f'.repeat(40) }]) { + f.put(packagePath, JSON.stringify({ ...pkg, verification })); + fail(f.run('verify'), /PACKAGE_CHANGED: verification/); + } +}); + +test('declaring a shell runtime cannot authorize arbitrary mutable acceptance argv', t => { + const f = fixture(t, entry('Fix value', 'src/', [{ + argv: ['/bin/sh', '-c', 'node src/check.cjs'], inputs: [{ path: '/bin/sh' }], + }])); + f.put('src/check.cjs', 'process.exit(0);'); + commit(f); + pass(f.run('select')); + fail(f.run('verify'), /ACCEPTANCE_IMMUTABILITY_VIOLATION/); +}); + +test('an executable acceptance script is extracted from Git', t => { + const f = fixture(t, entry('Fix value', 'src/', [{ + argv: ['checks/value.sh'], inputs: [{ path: 'checks/value.sh', ref: 'HEAD' }], + }]), { 'checks/value.sh': '#!/bin/sh\n[ "$(cat src/value.txt)" = fixed ]\n' }); + pass(f.run('select')); + fail(f.run('verify'), /PACKAGE_CHECK_FAILED/); + f.put('src/value.txt', 'fixed'); + pass(f.run('verify')); +}); + +test('isolated report refuses an outside edit and names the DoD', t => { + const f = fixture(t); + pass(f.run('select')); + f.put('outside.txt', 'changed'); + const result = f.run('report'); + fail(result, /OUT_OF_SCOPE: outside.txt; DoD:/); + assert.doesNotMatch(result.stdout, /REPORT |PACKAGE_VERIFIED/); +}); + +test('a check that violates scope never emits PACKAGE_VERIFIED', t => { + const f = fixture(t, entry('Fix value', 'src/value.txt', [ + ['node', '-e', "require('node:fs').writeFileSync('outside.txt','changed')"], + ])); + pass(f.run('select')); + const result = f.run('verify'); + fail(result, /OUT_OF_SCOPE/); + assert.doesNotMatch(result.stdout, /PACKAGE_VERIFIED/); +}); diff --git a/ops/local-work-package.mjs b/ops/local-work-package.mjs index ecc3eeef1..75eb62b78 100644 --- a/ops/local-work-package.mjs +++ b/ops/local-work-package.mjs @@ -23,7 +23,8 @@ import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync, } from 'node:fs'; import { dirname } from 'node:path'; -import { checkScope, runChecks, verificationCommands } from './local-work-verification.mjs'; +import { checkScope, runChecks } from './local-work-verification.mjs'; +import { acceptanceContract, validateAcceptance } from './local-work-acceptance.mjs'; const packagePath = '.relayflow/drive-local/package.json'; const backlogPath = 'ops/BACKLOG.md'; @@ -82,7 +83,7 @@ async function choose(markdown, { pathExists, log = true }) { // entry -- the first top-level bullet with a bold title -- so "next" is found // by removing the one just rejected and asking it again, rather than writing // a second parser that could disagree with it about what an entry is. - let commands; + let verification; while (markdown.length > 0) { const candidateEntry = selectBacklogEntry(markdown); if (!candidateEntry) break; @@ -108,12 +109,13 @@ async function choose(markdown, { pathExists, log = true }) { // rotted entry can never silently become an agent's instruction, and the // skip line names the missing paths so the entry can be repaired. const missing = unbounded ? [] : scope.filter((path) => !pathExists(path)); - const checks = verificationCommands(candidateEntry.body); + const contract = acceptanceContract(candidateEntry.body, baseline.head); + const checks = contract.checks; if (result.accepted && !unbounded && missing.length === 0 && checks.length > 0) { entry = candidateEntry; validation = result; - commands = checks; + verification = contract; break; } skipped.push({ @@ -145,7 +147,8 @@ async function choose(markdown, { pathExists, log = true }) { title: validation.work.title, filesInScope: validation.work.files_in_scope, definitionOfDone: validation.work.definition_of_done, - verificationCommands: commands, + verificationCommands: verification.checks.map(check => check.argv), + verification, brief: renderWorkPackage(entry), }; } @@ -205,9 +208,15 @@ async function verifiedPackage() { { stdio: 'ignore' }).status === 0, }); for (const key of Object.keys(selected)) { - assert.deepEqual(pkg[key], selected[key], `PACKAGE_CHANGED: ${key}`); + assert.deepEqual(pkg[key], selected[key], `PACKAGE_CHANGED: ${key}` + + (key === 'verification' ? '; ACCEPTANCE_IMMUTABILITY_VIOLATION: verification contract changed' : '')); + } + try { + checkScope(pkg); + validateAcceptance(pkg); + } catch (cause) { + throw new Error(`${cause.message}; DoD: ${pkg.definitionOfDone.join('; ')}`, { cause }); } - checkScope(pkg); return pkg; } @@ -235,6 +244,7 @@ try { const pkg = await verifiedPackage(); runChecks(pkg); await verifiedPackage(); + console.log(`PACKAGE_VERIFIED: ${pkg.verificationCommands.length} check(s)`); } else { console.error('usage: local-work-package.mjs '); process.exitCode = 2; diff --git a/ops/local-work-snapshot.mjs b/ops/local-work-snapshot.mjs index ed528128f..520a1bc23 100644 --- a/ops/local-work-snapshot.mjs +++ b/ops/local-work-snapshot.mjs @@ -16,6 +16,7 @@ try { for (const [source, destination] of [ ['ops/local-work-package.mjs', 'local-work-package.mjs'], ['ops/local-work-verification.mjs', 'local-work-verification.mjs'], + ['ops/local-work-acceptance.mjs', 'local-work-acceptance.mjs'], ['packages/sdk/src/backlog-picker.ts', 'backlog-picker.ts'], ]) { writeFileSync(join(directory, destination), git('show', `${head}:${source}`), { flag: 'wx', mode: 0o600 }); diff --git a/ops/local-work-test-fixture.mjs b/ops/local-work-test-fixture.mjs index 80689a97f..3b48ed7eb 100644 --- a/ops/local-work-test-fixture.mjs +++ b/ops/local-work-test-fixture.mjs @@ -16,7 +16,7 @@ export const entry = (title = 'Fix value', path = 'src/value.txt', commands = [c `- **${title}** Update \`${path}\` to the required value.\n` + commands.map(argv => ` Verify: ${JSON.stringify(argv)}\n`).join(''); -export function fixture(t, backlog = entry()) { +export function fixture(t, backlog = entry(), files = {}) { mkdirSync('.relayflow', { recursive: true }); const root = mkdtempSync(resolve('.relayflow/package-tests-')); t.after(() => rmSync(root, { recursive: true, force: true })); @@ -31,10 +31,12 @@ export function fixture(t, backlog = entry()) { put('outside.txt', 'original'); put('ops/BACKLOG.md', backlog); for (const path of ['ops/local-work-package.mjs', 'ops/local-work-verification.mjs', - 'ops/local-work-snapshot.mjs', 'packages/sdk/src/backlog-picker.ts', 'workflows/drive-local.yaml']) { + 'ops/local-work-snapshot.mjs', 'ops/local-work-acceptance.mjs', + 'packages/sdk/src/backlog-picker.ts', 'workflows/drive-local.yaml']) { put(path, readFileSync(path)); } put('packages/sdk/dist/backlog-picker.js', readFileSync('packages/sdk/dist/backlog-picker.js')); + for (const [path, contents] of Object.entries(files)) put(path, contents); git('add', '.'); git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.test', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture'); diff --git a/ops/local-work-verification.mjs b/ops/local-work-verification.mjs index e22544ffd..d01fee11d 100644 --- a/ops/local-work-verification.mjs +++ b/ops/local-work-verification.mjs @@ -1,26 +1,13 @@ import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; +import { execFileSync } from 'node:child_process'; import { lstatSync, readdirSync, realpathSync } from 'node:fs'; import { resolve, sep } from 'node:path'; - -// An explicit argv contract avoids treating prose or arbitrary backticked -// snippets as shell commands. Checks are authored before implementation. -export function verificationCommands(body) { - const lines = body.split('\n').filter(line => /^\s*Verify:/.test(line)); - return lines.map(line => { - let argv; - try { argv = JSON.parse(line.replace(/^\s*Verify:\s*/, '')); } - catch { throw new Error('INVALID_EXECUTABLE_CHECK: Verify must contain a JSON argv array'); } - assert(Array.isArray(argv) && argv.length > 0 && - argv.every(arg => typeof arg === 'string' && !arg.includes('\0')) && argv[0].trim(), - 'INVALID_EXECUTABLE_CHECK: expected nonempty command argv'); - return argv; - }); -} +import { runAcceptance } from './local-work-acceptance.mjs'; const protectedPaths = [ 'ops/BACKLOG.md', 'ops/local-work-package.mjs', 'ops/local-work-verification.mjs', 'ops/local-work-gate.mjs', 'ops/local-work-snapshot.mjs', 'scripts/run-drive-local.mjs', + 'ops/local-work-acceptance.mjs', 'workflows/drive-local.yaml', 'workflows/gates', 'packages/sdk/src/backlog-picker.ts', ]; const within = (path, scope) => path === scope || path.startsWith(`${scope}/`); @@ -105,12 +92,5 @@ export function checkScope(pkg) { } export function runChecks(pkg) { - assert(pkg.verificationCommands?.length > 0, 'MISSING_EXECUTABLE_CHECKS'); - for (const argv of pkg.verificationCommands) { - console.log(`CHECK ${JSON.stringify(argv)}`); - const result = spawnSync(argv[0], argv.slice(1), { stdio: 'inherit', timeout: 120000 }); - assert(!result.error && result.status === 0, - `PACKAGE_CHECK_FAILED: ${JSON.stringify(argv)} (${result.error?.message ?? result.signal ?? result.status})`); - } - console.log(`PACKAGE_VERIFIED: ${pkg.verificationCommands.length} check(s)`); + runAcceptance(pkg); } diff --git a/ops/runtime-evidence/drive-284-immutable-acceptance-0910.txt b/ops/runtime-evidence/drive-284-immutable-acceptance-0910.txt new file mode 100644 index 000000000..1fdbc6adc --- /dev/null +++ b/ops/runtime-evidence/drive-284-immutable-acceptance-0910.txt @@ -0,0 +1,144 @@ +Issue #284 verification, 2026-09-10 + +SDK and surface were built from this worktree; SDK dependencies are local to it. +The daemon was built with PATH=/Users/khaliqgant/.cargo/bin:$PATH sh ../ops/cargo.sh build. +The launcher uses socketPathFor to agree with the daemon socket relocation from #262. + +Working directory: packages/sdk +$ NPM_CONFIG_USERCONFIG=/dev/null npx --no-install tsc --noEmit + +[exit 0] + +Working directory: repository root +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/2814932978/debug/relayflowd node --test ops/local-work-acceptance.test.mjs ops/local-work-package.test.mjs ops/local-work-git-input.test.mjs ops/local-work-gate.test.mjs ops/local-work-review.test.mjs ops/drive-local-flow.test.mjs scripts/run-local-workflow.test.mjs +✔ drive-local journals failure and blocks reporting for outside edit (2348.330958ms) +✔ drive-local journals failure and blocks reporting for verifier edit (2414.250666ms) +✔ drive-local journals failure and blocks reporting for unchanged package (2384.257ms) +✔ drive-local journals failure and blocks reporting for HEAD repin (1785.088291ms) +✔ drive-local journals failure and blocks reporting for forged snapshot (1938.18475ms) +✔ drive-local journals failure and blocks reporting for edited pinned acceptance (2011.142417ms) +✔ drive-local journals failure and blocks reporting for undeclared acceptance (1201.992042ms) +✔ edited in-scope acceptance argv refuses while implementation stays broken (#284) (1567.470375ms) +✔ pinned acceptance executes Git bytes despite an edited checkout script (2773.775208ms) +✔ missing or unavailable script pin refuses: undefined (1526.077125ms) +✔ missing or unavailable script pin refuses: (1365.750875ms) +✔ missing or unavailable script pin refuses: main (1251.065125ms) +✔ missing or unavailable script pin refuses: ffffffffffffffffffffffffffffffffffffffff (1216.975875ms) +✔ all inputs are validated before any acceptance command executes (998.611583ms) +✔ external absolute acceptance reads the changed implementation (1304.858208ms) +✔ an absolute acceptance path inside write scope refuses (977.067792ms) +✔ an absolute acceptance path inside write scope refuses through a symlink (966.851459ms) +✔ a symlink beneath writable scope cannot grant access to an external acceptance file (929.881792ms) +✔ pinned relative dependencies also come from Git (1553.806375ms) +✔ an external input with an in-scope hard link is refused (893.072ms) +✔ package metadata cannot delete or repin the verification block (839.436167ms) +✔ declaring a shell runtime cannot authorize arbitrary mutable acceptance argv (1119.170792ms) +✔ an executable acceptance script is extracted from Git (1857.87075ms) +✔ isolated report refuses an outside edit and names the DoD (822.944167ms) +✔ a check that violates scope never emits PACKAGE_VERIFIED (935.567458ms) +✔ committing outside scope and repinning metadata cannot replace the submitted HEAD (1686.649667ms) +✔ forged legacy checkout snapshot and compiled picker cannot replace the Git-extracted judge (2852.912042ms) +✔ raw template refuses selection before implementation without captured gate inputs (436.66975ms) +✔ selection advances past rejected duplicate title: Same title (1107.619833ms) +✔ selection advances past rejected duplicate title: Regex [a].* (b) + $ (973.025916ms) +✔ preparation reads the committed gate instead of a working-tree replacement (1867.2475ms) +✔ gate-snapshot builds committed picker source outside the checkout (2197.015416ms) +✔ a committed picker compile failure prevents package selection (1118.3795ms) +✔ an ignored compiled picker cannot become the input to selection (1738.46ms) +✔ Git replacement objects cannot substitute gate source at the pinned ref (1538.807875ms) +✔ skip cursor passes a later title mentioned in an earlier body (1355.620292ms) +✔ selection skips missing checks and stale paths with reasons (1818.812792ms) +✔ a non-SDK package fails unchanged and passes only after its check holds (2158.062584ms) +✔ multiple acceptance checks all execute and failures propagate (1360.729792ms) +✔ scope refuses an outside unstaged path (1142.692542ms) +✔ scope refuses an outside staged path (1100.264167ms) +✔ scope refuses an outside untracked path (1031.287667ms) +✔ scope refuses an outside staged reversal path (985.621375ms) +✔ scope refuses an outside rename path (996.860292ms) +✔ scope refuses an outside deleted path (858.157542ms) +✔ the submitted scope command refuses a changed ops/local-work-package.mjs before loading it (849.030667ms) +✔ the submitted scope command refuses a changed ops/local-work-verification.mjs before loading it (831.58475ms) +✔ the submitted scope command refuses a changed ops/BACKLOG.md before loading it (744.950917ms) +✔ editing ignored package metadata cannot widen scope or replace checks (855.163292ms) +✔ scope allows directory children but rejects a sibling with the same prefix (908.193166ms) +✔ scope allows in-scope deletions and rejects symlink escapes (913.432208ms) +✔ replacing a scoped file with a directory does not authorize its children (825.9675ms) +✔ a check that writes outside scope fails verification (1183.466625ms) +✔ interrupted package write preserves the original and retry replaces it atomically (1656.624958ms) +✔ report includes an allowed unstaged change (1626.723667ms) +✔ report includes an allowed staged change (2199.780542ms) +✔ report includes an allowed untracked change (1727.172625ms) +✔ directory scope checks a pre-existing dangling escape symlink (1377.846125ms) +✔ directory scope checks a pre-existing existing internal symlink (1193.231417ms) +✔ selection skips untracked scope and accepts the next committed scope (919.005958ms) +✔ reporting after SDK suite effects enforces scope for outside.txt (1361.132292ms) +✔ reporting after SDK suite effects enforces scope for src/value.txt (1464.640166ms) +✔ local launcher journals deterministic effects and reads more than one journal page (1144.53775ms) +✔ a failed command fails the run and prevents dependent effects (245.741875ms) +✔ the SDK worker completes an agent step through the local journal protocol (751.097917ms) +✔ missing daemon is refused before a data directory or run is created (325.319542ms) +✔ an agent followed by work over 30 seconds reaches report and exports its journal (32642.067167ms) +✔ a rejected worker completion preserves the protocol error and cannot report success (421.480541ms) +ℹ tests 68 +ℹ suites 0 +ℹ pass 68 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 35624.611417 + +[exit 0] + +Working directory: packages/sdk +$ node node_modules/vitest/vitest.mjs run tests/backlog-picker.test.ts tests/backlog-picker-flow.test.ts tests/spec-parity.test.ts + + RUN v2.1.9 /Users/khaliqgant/flows-284/packages/sdk + + ✓ tests/backlog-picker.test.ts (14 tests) 103ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 786ms + ✓ backlog-picker flow > does not emit a stale entry left by a previous run 375ms + ✓ tests/spec-parity.test.ts (31 tests) 344ms + + Test Files 3 passed (3) + Tests 51 passed (51) + Start at 22:29:23 + Duration 1.45s (transform 429ms, setup 0ms, collect 818ms, tests 1.23s, environment 0ms, prepare 183ms) + + +[exit 0] + +Direct regression probe (repository root): +$ node --input-type=module <<'JS' +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { entry, fixture, pass, fail } from './ops/local-work-test-fixture.mjs'; +const cleanup = []; +try { + const f = fixture({ after: fn => cleanup.push(fn) }, entry('Fix value', 'src/', [['node', 'src/check.cjs']]), { + 'src/check.cjs': "require('node:assert/strict').equal(require('node:fs').readFileSync('src/value.txt','utf8'),'fixed');", + }); + pass(f.run('select')); + f.put('src/check.cjs', 'process.exit(0);'); + const result = f.run('verify'); + fail(result, /ACCEPTANCE_IMMUTABILITY_VIOLATION/); + console.log('VERIFY_EXIT=' + result.status); + process.stdout.write(result.stdout + result.stderr); + console.log('IMPLEMENTATION=' + readFileSync(join(f.root, 'src/value.txt'), 'utf8')); + const reportFixture = fixture({ after: fn => cleanup.push(fn) }); + pass(reportFixture.run('select')); + reportFixture.put('outside.txt', 'changed'); + const report = reportFixture.run('report'); + fail(report, /OUT_OF_SCOPE: outside.txt; DoD:/); + console.log('REPORT_EXIT=' + report.status); + process.stdout.write(report.stdout + report.stderr); +} finally { for (const fn of cleanup) fn(); } +JS +VERIFY_EXIT=1 +SCOPE_OK: 1 changed path(s) +ACCEPTANCE_IMMUTABILITY_VIOLATION: acceptance script is implementation-writable: src/check.cjs; DoD: Fix value +IMPLEMENTATION=broken +REPORT_EXIT=1 +OUT_OF_SCOPE: outside.txt; DoD: Fix value + +[probe exit 0; verify and report each refused with exit 1] diff --git a/scripts/run-local-workflow.mjs b/scripts/run-local-workflow.mjs index fc896d317..2457a596e 100644 --- a/scripts/run-local-workflow.mjs +++ b/scripts/run-local-workflow.mjs @@ -21,6 +21,7 @@ async function main() { const { classifyOutcome } = await import('../packages/sdk/dist/cli/run.js'); const { toKernelSpec } = await import('../packages/sdk/dist/compile.js'); const { JournalClient } = await import('../packages/sdk/dist/journal-client.js'); + const { socketPathFor } = await import('../packages/sdk/dist/daemon-connection.js'); const { AgentWorker } = await import('../packages/sdk/dist/worker.js'); const checked = checkFlow(path); for (const diagnostic of checked.report.diagnostics) console.error(JSON.stringify(diagnostic)); @@ -47,7 +48,7 @@ async function main() { }); await mkdir('.relayflow', { recursive: true }); const dataDir = await mkdtemp(join(root, '.relayflow', 'local-')); - const socket = join(dataDir, 'relayflowd.sock'); + const socket = socketPathFor(dataDir); if (Buffer.byteLength(socket) >= 104) throw new Error(`LOCAL_SOCKET_PATH_TOO_LONG: ${socket}`); console.log(`LOCAL_DATA_DIR=${dataDir}`); const daemon = spawn(binary, ['--data-dir', dataDir, 'serve'], { stdio: ['ignore', 'ignore', 'inherit'] }); diff --git a/scripts/run-local-workflow.test.mjs b/scripts/run-local-workflow.test.mjs index d00ed5fa0..c0a026ee9 100644 --- a/scripts/run-local-workflow.test.mjs +++ b/scripts/run-local-workflow.test.mjs @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import test, { after } from 'node:test'; +import { socketPathFor } from '../packages/sdk/dist/daemon-connection.js'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); mkdirSync(join(root, '.relayflow'), { recursive: true }); @@ -24,7 +25,7 @@ function run(name, steps, env = {}, timeout = 15000) { if (dataDir) dataDirectories.add(dataDir); assert.equal(result.error, undefined, result.stderr); const records = result.stdout.split('\n').filter(line => line.startsWith('{')).map(line => JSON.parse(line)); - if (dataDir) assert(!existsSync(join(dataDir, 'relayflowd.sock')), 'owned daemon socket must be removed on exit'); + if (dataDir) assert(!existsSync(socketPathFor(dataDir)), 'owned daemon socket must be removed on exit'); return { ...result, records, dataDir }; }