diff --git a/.github/workflows/review-swarm.yml b/.github/workflows/review-swarm.yml index ca73d81f0..02c3c12ef 100644 --- a/.github/workflows/review-swarm.yml +++ b/.github/workflows/review-swarm.yml @@ -46,6 +46,42 @@ jobs: .github/workflows/scripts/swarm-post.sh .github/workflows/scripts/swarm-prepare.sh .github/workflows/scripts/swarm-verdict.sh + .github/workflows/scripts/swarm-gate.test.sh + + # The gate decides whether code merges, so before it judges anything it + # proves it can still say no. `swarm-verdict.sh` is a handful of lines of + # shell; the failure that matters is not this gate going red -- a red gate + # announces itself -- but this gate quietly losing the ability to go red, + # which announces nothing and surfaces only after something broken has + # merged behind a green check. That is not hypothetical here: on + # 2026-09-09 a release shipped past a smoke test that could not fail. + # + # The suite asserts both directions. A genuine REVIEW_FAILED must fail the + # gate, and three clean passes must pass it -- without that second half + # every assertion would be satisfiable by an unconditional `exit 1`, and + # an always-red gate is as useless as an always-green one. It is hermetic + # (agent-relay and gh are stubbed) and runs offline in a few seconds, so + # it costs nothing to run ahead of a 20-minute cloud swarm. + # + # It runs the copy from main, alongside the scripts it tests, so a pull + # request cannot weaken the gate by editing the test that guards it. The + # guard is skipped only while the file has not yet reached main. + - name: Self-test the gate's verdict logic + env: + REVIEW_PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + test_script=gate-files/.github/workflows/scripts/swarm-gate.test.sh + if [ ! -f "$test_script" ]; then + # Only the introducing PR may bootstrap before its test is on main. + # Later deletion, bad paths or incomplete checkouts must fail closed. + if [ "$REVIEW_PR_NUMBER" = 248 ]; then + echo "::notice::PR #248 bootstrap: self-test is not on main yet." + exit 0 + fi + echo "::error::main-owned swarm-gate.test.sh is missing" >&2 + exit 1 + fi + bash "$test_script" # Fail here, in seconds, rather than in `Launch cloud swarm` ten minutes # later. WorkflowApiKeyClient.fromEnv requires CLOUD_API_URL and diff --git a/.github/workflows/scripts/swarm-gate.test.sh b/.github/workflows/scripts/swarm-gate.test.sh new file mode 100755 index 000000000..765bc758c --- /dev/null +++ b/.github/workflows/scripts/swarm-gate.test.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Regression test for the review-swarm gate's verdict path. +# +# Why this exists: the gate is three lines of shell deciding whether code +# merges. The failure mode that matters is not "the gate is red" -- a red gate +# announces itself. It is a gate that has quietly become incapable of being +# red, which announces nothing and is discovered only after something broken +# ships behind it. So this suite asserts BOTH directions: a genuine objection +# must fail, and a genuine pass must pass. A suite that only checked the happy +# path would itself be the vacuous green it is meant to prevent. +# +# Hermetic: `agent-relay` and `gh` are stubbed on PATH, so this runs offline +# and exercises the real swarm-post.sh / swarm-verdict.sh, not a paraphrase. +set -uo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +pass=0 +fail=0 + +ok() { pass=$((pass+1)); printf ' ok %s\n' "$1"; } +notok(){ fail=$((fail+1)); printf ' FAIL %s\n expected: %s\n actual: %s\n' "$1" "$2" "$3"; } + +expect_eq() { + local label=$1 want=$2 got=$3 + [ "$want" = "$got" ] && ok "$label" || notok "$label" "$want" "$got" +} + +# --------------------------------------------------------------------------- +# Unit: verdict extraction from a transcript's final non-empty line. +# --------------------------------------------------------------------------- +# shellcheck source=swarm-verdict.sh +source "$script_dir/swarm-verdict.sh" + +verdict_of() { + local tmp; tmp=$(mktemp) + printf '%s' "$1" > "$tmp" + swarm_transcript_verdict "$tmp" + rm -f "$tmp" +} + +echo "== verdict extraction ==" +expect_eq "a bare REVIEW_FAILED is FAILED" \ + FAILED "$(verdict_of 'Findings: P1 leak. +REVIEW_FAILED')" + +expect_eq "a bare REVIEW_PASSED is PASSED" \ + PASSED "$(verdict_of 'Looks good. +REVIEW_PASSED')" + +expect_eq "trailing blank lines do not hide the marker" \ + FAILED "$(verdict_of 'REVIEW_FAILED + +')" + +expect_eq "surrounding whitespace is trimmed" \ + PASSED "$(verdict_of ' REVIEW_PASSED ')" + +# The fix is prompt-side: agents must not append a sign-off. This test pins +# the parser's correct refusal of trailing text; it does not prove agents obey +# the prompt in a live run (the failure observed on PR #240). +expect_eq "a marker followed by a sign-off is UNCLEAR (PR #240 bug)" \ + UNCLEAR "$(verdict_of 'REVIEW_PASSED + +**Review completed:** 2026-09-09 08:45')" + +# The safety property that must survive the #248 prompt change. If a lens +# genuinely objects, no amount of prompt wording may turn that into a pass. +expect_eq "REVIEW_FAILED is never upgraded by surrounding prose" \ + UNCLEAR "$(verdict_of 'REVIEW_FAILED +structure-only review.')" + +expect_eq "an empty transcript is UNCLEAR, not PASSED" \ + UNCLEAR "$(verdict_of '')" + +expect_eq "a transcript merely containing the word is UNCLEAR" \ + UNCLEAR "$(verdict_of 'I considered emitting REVIEW_PASSED but did not.')" + +# --------------------------------------------------------------------------- +# Unit: lens selection -- missing and stale transcripts must be fail-closed. +# --------------------------------------------------------------------------- +echo "== lens selection ==" +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +mkdir -p "$work/ops/reviews" + +expect_eq "no reviews directory yields MISSING" \ + "MISSING " "$(swarm_lens_result "$work/nope" 246 structure "")" + +expect_eq "an absent transcript yields MISSING" \ + "MISSING " "$(swarm_lens_result "$work/ops/reviews" 246 structure "")" + +# Fixed timestamps exercise mtime-based freshness without a wall-clock race. +# They do not test invalidation of reviews by a Git force-push. +marker="$work/marker"; touch -t 202601010001 "$marker" +old="$work/ops/reviews/20260101-0000-pr246-structure.md" +printf 'REVIEW_PASSED\n' > "$old" +touch -t 202601010000 "$old" # older than the marker +expect_eq "a transcript predating the run yields STALE" \ + STALE "$(swarm_lens_result "$work/ops/reviews" 246 structure "$marker" | cut -f1)" +touch -r "$marker" "$old" +expect_eq "a transcript with the marker's exact mtime yields STALE" \ + STALE "$(swarm_lens_result "$work/ops/reviews" 246 structure "$marker" | cut -f1)" + +fresh="$work/ops/reviews/20260909-1200-pr246-structure.md" +printf 'REVIEW_PASSED\n' > "$fresh" +touch -t 202601010002 "$fresh" +expect_eq "the newest fresh transcript wins" \ + PASSED "$(swarm_lens_result "$work/ops/reviews" 246 structure "$marker" | cut -f1)" + +# --------------------------------------------------------------------------- +# End-to-end: the real swarm-post.sh, with agent-relay and gh stubbed. +# --------------------------------------------------------------------------- +echo "== swarm-post.sh end to end ==" + +# $1 = sync behaviour: 'writes:=,...' or 'nochanges' +run_post() { + local spec=$1 sandbox bin + sandbox=$(mktemp -d) + bin="$sandbox/bin"; mkdir -p "$bin" "$sandbox/repo" + + cat > "$bin/agent-relay" < "ops/reviews/20260909-1200-pr999-\$lens.md" + done + echo "Synced." +fi +exit 0 +STUB + + # gh must never reach the network from a test. Record calls instead. + cat > "$bin/gh" <> "$sandbox/gh-calls.log" +exit 0 +STUB + chmod +x "$bin/agent-relay" "$bin/gh" + + ( cd "$sandbox/repo" && PATH="$bin:$PATH" \ + bash "$script_dir/swarm-post.sh" test-run-id 999 >"$sandbox/out" 2>&1 ) + local rc=$? + post_out=$(cat "$sandbox/out") + post_log=$(cat "$sandbox/gh-calls.log" 2>/dev/null || true) + rm -rf "$sandbox" + return $rc +} + +# Did the run report its verdict to the PR at all? A gate that fails silently +# is only half a gate: the check is red but nothing says which lens objected. +posted_a_rollup() { case "$post_log" in *"pr comment"*) return 0 ;; *) return 1 ;; esac; } + +# THE load-bearing assertion. A genuine objection from one lens must fail the +# gate even when the other two pass. +run_post 'writes:maintainability=REVIEW_PASSED,history=REVIEW_PASSED,structure=REVIEW_FAILED' +expect_eq "one lens REVIEW_FAILED fails the gate (exit 1)" 1 "$?" +case "$post_log" in + *"- structure: FAILED"*) ok "the objection is reported as FAILED, not STALE" ;; + *) notok "the objection is reported as FAILED, not STALE" "structure: FAILED" "$post_log" ;; +esac +posted_a_rollup \ + && ok "a failing run still reports its verdict to the PR" \ + || notok "a failing run still reports its verdict to the PR" "a gh comment" "none" + +# The counterweight: a gate that cannot pass is as broken as one that cannot +# fail. Without this, every assertion above is satisfiable by \`exit 1\`. +run_post 'writes:maintainability=REVIEW_PASSED,history=REVIEW_PASSED,structure=REVIEW_PASSED' +expect_eq "three clean passes pass the gate (exit 0)" 0 "$?" + +run_post 'writes:maintainability=REVIEW_PASSED,history=REVIEW_PASSED,structure=UNCLEAR_JUNK' +expect_eq "an UNCLEAR lens fails the gate" 1 "$?" + +run_post 'writes:maintainability=REVIEW_PASSED,history=REVIEW_PASSED' +expect_eq "a lens with no transcript at all fails the gate" 1 "$?" + +# The #246/#247/#248 production shape: swarm died, nothing synced. +run_post nochanges +rc=$? +[ "$rc" -ne 0 ] && ok "an empty sync fails the gate (exit $rc)" \ + || notok "an empty sync fails the gate" "non-zero" "$rc" + +case "$post_out" in + *"No changes to sync"*) ok "the empty-sync reason reaches the step log" ;; + *) notok "the empty-sync reason reaches the step log" "the CLI message" "$post_out" ;; +esac + +# TODO (PR #248): give empty syncs a current failure comment. `set -e` kills +# swarm-post.sh at `agent-relay cloud sync`, so when the swarm dies the PR gets +# no comment from this run and the previous run's rollup stays visible. The +# gate is still red -- `Enforce swarm result` is a separate step keyed on +# swarm_status -- but a reader looking only at PR comments sees a stale verdict. +# This is deliberately outside pass/fail accounting. When the posting path is +# repaired, add a positive assertion for the new contract; never require the bug. + +echo +printf '%d passed, %d failed\n' "$pass" "$fail" +[ "$fail" -eq 0 ] diff --git a/ops/runtime-evidence/swarm-threads-0909-after.txt b/ops/runtime-evidence/swarm-threads-0909-after.txt new file mode 100644 index 000000000..99841edcf --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-after.txt @@ -0,0 +1,30 @@ +$ bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok a transcript with the marker's exact mtime yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok the objection is reported as FAILED, not STALE + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + NOTE PR #248 limitation: empty sync posts no comment; not a passing assertion + +21 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-baseline-probe.py b/ops/runtime-evidence/swarm-threads-0909-baseline-probe.py new file mode 100644 index 000000000..1521cfe74 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-baseline-probe.py @@ -0,0 +1,24 @@ +"""Run the original PR self-test with the corrected whole-second comparator.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path.cwd() +Path('.relayflow').mkdir(exist_ok=True) +with tempfile.TemporaryDirectory(dir='.relayflow', prefix='swarm-baseline-') as directory: + target = Path(directory).resolve() + for name in ['swarm-gate.test.sh', 'swarm-post.sh', 'swarm-verdict.sh']: + source = f'066e2deecea5ffb88fdce088a98da111b547d803:.github/workflows/scripts/{name}' + (target / name).write_bytes(subprocess.check_output(['git', 'show', source])) + env = dict(os.environ, BASH_ENV=str(root / 'ops/runtime-evidence/swarm-threads-0909-coarse.bash')) + # This is a timing race; preserve every attempt, including passing ones. + for attempt in range(1, 6): + result = subprocess.run(['bash', str(target / 'swarm-gate.test.sh')], env=env, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(f'BASELINE_ATTEMPT={attempt}') + print(result.stdout, end='') + print(f'EXIT_CODE={result.returncode}') + if result.returncode: + raise SystemExit(result.returncode) + print('NO_FAILURE_OBSERVED_IN_FIVE_ATTEMPTS') diff --git a/ops/runtime-evidence/swarm-threads-0909-before.txt b/ops/runtime-evidence/swarm-threads-0909-before.txt new file mode 100644 index 000000000..4c62063d9 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-before.txt @@ -0,0 +1,28 @@ +$ bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + ok KNOWN: an empty sync posts no comment; the red check is the only signal + +20 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs b/ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs new file mode 100644 index 000000000..e5c727758 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join, resolve } from 'node:path'; + +const require = createRequire(resolve('packages/sdk/package.json')); +const source = process.argv[2] + ? execFileSync('git', ['show', `${process.argv[2]}:.github/workflows/review-swarm.yml`], { encoding: 'utf8' }) + : readFileSync('.github/workflows/review-swarm.yml', 'utf8'); +const flow = require('js-yaml').load(source); +const command = flow.jobs.review.steps.find(step => step.name === "Self-test the gate's verdict logic").run; +mkdirSync('.relayflow', { recursive: true }); +for (const [label, pr, script, expected] of [ + ['introducing PR without test', '248', null, 0], + ['later PR without test', '249', null, 1], + ['present passing test', '249', 'echo SELF_TEST_RAN; exit 0', 0], + ['present failing test', '249', 'echo SELF_TEST_RAN; exit 7', 7], +]) { + const cwd = mkdtempSync(resolve('.relayflow/bootstrap-probe-')); + try { + if (script !== null) { + const directory = join(cwd, 'gate-files/.github/workflows/scripts'); + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, 'swarm-gate.test.sh'), script); + } + const result = spawnSync('bash', ['-e', '-c', command], { + cwd, encoding: 'utf8', env: { ...process.env, REVIEW_PR_NUMBER: pr }, + }); + console.log(`${label}: exit=${result.status}, expected=${expected}`); + process.stdout.write(result.stdout + result.stderr); + assert.equal(result.status, expected); + if (script !== null) assert.match(result.stdout, /SELF_TEST_RAN/); + } finally { rmSync(cwd, { recursive: true, force: true }); } +} diff --git a/ops/runtime-evidence/swarm-threads-0909-coarse-after.txt b/ops/runtime-evidence/swarm-threads-0909-coarse-after.txt new file mode 100644 index 000000000..95b80ba2e --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-coarse-after.txt @@ -0,0 +1,33 @@ +SUPERSEDED: this comparator did not intercept the negated production check. +This is not evidence of simulated coarse timestamps; see corrected-* captures. + +$ BASH_ENV="$PWD/ops/runtime-evidence/swarm-threads-0909-coarse.bash" bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok a transcript with the marker's exact mtime yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok the objection is reported as FAILED, not STALE + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + NOTE PR #248 limitation: empty sync posts no comment; not a passing assertion + +21 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-coarse-before.txt b/ops/runtime-evidence/swarm-threads-0909-coarse-before.txt new file mode 100644 index 000000000..657809fb8 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-coarse-before.txt @@ -0,0 +1,33 @@ +SUPERSEDED: this comparator did not intercept the negated production check. +This is not evidence of simulated coarse timestamps; see corrected-* captures. + +$ BASH_ENV=/tmp/flows-coarse-mtime.bash bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok a failing run still reports its verdict to the PR + FAIL three clean passes pass the gate (exit 0) + expected: 0 + actual: 1 + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + ok KNOWN: an empty sync posts no comment; the red check is the only signal + +19 passed, 1 failed + +EXIT_CODE=1 diff --git a/ops/runtime-evidence/swarm-threads-0909-coarse.bash b/ops/runtime-evidence/swarm-threads-0909-coarse.bash new file mode 100644 index 000000000..90d434297 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-coarse.bash @@ -0,0 +1,18 @@ +# Reproduce whole-second filesystem timestamp comparison, without changing gates. +# macOS only: BSD stat's -f %m returns whole-second mtime (GNU stat differs). +# Only plain and negated -nt are replaced. All other expressions delegate to +# Bash unchanged; "$@" INCLUDES the caller's closing ] argument. Adding another +# ] to the builtin invocation would be an error, not a syntax repair. +if [[ $(uname -s) != Darwin ]]; then + printf '%s\n' 'COARSE_FIXTURE_REQUIRES_MACOS: use the native suite on other hosts' >&2 + exit 2 +fi +function [ { + if builtin [ "$#" -eq 4 ] && builtin [ "$2" = '-nt' ]; then + builtin [ "$(stat -f %m "$1")" -gt "$(stat -f %m "$3")" ] + elif builtin [ "$#" -eq 5 ] && builtin [ "$1" = '!' ] && builtin [ "$3" = '-nt' ]; then + builtin [ ! "$(stat -f %m "$2")" -gt "$(stat -f %m "$4")" ] + else + builtin [ "$@" + fi +} diff --git a/ops/runtime-evidence/swarm-threads-0909-comparator-after.txt b/ops/runtime-evidence/swarm-threads-0909-comparator-after.txt new file mode 100644 index 000000000..c75477a34 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-comparator-after.txt @@ -0,0 +1,3 @@ +$ python3 ops/runtime-evidence/swarm-threads-0909-comparator-probe.py +PLAIN=NOT_NEWER NEGATED=STALE STAT_CALLS=4 +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-comparator-before.txt b/ops/runtime-evidence/swarm-threads-0909-comparator-before.txt new file mode 100644 index 000000000..17ad89a93 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-comparator-before.txt @@ -0,0 +1,3 @@ +$ python3 /tmp/flows-248-comparator-probe.py +PLAIN=NOT_NEWER NEGATED=STALE STAT_CALLS=2 +EXIT_CODE=1 diff --git a/ops/runtime-evidence/swarm-threads-0909-comparator-probe.py b/ops/runtime-evidence/swarm-threads-0909-comparator-probe.py new file mode 100644 index 000000000..fcb88d45d --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-comparator-probe.py @@ -0,0 +1,28 @@ +import os, subprocess, tempfile +from pathlib import Path +# Native macOS Bash can already compare at whole-second precision, so checking +# outcomes alone missed the original wrapper's failure to intercept negated -nt. +# Two expressions, two operands each: four stat calls prove both routes used +# this frozen reproduction fixture. Outcomes are asserted separately below. +# If the fixture is redesigned to cache stats, adapt this interception probe too. +with tempfile.TemporaryDirectory() as directory: + marker=Path(directory)/'marker'; newer=Path(directory)/'newer' + marker.touch(); newer.touch() + os.utime(marker,ns=(1700000000100000000,1700000000100000000)) + os.utime(newer,ns=(1700000000900000000,1700000000900000000)) + script='''trace=$4 +stat() { printf "stat\\n" >> "$trace"; command stat "$@"; } +source "$1" +[ yes = yes ] || exit 1 +[ ! -e "$4.missing" ] || exit 1 +printf 'DELEGATION_OK\\n' +if [ "$3" -nt "$2" ]; then plain=NEWER; else plain=NOT_NEWER; fi +if [ ! "$3" -nt "$2" ]; then negated=STALE; else negated=FRESH; fi +calls=$(wc -l < "$trace") +printf 'PLAIN=%s NEGATED=%s STAT_CALLS=%d\\n' "$plain" "$negated" "$calls" +builtin [ "$plain" = NOT_NEWER ] && builtin [ "$negated" = STALE ] && builtin [ "$calls" -eq 4 ] +''' + result=subprocess.run(['bash','-c',script,'probe',str(Path('ops/runtime-evidence/swarm-threads-0909-coarse.bash').resolve()),str(marker),str(newer),str(Path(directory)/'calls')],text=True,capture_output=True) + print(result.stdout+result.stderr,end='') + print('EXIT_CODE='+str(result.returncode)) + raise SystemExit(result.returncode) diff --git a/ops/runtime-evidence/swarm-threads-0909-corrected-after.txt b/ops/runtime-evidence/swarm-threads-0909-corrected-after.txt new file mode 100644 index 000000000..9ea6fb279 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-corrected-after.txt @@ -0,0 +1,30 @@ +$ BASH_ENV="$PWD/ops/runtime-evidence/swarm-threads-0909-coarse.bash" bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok a transcript with the marker's exact mtime yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok the objection is reported as FAILED, not STALE + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + NOTE PR #248 limitation: empty sync posts no comment; not a passing assertion + +21 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-corrected-before-initial.txt b/ops/runtime-evidence/swarm-threads-0909-corrected-before-initial.txt new file mode 100644 index 000000000..537798ca6 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-corrected-before-initial.txt @@ -0,0 +1,28 @@ +$ python3 ops/runtime-evidence/swarm-threads-0909-baseline-probe.py +Initial single-attempt invocation (before adding the bounded retry loop): +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + ok KNOWN: an empty sync posts no comment; the red check is the only signal + +20 passed, 0 failed +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-corrected-before.txt b/ops/runtime-evidence/swarm-threads-0909-corrected-before.txt new file mode 100644 index 000000000..3bf3e4743 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-corrected-before.txt @@ -0,0 +1,30 @@ +$ python3 ops/runtime-evidence/swarm-threads-0909-baseline-probe.py +BASELINE_ATTEMPT=1 +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok a failing run still reports its verdict to the PR + FAIL three clean passes pass the gate (exit 0) + expected: 0 + actual: 1 + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + ok KNOWN: an empty sync posts no comment; the red check is the only signal + +19 passed, 1 failed +EXIT_CODE=1 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-after.txt b/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-after.txt new file mode 100644 index 000000000..9c13dc272 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-after.txt @@ -0,0 +1,11 @@ +$ node ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs +introducing PR without test: exit=0, expected=0 +::notice::PR #248 bootstrap: self-test is not on main yet. +later PR without test: exit=1, expected=1 +::error::main-owned swarm-gate.test.sh is missing +present passing test: exit=0, expected=0 +SELF_TEST_RAN +present failing test: exit=7, expected=7 +SELF_TEST_RAN + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-before.txt b/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-before.txt new file mode 100644 index 000000000..46f0ba09f --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-bootstrap-before.txt @@ -0,0 +1,28 @@ +$ node ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs f0e8e2a +introducing PR without test: exit=0, expected=0 +::notice::swarm-gate.test.sh is not on main yet; skipping the gate self-test. This skip disappears once it lands. +later PR without test: exit=0, expected=1 +::notice::swarm-gate.test.sh is not on main yet; skipping the gate self-test. This skip disappears once it lands. +node:internal/modules/run_main:107 + triggerUncaughtException( + ^ + +AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: + +0 !== 1 + + at file:///Users/khaliqgant/flows-threads-wt/ops/runtime-evidence/swarm-threads-0909-bootstrap-probe.mjs:32:12 + at ModuleJob.run (node:internal/modules/esm/module_job:569:25) + at async node:internal/modules/esm/loader:650:26 + at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) { + generatedMessage: true, + code: 'ERR_ASSERTION', + actual: 0, + expected: 1, + operator: 'strictEqual', + diff: 'simple' +} + +Node.js v26.7.0 + +EXIT_CODE=1 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-coarse.txt b/ops/runtime-evidence/swarm-threads-0909-round3-coarse.txt new file mode 100644 index 000000000..5bbbea73f --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-coarse.txt @@ -0,0 +1,29 @@ +$ BASH_ENV="$PWD/ops/runtime-evidence/swarm-threads-0909-coarse.bash" bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok a transcript with the marker's exact mtime yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok the objection is reported as FAILED, not STALE + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + +21 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-comparator.txt b/ops/runtime-evidence/swarm-threads-0909-round3-comparator.txt new file mode 100644 index 000000000..9423bb7d5 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-comparator.txt @@ -0,0 +1,4 @@ +$ python3 ops/runtime-evidence/swarm-threads-0909-comparator-probe.py +DELEGATION_OK +PLAIN=NOT_NEWER NEGATED=STALE STAT_CALLS=4 +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-native.txt b/ops/runtime-evidence/swarm-threads-0909-round3-native.txt new file mode 100644 index 000000000..4125fdf73 --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-native.txt @@ -0,0 +1,29 @@ +$ bash .github/workflows/scripts/swarm-gate.test.sh +== verdict extraction == + ok a bare REVIEW_FAILED is FAILED + ok a bare REVIEW_PASSED is PASSED + ok trailing blank lines do not hide the marker + ok surrounding whitespace is trimmed + ok a marker followed by a sign-off is UNCLEAR (PR #240 bug) + ok REVIEW_FAILED is never upgraded by surrounding prose + ok an empty transcript is UNCLEAR, not PASSED + ok a transcript merely containing the word is UNCLEAR +== lens selection == + ok no reviews directory yields MISSING + ok an absent transcript yields MISSING + ok a transcript predating the run yields STALE + ok a transcript with the marker's exact mtime yields STALE + ok the newest fresh transcript wins +== swarm-post.sh end to end == + ok one lens REVIEW_FAILED fails the gate (exit 1) + ok the objection is reported as FAILED, not STALE + ok a failing run still reports its verdict to the PR + ok three clean passes pass the gate (exit 0) + ok an UNCLEAR lens fails the gate + ok a lens with no transcript at all fails the gate + ok an empty sync fails the gate (exit 1) + ok the empty-sync reason reaches the step log + +21 passed, 0 failed + +EXIT_CODE=0 diff --git a/ops/runtime-evidence/swarm-threads-0909-round3-platform.txt b/ops/runtime-evidence/swarm-threads-0909-round3-platform.txt new file mode 100644 index 000000000..cd729e36b --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909-round3-platform.txt @@ -0,0 +1,3 @@ +$ bash -c 'uname(){ printf "%s\n" Linux; }; source "$1"' probe ops/runtime-evidence/swarm-threads-0909-coarse.bash +COARSE_FIXTURE_REQUIRES_MACOS: use the native suite on other hosts +EXIT_CODE=2 diff --git a/ops/runtime-evidence/swarm-threads-0909.md b/ops/runtime-evidence/swarm-threads-0909.md new file mode 100644 index 000000000..68ad9036d --- /dev/null +++ b/ops/runtime-evidence/swarm-threads-0909.md @@ -0,0 +1,64 @@ +# PR #248 review corrections + +The unsupported claim in commit b8c771c that four deliberate mutations each +failed and restored scripts then passed is withdrawn. The available record +does not contain the commands, failures, byte-for-byte restoration, and restored +passes needed to substantiate it. This correction does not rewrite that commit +or claim those historical experiments did or did not happen. No mutation +verification is claimed by this follow-up. + +The ordinary baseline suite passed on this host. The first comparator did not +intercept production's negated `[ ! file -nt marker ]` expression. Its original +coarse-before/coarse-after captures are NOT evidence of simulated coarse +timestamps; that claim is withdrawn. A probe counting `stat` calls reproduces +the missing interception (2 calls before, 4 after supporting both forms). +The corrected comparator is used in corrected-before/corrected-after captures. +The baseline probe executes the original 066e2de scripts in a temporary fixture; +it does not edit the current checkout's gate scripts. +Fixed timestamps replace the unit test's wall-clock dependency, and an explicit +equal-mtime case remains STALE. The end-to-end objection assertion also requires +the reported verdict to be FAILED, so a stale transcript cannot stand in for a +real objection. + +The complete commands/output and the comparator source are recorded in the +adjacent swarm-threads-0909-*.txt files. The comparator is a macOS reproduction +fixture that wraps Bash's `[` for plain and negated `-nt`, using integer +`stat -f %m` values; it does not alter the production parser or judging gate. +Native and corrected coarse-timestamp runs exercise the real swarm-post.sh with +offline CLI stubs. They do not prove a live agent follows a prompt. + +The YAML now names the exact parser file and function. Empty-sync missing-comment +behavior is diagnostic output outside pass/fail accounting, so repairing it +later will not fail a test that required the bug. The existing liveness-only +annotations are preserved. + +The CI review and its self-test come from the immutable main checkout. These +changes remain candidates for human review; they do not replace the gate that +judges this PR, and a local test pass is not a swarm signoff. + +## Evidence lifecycle for this batch + +These files are historical command captures and narrowly scoped reproduction +probes, not shared runtime helpers or automatically discovered test fixtures. +Keep superseded captures for the audit trail; their headers withdraw the claims +they cannot support. Do not regenerate an old capture in place. A later repair +gets a new descriptive suffix and its literal command/output, while its note +identifies what it supersedes. This convention applies to this review batch, +not a new policy for unrelated evidence directories. + +All names below start with `swarm-threads-0909-` (review task and capture date): + +| Suffix | Meaning | +| --- | --- | +| `before.txt`, `after.txt` | Native self-test before/after the first repair | +| `coarse-before.txt`, `coarse-after.txt` | Superseded first comparator captures; retained only for audit | +| `comparator-before.txt`, `comparator-after.txt` | Probe exposing the missing negated interception, then its repair | +| `corrected-before-initial.txt` | Original self-test happened to pass with the corrected comparator | +| `corrected-before.txt` | Bounded retry captured the timing failure; passing attempts are never hidden | +| `corrected-after.txt` | Fixed self-test under the corrected comparator | +| `round3-*.txt` | Follow-up captures for documentation, bootstrap refusal, and fixture repairs | + +Run the native shell suite for current regression checks. Use the macOS-only +comparator/probes only to investigate the specific historical timestamp issue. +The baseline probe requires the original `066e2de` Git objects and intentionally +reads those old scripts into a temporary fixture. Its output is timing-dependent. diff --git a/workflows/review-swarm.yaml b/workflows/review-swarm.yaml index 571253d48..e4d106674 100644 --- a/workflows/review-swarm.yaml +++ b/workflows/review-swarm.yaml @@ -19,6 +19,13 @@ swarm: maxConcurrency: 3 agents: + # Transcript contract: .github/workflows/scripts/swarm-verdict.sh, + # swarm_transcript_verdict(): last non-empty line, surrounding whitespace + # trimmed, exact REVIEW_PASSED/REVIEW_FAILED match. The aggregate uses this + # parser; changing the prompt must preserve that contract. + # Step liveness checks observe the agent's chat output. The aggregate opens + # its persisted transcript file and applies the binding parser there; chat + # output containing REVIEW_ cannot make a malformed transcript pass. # Deliberately three different model families: a shared blind spot in one # harness must not become the whole team's blind spot. - name: maintainability @@ -84,7 +91,15 @@ workflows: Read AGENTS.md and docs/RFC-0001-everything-is-a-relayflow.md first. Write your complete review to ops/reviews/$(date +%Y%m%d-%H%M)-pr$(cat .review-target/pr-number)-maintainability.md - and `git add` it. End your output with REVIEW_PASSED or REVIEW_FAILED. + and `git add` it. The LAST NON-EMPTY LINE OF THE TRANSCRIPT FILE must be exactly REVIEW_PASSED or REVIEW_FAILED, with nothing after it -- no sign-off, no timestamp, no closing sentence. swarm-verdict.sh reads only that final line; a marker followed by any trailing text is read as UNCLEAR and your entire review is discarded. + # LIVENESS CHECK ONLY -- deliberately weaker than the contract above. + # It asks "did this lens emit a verdict at all", nothing more. The + # binding check is the aggregate step, which reads the LAST NON-EMPTY + # LINE of the transcript file; a marker that is merely present but not + # last is UNCLEAR and fails the run. The kernel offers only + # exit_code / output_contains / json_schema, none of which can express + # "last line of this file equals this string", so passing here is not + # evidence the contract was met. verification: type: output_contains value: "REVIEW_" @@ -105,7 +120,15 @@ workflows: Does the commit message tell the truth about the diff? Write your complete review to ops/reviews/$(date +%Y%m%d-%H%M)-pr$(cat .review-target/pr-number)-history.md and - `git add` it. End your output with REVIEW_PASSED or REVIEW_FAILED. + `git add` it. The LAST NON-EMPTY LINE OF THE TRANSCRIPT FILE must be exactly REVIEW_PASSED or REVIEW_FAILED, with nothing after it -- no sign-off, no timestamp, no closing sentence. swarm-verdict.sh reads only that final line; a marker followed by any trailing text is read as UNCLEAR and your entire review is discarded. + # LIVENESS CHECK ONLY -- deliberately weaker than the contract above. + # It asks "did this lens emit a verdict at all", nothing more. The + # binding check is the aggregate step, which reads the LAST NON-EMPTY + # LINE of the transcript file; a marker that is merely present but not + # last is UNCLEAR and fails the run. The kernel offers only + # exit_code / output_contains / json_schema, none of which can express + # "last line of this file equals this string", so passing here is not + # evidence the contract was met. verification: type: output_contains value: "REVIEW_" @@ -125,7 +148,15 @@ workflows: a primitive instead of a helper, or grows a file past its purpose. Write your complete review to ops/reviews/$(date +%Y%m%d-%H%M)-pr$(cat .review-target/pr-number)-structure.md and - `git add` it. End your output with REVIEW_PASSED or REVIEW_FAILED. + `git add` it. The LAST NON-EMPTY LINE OF THE TRANSCRIPT FILE must be exactly REVIEW_PASSED or REVIEW_FAILED, with nothing after it -- no sign-off, no timestamp, no closing sentence. swarm-verdict.sh reads only that final line; a marker followed by any trailing text is read as UNCLEAR and your entire review is discarded. + # LIVENESS CHECK ONLY -- deliberately weaker than the contract above. + # It asks "did this lens emit a verdict at all", nothing more. The + # binding check is the aggregate step, which reads the LAST NON-EMPTY + # LINE of the transcript file; a marker that is merely present but not + # last is UNCLEAR and fails the run. The kernel offers only + # exit_code / output_contains / json_schema, none of which can express + # "last line of this file equals this string", so passing here is not + # evidence the contract was met. verification: type: output_contains value: "REVIEW_"