Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions .github/workflows/review-swarm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: Review swarm

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]

permissions:
contents: read
pull-requests: write

concurrency:
group: review-swarm-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
review:
# Ordering invariant: 75-minute job > 65-minute poll > 60-minute swarm.
timeout-minutes: 75
runs-on: ubuntu-latest
steps:
- name: Check out immutable gate
uses: actions/checkout@v4
with:
ref: main
path: gate

- name: Check out reviewed revision
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
path: target
fetch-depth: 0

- name: Validate workspace secret
env:
RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_WORKSPACE_KEY }}
run: |
if [ -z "${RELAY_WORKSPACE_KEY:-}" ]; then
echo "RELAY_WORKSPACE_KEY secret not configured; see README § Cloud review swarm" >&2
exit 1
fi

- name: Install Agent Relay
run: npm install --global agent-relay

- name: Prepare review input
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: gate/.github/workflows/scripts/swarm-prepare.sh target gate

- name: Launch swarm
id: launch
working-directory: target
env:
RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_WORKSPACE_KEY }}
run: |
response=$(agent-relay cloud run --sync-code ../gate/workflows/review-swarm.yaml --json)
run_id=$(jq -er '.runId // .run_id // .id' <<<"$response")
echo "run_id=$run_id" >> "$GITHUB_OUTPUT"

- name: Wait for swarm
id: wait
if: steps.launch.outputs.run_id != ''
env:
RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_WORKSPACE_KEY }}
RUN_ID: ${{ steps.launch.outputs.run_id }}
run: |
# Ordering invariant: 3900s poll > workflow timeoutMs 3600000ms.
deadline=$((SECONDS + 3900))
swarm_status=timed_out
while [ "$SECONDS" -lt "$deadline" ]; do
response=$(agent-relay cloud status "$RUN_ID" --json) || { sleep 10; continue; }
status=$(jq -r '.status // .run.status // empty' <<<"$response")
case "$status" in
completed|failed|cancelled)
swarm_status=$status
break
;;
esac
sleep 10
done
echo "swarm_status=$swarm_status" >> "$GITHUB_OUTPUT"
exit 0

- name: Post swarm evidence
if: always() && steps.launch.outputs.run_id != ''
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ steps.launch.outputs.run_id }}
SWARM_STATUS: ${{ steps.wait.outputs.swarm_status }}
RELAY_WORKSPACE_KEY: ${{ secrets.RELAY_WORKSPACE_KEY }}
run: gate/.github/workflows/scripts/swarm-post.sh post target

- name: Enforce swarm result
if: steps.wait.outputs.swarm_status != 'completed'
env:
SWARM_STATUS: ${{ steps.wait.outputs.swarm_status }}
run: |
echo "Review swarm did not complete successfully: ${SWARM_STATUS:-not_launched}" >&2
exit 1
80 changes: 80 additions & 0 deletions .github/workflows/scripts/swarm-post.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env bash
set -euo pipefail

lenses=(maintainability history structure)

latest_transcript() {
local root=$1 pr=$2 lens=$3
find "$root/ops/reviews" -maxdepth 1 -type f \
-name "*-pr${pr}-${lens}.md" -printf '%f\n' 2>/dev/null | sort | tail -n 1
}

transcript_verdict() {
local file=$1 started=$2 last
[ -n "$file" ] && [ -f "$file" ] || { printf 'MISSING\n'; return; }
[ "$(stat -c %Y "$file")" -ge "$started" ] || { printf 'STALE\n'; return; }
last=$(sed '/^[[:space:]]*$/d' "$file" | tail -n 1)
case "$last" in
*REVIEW_PASSED) printf 'PASSED\n' ;;
*REVIEW_FAILED) printf 'FAILED\n' ;;
*) printf 'UNCLEAR\n' ;;
esac
}

evaluate() {
local root=$1 pr started lens name file verdict overall=PASSED
pr=$(tr -d '[:space:]' < "$root/.review-target/pr-number")
started=$(tr -d '[:space:]' < "$root/.review-target/sync-started")
for lens in "${lenses[@]}"; do
name=$(latest_transcript "$root" "$pr" "$lens")
file=${name:+$root/ops/reviews/$name}
verdict=$(transcript_verdict "$file" "$started")
printf '%s\t%s\t%s\n' "$lens" "$verdict" "$file"
[ "$verdict" = PASSED ] || overall=FAILED
done
printf 'overall\t%s\n' "$overall"
[ "$overall" = PASSED ]
}

upsert_comment() {
local anchor=$1 body=$2 id
id=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \
--jq ".[] | select(.body | contains(\"$anchor\")) | .id" | sed -n '1p')
if [ -n "$id" ]; then
gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -f body="$body" >/dev/null
else
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -f body="$body" >/dev/null
fi
}

post() {
local root=$1 report rc lens verdict file anchor body overall sync_rc=0
agent-relay cloud sync "$RUN_ID" --dir "$root" || sync_rc=$?
report=$(mktemp)
rc=$sync_rc
evaluate "$root" > "$report" || rc=$?
for lens in "${lenses[@]}"; do
IFS=$'\t' read -r _ verdict file < <(awk -F '\t' -v lens="$lens" '$1 == lens { print; exit }' "$report")
anchor="<!-- swarm-lens: $lens -->"
if [ "$verdict" = PASSED ] || [ "$verdict" = FAILED ]; then
body=$(printf '%s\n\n%s' "$anchor" "$(cat "$file")")
else
body=$(printf '%s\n\nReview evidence is %s for cloud run `%s`.' "$anchor" "$verdict" "$RUN_ID")
fi
upsert_comment "$anchor" "$body"
done
overall=$(awk -F '\t' '$1 == "overall" { print $2 }' "$report")
[ "${SWARM_STATUS:-}" = completed ] || overall=FAILED
anchor='<!-- review-swarm: marker -->'
body=$(printf '%s\n\nReview swarm **%s** for cloud run `%s` (terminal status: `%s`).' \
"$anchor" "$overall" "$RUN_ID" "${SWARM_STATUS:-unknown}")
upsert_comment "$anchor" "$body"
rm -f "$report"
return "$rc"
}

case "${1:-}" in
verdict) evaluate "${2:-.}" ;;
post) post "${2:?usage: swarm-post.sh post TARGET_DIR}" ;;
*) echo "usage: swarm-post.sh verdict [ROOT] | post TARGET_DIR" >&2; exit 2 ;;
esac
20 changes: 20 additions & 0 deletions .github/workflows/scripts/swarm-prepare.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail

target=${1:?usage: swarm-prepare.sh TARGET_DIR GATE_DIR}
gate=${2:?usage: swarm-prepare.sh TARGET_DIR GATE_DIR}
: "${PR_NUMBER:?PR_NUMBER is required}"

case "$PR_NUMBER" in
*[!0-9]*|'') echo "PR_NUMBER must be numeric" >&2; exit 1 ;;
esac

mkdir -p "$target/.review-target" "$target/.review-gate"
printf '%s\n' "$PR_NUMBER" > "$target/.review-target/pr-number"
gh pr diff "$PR_NUMBER" > "$target/.review-target/pr.diff"
gh pr view "$PR_NUMBER" --json headRefName,headRefOid,title,url > "$target/.review-target/pr.json"
date +%s > "$target/.review-target/sync-started"
cp "$gate/.github/workflows/scripts/swarm-post.sh" "$target/.review-gate/swarm-post.sh"
chmod +x "$target/.review-gate/swarm-post.sh"

git -C "$target" add -f .review-target .review-gate/swarm-post.sh
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ dist/
.env
.agentworkforce/
.cargo-home/
.review-target

# Toolchains materialize inside the workspace in a cloud sandbox and must never
# be committed or delivered. Run f18ec684's patch carried .rustup-home/ files;
Expand Down
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,11 @@ Nine gates, in `docs/RFC-0001` §3. Gate 1 first: a relayflow can run — the he
ladder survives `kill -9` at every boundary.

Private while we build. YC 2026-09-15 runs on this base.

## Cloud review swarm

The `Review swarm` GitHub Actions workflow requires a repository Actions secret
named `RELAY_WORKSPACE_KEY`. Obtain the key for the canonical workspace with
`agent-relay workspace key`, then add it under **Settings → Secrets and
variables → Actions → New repository secret**. The workflow fails before cloud
launch when the secret is absent; it never falls back to interactive login.
11 changes: 11 additions & 0 deletions ops/NEEDS_HUMAN.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Build sandbox Git metadata is unavailable

The Track D implementation and its parse/syntax/SDK test commands pass, but the
definition of done cannot complete because this workspace's `.git` file points
to `/home/daytona/.project-git`, which does not exist. The repository is private
and this sandbox has no GitHub credentials, so the missing object store cannot
be reconstructed safely from `origin`.

Human/platform action: provide the worktree's Git directory at the path named by
`.git` (or seed the sandbox with a valid repository), then rerun
`git status --porcelain` from the repository root.
104 changes: 47 additions & 57 deletions ops/NEXT.md
Original file line number Diff line number Diff line change
@@ -1,87 +1,77 @@
# NEXT — work package for this tick

**Scope:** Build a minimal agent worker in the SDK. CODE task, SDK-side.
**Scope:** Build `.github/workflows/review-swarm.yml` correctly, addressing every architectural finding from the walked-away #75/#77 attempts. Parallel to Track A (hn-monitor); different territory (`.github/` + `workflows/` — no overlap with `sdk/` work).

This run is pinned to **gate 3** and must not work on any other gate.
This run is pinned to **gate 3, Track D** (Cloud review-swarm redesign).

## Objective

Promote the throwaway worker the tests already build into a real SDK component
that can execute agent steps by running their declared CLI as a subprocess.
Implement cloud-based review swarm automation that enforces RFC-0001 §2 rule 7 ("every PR met by a review swarm — our own, not a vendor's") by creating the GitHub Actions workflow and supporting scripts that address all 9 non-negotiable requirements from prior failed attempts (#75, #77).

## Context
The local `~/AgentWorkforce/review-swarm-loop.sh` works but lives on Khaliq's laptop. When the session ends, so does swarm enforcement. The cloud version must exist for gate 3+ work to be trustworthy.

Nothing in this repo can execute an agent step. Searching for `workerAttach` /
`step.complete` finds only TESTS (`sdk/tests/live-kernel.test.ts`,
`journal-client.test.ts`, `journal-client-loopback.ts`) and the protocol
definitions. `sdk/src/cli/run.ts` only OBSERVES worker leases and waits for one
that never arrives.
## Context from TARGET.md

The kernel's dispatch, lease and claim machinery is real and tested. The worker
side of the protocol is simply unimplemented, and that is what blocks gate 2
("a workload RUNS as a relayflow" — today a run can only be shown CREATED) and
gate 3 ("every claim/lease/retry served by the kernel").
Prior attempts (#75, #77) each shipped real code but were rejected on progressively deeper architectural findings:
1. Immutable gate violation (PR could control its own judge)
2. Duplicate verdict logic that could disagree
3. No auth secret validation (failed runs with no clear error)
4. Non-sticky comments (5 pushes → 20 comments instead of 4 edited)
5. Author whitelists (violates "every PR" requirement)
6. Cloud sandbox fetch failure (no `gh` auth in sandbox)
7. Timeout invariant violations
8. Transcript loss on rejecting swarms
9. Stale transcript binding

`sdk/tests/live-kernel.test.ts` around the `live-manual-agent` case (line 288)
shows the whole shape: connect, `hello`, `workerAttach` with pins, receive
`step.dispatch`, act, complete. The protocol is already proven there.
Every one was a legitimate swarm rejection. Address them or don't ship.

## Files in scope

- `sdk/src/worker.ts` — new file, the worker implementation
- `sdk/src/index.ts` — export the worker
- `sdk/tests/live-kernel.test.ts` OR a new test file — add a test that runs a
real flow with an agent step end to end against a live `relayflowd`, with
this worker attached, and asserts the step reaches `done`.
- `.github/workflows/review-swarm.yml` — the GHA trigger (NEW)
- `.github/workflows/scripts/swarm-post.sh` — sync + verdict + post script (NEW)
- `.github/workflows/scripts/swarm-prepare.sh` — launcher-side fetcher (NEW)
- `workflows/review-swarm.yaml` — refactor aggregate step to share verdict logic
- `.gitignore` — drop the `.review-target` mask
- `README.md` — document `RELAY_WORKSPACE_KEY` secret + how to obtain

## Definition of done

ALL of the following must hold:

1. The worker in `sdk/src/worker.ts`, exported from `sdk/src/index.ts`
1. All files parse:
```
python3 -c "import yaml; yaml.safe_load(open('.github/workflows/review-swarm.yml'))"
python3 -c "import yaml; yaml.safe_load(open('workflows/review-swarm.yaml'))"
bash -n .github/workflows/scripts/swarm-post.sh
bash -n .github/workflows/scripts/swarm-prepare.sh
```

2. A test that runs a real flow with an agent step end to end against a live
`relayflowd`, with this worker attached, and asserts the step reaches
`done`. `sdk/tests/live-kernel.test.ts` already starts a daemon — follow
that pattern.
2. Aggregate verdict logic exists in ONE file, both callers use it

3. **The worker must attach BEFORE the run starts.** A run that finds no worker
parks, and attaching afterwards does not re-drive it — `run.resume` is what
picks a parked run back up. That contract is pinned in the live-kernel
suite; do not fight it.
3. Author whitelist absent (no `if: github.event.pull_request.user.login == ...`)

4. The worker must:
- attach for `agent` steps with the pins it holds
- on `step.dispatch`, run the step's declared `cli` as a subprocess
- report the result back through the existing protocol (`step.complete`, and
the failure path when the CLI exits nonzero)
- nothing speculative: no retries of its own, no scheduling, no LLM calls.
The kernel owns retry and lease policy — do not reimplement it.
4. Immutable gate: two checkout steps with different paths

5. `cd sdk && npm test` must be green. Run it and paste the literal command and
output tail showing test counts.
5. PR body explicitly documents each of the 9 requirements above and shows where each is satisfied

6. `cd kernel && sh ../ops/cargo.sh test` must be green. Run it and paste the
literal command and output tail showing test counts.
6. SDK tests still pass:
```
cd sdk && npm test
```

7. EVERY new test confirmed to FAIL against current code, with the literal
failing output quoted in the summary.

8. As your LAST action, run `git status --porcelain` and paste it.
7. As final action:
```
git status --porcelain
```

## Explicitly OUT of scope

- LLM steps — not in the gate 3 scope
- Retry logic in the worker — the kernel owns retry policy
- Scheduling or lease management — the kernel owns lease policy
- Optimizations, abstractions, or speculative features
- Changes to the kernel
- Changes to existing tests (except adding new test cases)
- Work on any gate other than gate 3
- `sdk/` (Track A owns that)
- `kernel/` (gate 1 done, no changes)
- `ops/*` (chief owns briefs and state)
- Any GHA workflow other than review-swarm.yml
- Actually TESTING the workflow in CI (requires `RELAY_WORKSPACE_KEY` secret set which is a human step; the DoD is the workflow being correct, not proven live)

## If blocked

If gate 3 is genuinely unreachable from the current state, write
ops/NEEDS_HUMAN.md saying exactly why and still end with ASSESS_DONE. Do not
silently substitute different work: a run that reports progress on the wrong
gate is worse than one that reports it is blocked.
If gate 3 Track D is genuinely unreachable from the current state, write ops/NEEDS_HUMAN.md saying exactly why and still end with ASSESS_DONE. Do not silently substitute different work: a run that reports progress on the wrong gate is worse than one that reports it is blocked.
Loading
Loading