From 474b260cd2657bead82bf8b42be0a5d63cc3541e Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Sat, 29 Aug 2026 17:04:51 -0400 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20build=20outside=20the=20propagated?= =?UTF-8?q?=20tree=20=E2=80=94=20this=20is=20the=20silent=20file=20loss?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of runs that complete and deliver nothing. kernel/target/debug is ~4900 files; with it inside the propagated tree the sandbox's relayfile mount flush is rejected as too large: relayfile flush failed after the command succeeded (exit 0); a later agent step may see stale files: Failed to flush relayfile mount: notify flush: daemon pid 345 flush failed: http 413 payload... The flush failure is NON-FATAL, so the workflow reports success while later steps read stale files and the delivered patch loses the run's real work. That is the symptom that has been unexplained since this morning: a build log quoting a diff verbatim while the delivered patch contains none of it. Evidence — three runs that lost their work against one that did not: fdb49a9c 4127 changed files 3 flush failures 3x http 413 work lost ad98c2c3 4175 changed files 3 flush failures 3x http 413 work lost 52fa0752 3247 changed files 3 flush failures 3x http 413 work lost 76a4a8d1 489 changed files 0 flush failures 0x http 413 (failed for an unrelated reason) Of 52fa0752's changed paths, 4914 matched target/debug and 1 matched node_modules, so this one directory is the whole of it. ops/cargo.sh already kept CARGO_HOME and RUSTUP_HOME outside the tree for a related reason. This extends that to CARGO_TARGET_DIR, and exports RELAYFLOWD_BIN so anything reading the binary follows the build. The test-side resolution had to move with it. Without that change the live-kernel cases do not fail — they SKIP, 8 of them, which reports a false green. Resolution now prefers the toolchain target and falls back to the old in-repo path, so it works in both layouts. Measured effect on the working tree: kernel/target files in tree: ~4900 -> 0 total files in tree: 1550 (1226 of them node_modules) Residual, stated rather than hidden: node_modules is now the largest remaining contributor at 1226 files. 1550 is well under the 3247 that triggered a 413 and above the 489 that did not, so this should clear it, but I have not proven a cloud run passes — that needs the next tick to deliver. Verified: sdk 184 passed (13 files), kernel 0 failures, tsc clean. Co-Authored-By: Claude Fable 5 --- ops/cargo.sh | 20 ++++++++++++++++++++ sdk/tests/live-kernel.test.ts | 20 ++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ops/cargo.sh b/ops/cargo.sh index 409e43a2d..83b6f1e6b 100755 --- a/ops/cargo.sh +++ b/ops/cargo.sh @@ -24,6 +24,26 @@ toolchain_home="${RELAYFLOWS_TOOLCHAIN_HOME:-$HOME/.relayflows-toolchain}" export CARGO_HOME="$toolchain_home/cargo" export RUSTUP_HOME="$toolchain_home/rustup" +# Build OUTPUT must stay out of the propagated tree too, for a different and +# sharper reason than the toolchain above. +# +# `kernel/target/debug` is about 4900 files. With it inside the tree, the +# relayfile mount flush is rejected as too large and the run keeps going: +# relayfile flush failed after the command succeeded (exit 0); a later agent +# step may see stale files: ... notify flush: ... http 413 payload... +# The failure is non-fatal, so the workflow reports success while later steps +# read stale files and the delivered patch silently loses the run's real work. +# Three runs lost their work exactly this way (fdb49a9c 4127 changed files, +# ad98c2c3 4175, 52fa0752 3247 — three 413s each), against one that did not +# (76a4a8d1, 489 files, no flush failure). Of 52fa0752's changed paths, 4914 +# matched target/debug and 1 matched node_modules, so this directory is the +# whole of the problem. +# +# Anything that hardcodes kernel/target/debug must read RELAYFLOWD_BIN instead; +# sdk/tests/live-kernel.test.ts already does. +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$toolchain_home/target}" +export RELAYFLOWD_BIN="${RELAYFLOWD_BIN:-$CARGO_TARGET_DIR/debug/relayflowd}" + if command -v cargo >/dev/null 2>&1; then cargo_bin=cargo elif [ -x "${CARGO_INSTALL_ROOT:-}/bin/cargo" ]; then diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index 2f2b0777b..d4c38cdbc 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -9,7 +9,7 @@ import { writeFileSync, readFileSync, } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import { fileURLToPath } from 'node:url'; @@ -22,9 +22,25 @@ const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const SDK = join(ROOT, 'sdk'); const BUILT_CLI = join(SDK, 'dist', 'cli.js'); const TESTDATA = join(ROOT, 'testdata'); +// ops/cargo.sh builds into a target dir OUTSIDE the repo, because +// kernel/target/debug is ~4900 files and its presence in the propagated tree +// makes the sandbox's relayfile flush fail with HTTP 413 — non-fatally, so runs +// silently lose their work. Resolution has to follow the build, or these cases +// SKIP rather than fail and the suite reports a false green. +const TOOLCHAIN_TARGET = + process.env['CARGO_TARGET_DIR'] ?? + join(process.env['RELAYFLOWS_TOOLCHAIN_HOME'] ?? join(homedir(), '.relayflows-toolchain'), 'target'); const RELAYFLOWD = resolve( - process.env['RELAYFLOWD_BIN'] ?? join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'), + process.env['RELAYFLOWD_BIN'] ?? + firstExisting([ + join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'), + join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'), + ]), ); + +function firstExisting(candidates: string[]): string { + return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0]!; +} const temporaryDirectories: string[] = []; const daemons: ChildProcess[] = []; const clients: JournalClient[] = []; From a16c4c956af2c24eb2e6e6b6c655f0bfca9254b5 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Sat, 29 Aug 2026 17:14:35 -0400 Subject: [PATCH 2/2] fix: key the target dir per worktree, and make resolution find it (PR #38 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review was right: a single shared target dir is used by every worktree under the same HOME. Cargo locks it so builds stay safe, but two different source trees sharing one target thrash each other's artifacts and serialise behind the lock. Keyed now by a hash of the worktree's own path, so isolation needs no bookkeeping. The test-side resolution had to follow, and getting there exposed something worse than the skip this PR already fixed. With the target keyed but resolution still pointing at the unkeyed path, the suite reported 8 PASSED — against a stale binary left at the old location by an earlier build. A false green that exercises the wrong artifact is harder to notice than a skip, and I only caught it by deleting the stale binary and watching the 8 turn into skips. Resolution now finds the keyed directory without duplicating the hash: exact path first, then the most recently built /debug/relayflowd beneath the target root, then the old in-repo path. Verified: worktree keys distinct: /tmp/manual-work 3621016306, /tmp/flows-ops 3447627430 resolved binary: .relayflows-toolchain/target/1803092378/debug/relayflowd kernel/target files in tree: 0 sdk 184 passed (13 files), kernel 0 failures, tsc clean Co-Authored-By: Claude Fable 5 --- ops/cargo.sh | 9 ++++++++- sdk/tests/live-kernel.test.ts | 34 ++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/ops/cargo.sh b/ops/cargo.sh index 83b6f1e6b..18018e8e7 100755 --- a/ops/cargo.sh +++ b/ops/cargo.sh @@ -41,7 +41,14 @@ export RUSTUP_HOME="$toolchain_home/rustup" # # Anything that hardcodes kernel/target/debug must read RELAYFLOWD_BIN instead; # sdk/tests/live-kernel.test.ts already does. -export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$toolchain_home/target}" +# Keyed per worktree. Review caught that a single shared target dir would be +# used by every worktree under the same HOME (PR #38): cargo locks it, so the +# builds are safe, but two different source trees sharing one target thrash +# each other's artifacts and serialise behind the lock. The key is a hash of +# the worktree's own path, so isolation is automatic and needs no bookkeeping. +_worktree_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) +_worktree_key=$(printf '%s' "$_worktree_root" | cksum | cut -d' ' -f1) +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$toolchain_home/target/$_worktree_key}" export RELAYFLOWD_BIN="${RELAYFLOWD_BIN:-$CARGO_TARGET_DIR/debug/relayflowd}" if command -v cargo >/dev/null 2>&1; then diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index d4c38cdbc..47eed6c2b 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -30,16 +30,30 @@ const TESTDATA = join(ROOT, 'testdata'); const TOOLCHAIN_TARGET = process.env['CARGO_TARGET_DIR'] ?? join(process.env['RELAYFLOWS_TOOLCHAIN_HOME'] ?? join(homedir(), '.relayflows-toolchain'), 'target'); -const RELAYFLOWD = resolve( - process.env['RELAYFLOWD_BIN'] ?? - firstExisting([ - join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'), - join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'), - ]), -); - -function firstExisting(candidates: string[]): string { - return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0]!; +const RELAYFLOWD = resolve(process.env['RELAYFLOWD_BIN'] ?? locateRelayflowd()); + +/** + * ops/cargo.sh builds into a target dir outside the repo, keyed per worktree so + * concurrent worktrees do not share one target. Resolution has to find that + * key without duplicating the hash, or these cases SKIP instead of failing and + * the suite reports a false green — and worse, a stale binary left at an older + * path gets exercised in place of the one just built. + */ +function locateRelayflowd(): string { + const direct = join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'); + if (existsSync(direct)) return direct; + + // One level down: $toolchain/target//debug/relayflowd. Pick the + // most recently built, which is the one this checkout just produced. + const keyed = existsSync(TOOLCHAIN_TARGET) + ? readdirSync(TOOLCHAIN_TARGET) + .map((entry) => join(TOOLCHAIN_TARGET, entry, 'debug', 'relayflowd')) + .filter((candidate) => existsSync(candidate)) + .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs) + : []; + if (keyed[0] !== undefined) return keyed[0]; + + return join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'); } const temporaryDirectories: string[] = []; const daemons: ChildProcess[] = [];