diff --git a/.gitignore b/.gitignore index d249b39fd..95c9bf17c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ dist/ # Legacy drive-local snapshots; the preparing launcher no longer executes them. .drive-gate/ +.flows/build.key diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 184a2e3f7..f24f25b1d 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -448,6 +448,35 @@ The herdr model: first-party helpers are just plugins that ship in the box; the `flows build` seals a flow into a content-addressed, immutable bundle: canonical spec JSON, compiled TS with pinned deps, helper/plugin lockfile, assets, preflight declaration, identity signature — `flow@sha256:…`, pushed to a bucket/registry. `flows deploy` points a trigger at a digest; `flows run flow@sha256:…` executes from the bucket on any cell, no checkout. Preflight runs at build time for everything build-provable and again at deploy time for environment facts (credentials, workers, MCP servers). The working tree is for authoring; **production only ever runs digests.** +The local build and verification commands are available now: + +```text +flows build [--out ] +flows build --verify +``` + +Output defaults to `dist/flows/@sha256:/`. The canonical manifest +hashes the payload files; `manifest.json` and `identity.json` are excluded from +its entries to avoid circular hashing. A stable signing seed gives identical +identity bytes across builds. Ephemeral keys preserve the payload digest but +produce different identity signatures. An existing valid bundle is reused. +Verification refuses changed, missing, unlisted, or symlinked files with exit 2. + +TypeScript builds require Bun on PATH and an installed npm workspace matching +its `package-lock.json`. The full lockfile is retained in this first slice. +A TS module can default-export a declarative spec, or default-export `flow()` +with an additional exported `spec` declaration for build-time checks. The +authored body is retained in the executable and is not run during build; +nonempty authored headers are currently refused. Module initialization must +be deterministic. `preflight.json` preserves the preflight report, including +uncollected environment facts; `metadata.json` separately records the platform, +compiler, executable asset paths, and checks deferred until deployment. + +Signing uses `FLOWS_BUILD_KEY` or the repository's `.flows/build.key`, each a +base64 32-byte Ed25519 seed. Without either, stderr reports +`identity_ephemeral: bundle can be verified but not attributed`. +Deployment, remote upload, and execution by digest remain future slices. + ## 5. Invocation: the gate-1 CLI Gate 1 ships three CLI verbs over the journal protocol, plus one out-of-band diff --git a/packages/sdk/src/bundle-typescript.ts b/packages/sdk/src/bundle-typescript.ts new file mode 100644 index 000000000..8164c1a07 --- /dev/null +++ b/packages/sdk/src/bundle-typescript.ts @@ -0,0 +1,98 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { canonicalize } from './canonical.js'; +import { compileSpec } from './compile.js'; +import type { BundleFile } from './bundle.js'; + +/** Bun is an explicit build dependency, never an implicit install. */ +function bun(args: string[], cwd: string): string { + const result = spawnSync('bun', ['--no-env-file', ...args], { + cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024, + env: { PATH: process.env['PATH'], NODE_ENV: 'production', TZ: 'UTC' }, + }); + if (result.error || result.status !== 0) { + throw new Error(`TypeScript build requires Bun: ${result.error?.message ?? result.stderr}`); + } + return result.stdout.trim(); +} + +export async function buildTypescript(input: string) { + const directory = dirname(input); + const lockPath = await findLock(directory); + const lock = JSON.parse(await readFile(lockPath, 'utf8')); + if (![2, 3].includes(lock.lockfileVersion) || !lock.packages) { + throw new Error('package-lock.json: expected npm lockfile version 2 or 3 with pinned packages'); + } + await checkInstalledVersions(dirname(lockPath), lock.packages); + const compiler = `bun@${bun(['--version'], directory)}`; + const staging = await mkdtemp(join(tmpdir(), 'flows-ts-')); + try { + // Evaluate module declarations, never the authored body. Resolve the runtime + // beside the input, keeping flow()'s WeakMap identity in the same module. + const probe = ` +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +const module = await import(${JSON.stringify(pathToFileURL(input).href)}); +let spec = module.default; +let authored = false; +if (!spec || !Array.isArray(spec.steps)) { + const require = createRequire(${JSON.stringify(pathToFileURL(input).href)}); + const { getFlowDefinition } = await import(pathToFileURL(require.resolve('@relayflows/surface/runtime')).href); + const definition = getFlowDefinition(spec); + if (Object.keys(definition.header).length) throw new Error('unsupported authored flow header'); + if (!module.spec) throw new Error('authored flow() requires an exported spec declaration for build-time preflight; the body is not executed during build'); + spec = module.spec; + if (spec.name !== definition.name) throw new Error('exported spec name must match flow() name'); + authored = true; +} + +process.stdout.write(JSON.stringify({ spec, authored })); +`; + const probePath = join(staging, 'inspect.ts'); + await writeFile(probePath, probe); + const inspected = JSON.parse(bun([probePath], directory)); + const spec = compileSpec(inspected.spec); + const executable = join(staging, 'flow'); + bun(['build', '--compile', '--env=disable', '--no-compile-autoload-dotenv', + '--no-compile-autoload-bunfig', '--outfile', executable, input], directory); + const files: BundleFile[] = [ + { path: 'flow', data: await readFile(executable) }, + { path: 'lockfile.json', data: canonicalize(lock) }, + ]; + return { spec, authored: inspected.authored === true, compiler, files }; + } finally { await rm(staging, { recursive: true, force: true }); } +} + +/** Do not label a stale node_modules tree with a newer lockfile's pins. */ +async function checkInstalledVersions(root: string, packages: Record): Promise { + for (const [path, entry] of Object.entries(packages)) { + if (path === '') continue; + if (path.startsWith('/') || path.includes('\\') || path.split('/').includes('..') + || (entry.resolved !== undefined && /^(?:file:|\/|[A-Za-z]:)/.test(entry.resolved))) { + throw new Error(`package-lock.json: ${path} must use portable, pinned dependency locations`); + } + if (entry.link) continue; // Workspace source is captured by the compiler. + let installed; + try { installed = JSON.parse(await readFile(join(root, path, 'package.json'), 'utf8')); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT' && (entry.optional || entry.dev)) continue; + throw new Error(`package-lock.json: ${path} is missing; run npm ci before building`); + } + if (typeof entry.version !== 'string' || installed.version !== entry.version) { + throw new Error(`package-lock.json: ${path} does not match its pinned version; run npm ci before building`); + } + } +} +async function findLock(start: string): Promise { + for (let directory = start; ; directory = dirname(directory)) { + const candidate = join(directory, 'package-lock.json'); + try { await readFile(candidate); return candidate; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + if (dirname(directory) === directory) throw new Error('package-lock.json: no workspace lockfile found for TS flow'); + } +} diff --git a/packages/sdk/src/bundle.ts b/packages/sdk/src/bundle.ts new file mode 100644 index 000000000..73b28ef4b --- /dev/null +++ b/packages/sdk/src/bundle.ts @@ -0,0 +1,179 @@ +import { createHash, createPrivateKey, createPublicKey, randomBytes, sign, verify } from 'node:crypto'; +import { lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; +import { canonicalize } from './canonical.js'; + +export interface BundleEntry { path: string; sha256: string; bytes: number } +export interface BundleFile { path: string; data: Uint8Array | string; executable?: boolean } +export interface BundleOptions { + name: string; + out: string; + repo: string; + files: BundleFile[]; + env?: NodeJS.ProcessEnv; + warn(message: string): void; +} + +export function sha256(data: Uint8Array | string): string { + return createHash('sha256').update(data).digest('hex'); +} + +function safePath(path: string): boolean { + return path.length > 0 && !path.includes('\\') && !path.includes('\0') + && path.split('/').every(part => part !== '' && part !== '.' && part !== '..') + && !path.includes(':'); +} + +/** Manifest and identity are envelopes, excluded to avoid circular hashing. */ +export async function sealBundle(options: BundleOptions): Promise { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(options.name)) { + throw new Error('spec.canonical.json: name must be a safe single directory component'); + } + const files = options.files.map(file => ({ ...file, data: Buffer.from(file.data) })) + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); + const paths = new Set(); + for (const file of files) { + if (!safePath(file.path) || ['manifest.json', 'identity.json'].includes(file.path) || paths.has(file.path)) { + throw new Error(`${file.path}: invalid or duplicate bundle path`); + } + paths.add(file.path); + } + for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) { + if (!paths.has(required)) throw new Error(`${required}: missing bundle file`); + } + const manifest = canonicalize(files.map(file => ({ + path: file.path, sha256: sha256(file.data), bytes: file.data.length, + }))); + const digest = sha256(manifest); + const out = resolve(options.out); + const target = join(out, `${options.name}@sha256:${digest}`); + let exists = false; + try { await lstat(target); exists = true; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + if (exists) { await verifyBundle(target); return target; } + const identity = await signDigest(digest, options); + await mkdir(out, { recursive: true }); + const staging = await mkdtemp(join(out, '.build-')); + try { + for (const file of files) { + await mkdir(dirname(join(staging, file.path)), { recursive: true }); + await writeFile(join(staging, file.path), file.data, { mode: file.path === 'flow' || file.executable ? 0o755 : 0o644 }); + } + await writeFile(join(staging, 'manifest.json'), manifest); + await writeFile(join(staging, 'identity.json'), canonicalize(identity)); + try { await rename(staging, target); } + catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; + await verifyBundle(target); + } + return target; + } finally { await rm(staging, { recursive: true, force: true }); } +} + +async function signDigest(digest: string, options: BundleOptions) { + let seedText = (options.env ?? process.env)['FLOWS_BUILD_KEY']; + if (seedText === undefined) { + try { seedText = (await readFile(join(options.repo, '.flows/build.key'), 'utf8')).trim(); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + } + let seed: Buffer; + if (seedText === undefined) { + options.warn('identity_ephemeral: bundle can be verified but not attributed'); + seed = randomBytes(32); + } else { + seed = Buffer.from(seedText, 'base64'); + if (seed.length !== 32 || seed.toString('base64') !== seedText) { + throw new Error('build key: expected a base64-encoded 32-byte seed'); + } + } + const privateKey = createPrivateKey({ + key: Buffer.concat([Buffer.from('302e020100300506032b657004220420', 'hex'), seed]), + format: 'der', type: 'pkcs8', + }); + const publicKey = createPublicKey(privateKey).export({ format: 'der', type: 'spki' }).subarray(-32); + return { + algorithm: 'ed25519', keyid: sha256(publicKey).slice(0, 16), pubkey_b64: publicKey.toString('base64'), + signature_hex: sign(null, Buffer.from(digest, 'hex'), privateKey).toString('hex'), + }; +} + +async function regularFile(root: string, path: string): Promise { + try { + const parts = path.split('/'); + for (let i = 1; i <= parts.length; i++) { + const stat = await lstat(join(root, ...parts.slice(0, i))); + if (i === parts.length ? !stat.isFile() : !stat.isDirectory()) { + throw new Error('expected a regular file, without symlinks'); + } + } + return await readFile(join(root, path)); + } catch (error) { + throw new Error(`${path}: ${error instanceof Error ? error.message : 'unreadable'}`); + } +} + +/** Verify an untrusted directory without following symlinks or manifest traversal. */ +export async function verifyBundle(directory: string): Promise { + const root = resolve(directory); + if (!(await lstat(root)).isDirectory()) throw new Error('manifest.json: bundle must be a directory, not a symlink'); + const raw = (await regularFile(root, 'manifest.json')).toString('utf8'); + let manifest: BundleEntry[]; + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error('expected an array'); + manifest = parsed; + } catch { throw new Error('manifest.json: invalid manifest'); } + const paths = new Set(); + let previous = ''; + for (const entry of manifest) { + if (entry === null || typeof entry !== 'object' || typeof entry.path !== 'string' + || !safePath(entry.path) || entry.path <= previous + || ['manifest.json', 'identity.json'].includes(entry.path) + || Object.keys(entry).sort().join(',') !== 'bytes,path,sha256' + || !/^[a-f0-9]{64}$/.test(entry.sha256) || !Number.isSafeInteger(entry.bytes) || entry.bytes < 0) { + throw new Error('manifest.json: invalid, duplicate, or unsorted entry'); + } + previous = entry.path; + paths.add(entry.path); + const bytes = await regularFile(root, entry.path); + if (bytes.length !== entry.bytes || sha256(bytes) !== entry.sha256) { + throw new Error(`${entry.path}: sha256 or byte count mismatch`); + } + } + for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) { + if (!paths.has(required)) throw new Error(`${required}: missing manifest entry`); + } + const canonical = canonicalize(manifest); + if (raw !== canonical) throw new Error('manifest.json: noncanonical bytes'); + const digest = sha256(canonical); + if (!basename(root).endsWith(`@sha256:${digest}`)) throw new Error('manifest.json: directory digest mismatch'); + await rejectExtras(root, '', new Set([...paths, 'manifest.json', 'identity.json'])); + try { + const identity = JSON.parse((await regularFile(root, 'identity.json')).toString('utf8')); + const publicKey = Buffer.from(identity.pubkey_b64, 'base64'); + if (identity.algorithm !== 'ed25519' || publicKey.length !== 32 + || publicKey.toString('base64') !== identity.pubkey_b64 + || identity.keyid !== sha256(publicKey).slice(0, 16) + || !/^[a-f0-9]{128}$/.test(identity.signature_hex)) throw new Error('invalid identity'); + const key = createPublicKey({ + key: Buffer.concat([Buffer.from('302a300506032b6570032100', 'hex'), publicKey]), format: 'der', type: 'spki', + }); + if (!verify(null, Buffer.from(digest, 'hex'), key, Buffer.from(identity.signature_hex, 'hex'))) { + throw new Error('signature mismatch'); + } + } catch (error) { + throw new Error(`identity.json: ${error instanceof Error ? error.message : 'signature verification failed'}`); + } + return digest; +} + +async function rejectExtras(root: string, prefix: string, paths: Set): Promise { + for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { + const path = prefix + entry.name; + if (entry.isDirectory() && [...paths].some(file => file.startsWith(`${path}/`))) { + await rejectExtras(root, `${path}/`, paths); + } else if (!entry.isFile() || !paths.has(path)) { + throw new Error(`${path}: unlisted file or unsupported file type`); + } + } +} diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index b51382628..8db12ab7d 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -21,6 +21,7 @@ import { parseReplayArgs, replayJournal, type ReplayArgs } from './cli/replay.js import { checkTypeScriptFlow } from './cli/check-typescript.js'; import { runCloudCli } from './cli/cloud-run.js'; import { isAuthoredFlowPath } from './direct-input.js'; +import { parseBuildArgs, runBuild, type BuildArgs } from './cli/build.js'; import { runHnMonitor } from './cli/hn-monitor.js'; import { runTickRunner } from './cli/tick-runner.js'; import { @@ -39,6 +40,7 @@ export interface CliIo { type CliExitCode = 0 | 1 | 2 | 3; type ParsedArgs = | ReplayArgs + | BuildArgs | { command: 'cloud-run'; value: string; json: boolean; wait: boolean } | { command: 'check'; json: boolean; value: string } | { command: 'run'; reuseFromRunId: string | undefined; localAgent: boolean; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; value: string } @@ -52,6 +54,8 @@ type ParsedArgs = const DEFAULT_DATA_DIR = '.relayflowd'; const USAGE = [ 'Usage:', + 'flows build [--out ] ', + 'flows build --verify ', 'flows check [--json] ', 'flows run [--json] [--no-spawn] [--no-observer-link] [--data-dir ] [--local-agent] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] ', @@ -96,6 +100,7 @@ export async function runCli( if (parsed.command === 'cloud-run') return runCloudCli(parsed, io); if (parsed.command === 'replay') return replayJournal(parsed, io); + if (parsed.command === 'build') return runBuild(parsed, io); if (parsed.command === 'check') { // Deliberately daemon-free (kernel/DAEMON-LIFECYCLE.md §4). `checkFlow` is @@ -385,6 +390,7 @@ function emitWait( function parseArgs(args: readonly string[]): ParsedArgs | undefined { const command = args[0]; if (command === 'replay') return parseReplayArgs(args.slice(1)); + if (command === 'build') return parseBuildArgs(args.slice(1)); if (command === 'hn-monitor') return parseHnMonitorArgs(args.slice(1)); if (command === 'tick') return parseTickArgs(args.slice(1)); if (command === 'observer') return parseObserverArgs(args.slice(1)); diff --git a/packages/sdk/src/cli/build.ts b/packages/sdk/src/cli/build.ts new file mode 100644 index 000000000..e0defc4f8 --- /dev/null +++ b/packages/sdk/src/cli/build.ts @@ -0,0 +1,188 @@ +import { readFile, lstat } from 'node:fs/promises'; +import { dirname, extname, join, resolve } from 'node:path'; +import { parse } from 'yaml'; +import { sealBundle, verifyBundle, type BundleFile } from '../bundle.js'; +import { canonicalize } from '../canonical.js'; +import { compileSpec, toKernelSpec } from '../compile.js'; +import { preflight } from '../preflight.js'; +import { buildTypescript } from '../bundle-typescript.js'; +import { readProjectConfig } from './check.js'; +import type { CliIo } from '../cli.js'; +import type { FlowSpec } from '../spec.js'; + +export interface BuildArgs { command: 'build'; value: string; out?: string; verify: boolean } + +export function parseBuildArgs(args: readonly string[]): BuildArgs | undefined { + if (args[0] === '--verify') { + return args.length === 2 && !args[1]!.startsWith('-') + ? { command: 'build', value: args[1]!, verify: true } : undefined; + } + let out: string | undefined; + let value: string | undefined; + for (let i = 0; i < args.length; i++) { + const arg = args[i]!; + if (arg === '--out') { + if (out !== undefined || args[i + 1] === undefined || args[i + 1]!.startsWith('-')) return undefined; + out = args[++i]; + } else if (arg.startsWith('-') || value !== undefined) return undefined; + else value = arg; + } + return value === undefined ? undefined : { command: 'build', value, out, verify: false }; +} + +export async function runBuild(args: BuildArgs, io: CliIo): Promise<0 | 2> { + try { + if (args.verify) { + io.stdout(`VERIFIED sha256:${await verifyBundle(args.value)}`); + return 0; + } + io.stdout(await buildFlow(args.value, args.out ?? 'dist/flows', io.stderr)); + return 0; + } catch (error) { + io.stderr(`REFUSED [bundle_invalid] ${error instanceof Error ? error.message : String(error)}`); + return 2; + } +} + +/** Build checks never probe credentials, workers, or the author's PATH. */ +export async function buildFlow(path: string, out: string, warn: (line: string) => void): Promise { + const input = resolve(path); + const directory = dirname(input); + const files: BundleFile[] = []; + let authoring: FlowSpec; + let authored = false; + let compiler: string | undefined; + if (extname(input) === '.ts') { + const result = await buildTypescript(input); + authoring = result.spec; + authored = result.authored; + compiler = result.compiler; + files.push(...result.files); + } else if (['.yaml', '.yml'].includes(extname(input))) { + authoring = compileSpec(parse(await readFile(input, 'utf8'))); + files.push({ path: 'lockfile.json', data: canonicalize({ version: 1, adapters: [] }) }); + } else throw new Error('build expects a .yaml, .yml, or .ts flow'); + + // The current preflight API reports uncollected environment facts as + // probe_failed. Preserve that truthful report verbatim and separately declare + // the deployment obligations; never manufacture successful auth probes. + // + // Load the nearest flows.json so declared model allowlists and the project + // CLI participate in build-time preflight. Without them a declared `model:` + // that appears in flows.json/models refuses the whole build as + // `model_unknown`, because preflight defaults treat "no registry" as "no + // model is known" and only environment probes (deferred here) would rescue + // it. Model existence is a build-provable fact; only the live probe is not. + const config = readProjectConfig(directory); + const report = preflight(authoring, { + ...(config.cli !== undefined ? { projectCli: config.cli } : {}), + ...(config.path !== undefined ? { projectConfigPath: config.path } : {}), + projectSearchStart: directory, + models: config.models, + ...(config.path !== undefined ? { modelRegistryPath: config.path } : {}), + probes: { + cli: () => { throw new Error('deferred to deployment'); }, + executor: () => { throw new Error('deferred to deployment'); }, + command: () => { throw new Error('deferred to deployment'); }, + }, + }); + const refusals = report.diagnostics.filter(d => d.severity === 'refusal' && d.kind !== 'probe_failed'); + if (refusals.length > 0) throw new Error(refusals.map(d => `[${d.kind}] ${d.message}`).join('\n')); + authoring = await captureFiles(authoring, directory, files); + files.push( + { path: 'spec.canonical.json', data: canonicalize(toKernelSpec(authoring)) }, + { path: 'preflight.json', data: canonicalize(report) }, + { path: 'metadata.json', data: canonicalize({ + version: 1, platform: { os: process.platform, arch: process.arch }, + executables: files.filter(file => file.path === 'flow' || file.executable).map(file => file.path).sort(), + kind: authored ? 'authored-typescript' : 'declarative', + ...(compiler === undefined ? {} : { compiler }), + preflight: { buildPassed: true, deferred: ['commands', 'credentials', 'workers', 'mcp_servers'], + ...(authored ? { dynamicSteps: 'exported spec declaration checked; body was not executed during build' } : {}) }, + }) }, + ); + return sealBundle({ name: authoring.name ?? 'flow', out, files, + repo: await repositoryRoot(directory), warn }); +} + +async function repositoryRoot(start: string): Promise { + for (let dir = start; ; dir = dirname(dir)) { + try { await lstat(join(dir, '.git')); return dir; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + if (dirname(dir) === dir) return start; + } +} + +/** Relative CLI executables are the file references in the current spec schema. + * Shell commands are opaque; explicit ./file words are captured without + * interpreting substitutions or running a shell at build time. */ +async function captureFiles(flow: FlowSpec, directory: string, files: BundleFile[]): Promise { + // Cache the in-flight *promise*, not the resolved string. Two concurrent + // captures of the same path (Promise.all over named agents that share a CLI) + // both pass a `captured.has(path)` guard before either has finished awaiting + // its lstat/readFile, so both would push the same `assets/...` entry -- and + // sealBundle then refuses the whole flow as a duplicate. Storing the promise + // dedups on the first synchronous look-up. + const captured = new Map>(); + function capture(path: string): Promise { + if (!path.includes('/')) return Promise.resolve(path); + if (!path.startsWith('./') || path.split('/').includes('..') || path.includes('\\')) { + return Promise.reject(new Error(`${path}: bundle file references must start with ./ and stay inside the flow directory`)); + } + const existing = captured.get(path); + if (existing !== undefined) return existing; + const pending = (async () => { + const parts = path.slice(2).split('/'); + if (parts.some(p => p === '' || p === '.')) throw new Error(`${path}: invalid file reference`); + let executable = false; + for (let i = 1; i <= parts.length; i++) { + const stat = await lstat(join(directory, ...parts.slice(0, i))); + if (i === parts.length ? !stat.isFile() : !stat.isDirectory()) throw new Error(`${path}: expected regular file without symlinks`); + if (i === parts.length) executable = (stat.mode & 0o111) !== 0; + } + const target = `assets/${parts.join('/')}`; + files.push({ path: target, data: await readFile(join(directory, ...parts)), executable }); + return `./${target}`; + })(); + captured.set(path, pending); + return pending; + } + // Capture the flow-level `cli` default and every named-agent `cli` BEFORE + // walking steps. Otherwise a step that inherits its CLI from the flow header + // or an agents-map entry never emits its own `cli`, so the step-level walk + // below misses it, the executable never enters the bundle, and the sealed + // spec still points at the author's absolute path outside the bundle. + const flowCli = flow.cli !== undefined ? await capture(flow.cli) : undefined; + const agents: FlowSpec['agents'] = flow.agents === undefined ? undefined + : Object.fromEntries(await Promise.all(Object.entries(flow.agents).map( + async ([name, agent]) => [name, { ...agent, cli: await capture(agent.cli) }] as const, + ))); + const compiled = toKernelSpec(flow); + // Lower named/flow CLI resolution first so every runtime reference is bound. + const steps: FlowSpec['steps'] = []; + for (const [index, step] of flow.steps.entries()) { + if (step.type !== 'deterministic') { + const lowered = compiled.steps[index]; + const cli = lowered && lowered.type !== 'deterministic' ? lowered.cli : undefined; + steps.push(cli === undefined ? step : { ...step, cli: await capture(cli) }); + continue; + } + let command = step.command; + const references = [...command.matchAll(/(?:^|[\s;|&<>])(?:"(\.\/[^"$`]+)"|'(\.\/[^']+)'|(\.\/[^\s;|&<>"'$`]+))/g)]; + // Replace only each complete matched word, from right to left so offsets + // stay valid and similarly prefixed filenames cannot rewrite one another. + for (const match of references.reverse()) { + const ref = (match[1] ?? match[2] ?? match[3])!; + const start = match.index! + match[0].indexOf(ref); + command = command.slice(0, start) + await capture(ref) + command.slice(start + ref.length); + } + if (/^\s*["']?\//.test(command)) throw new Error(`${step.id}: absolute command paths cannot be bundled`); + steps.push({ ...step, command }); + } + return { + ...flow, + ...(flowCli !== undefined ? { cli: flowCli } : {}), + ...(agents !== undefined ? { agents } : {}), + steps, + }; +} diff --git a/packages/sdk/tests/bundle.test.ts b/packages/sdk/tests/bundle.test.ts new file mode 100644 index 000000000..832c85287 --- /dev/null +++ b/packages/sdk/tests/bundle.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { mkdtemp, readFile, readdir, writeFile, rm, mkdir, symlink, stat, chmod, rename } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { canonicalize } from '../src/canonical.js'; +import { sealBundle, sha256, verifyBundle } from '../src/bundle.js'; +import { buildFlow } from '../src/cli/build.js'; + +const sdk = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const repo = resolve(sdk, '../..'); +const cli = join(sdk, 'dist/cli.js'); +const temporary: string[] = []; +const key = Buffer.alloc(32, 7).toString('base64'); +async function temp() { + const path = await mkdtemp(join(tmpdir(), 'flows-bundle-test-')); + temporary.push(path); + return path; +} +function invoke(args: string[], cwd = repo, env: NodeJS.ProcessEnv = {}) { + return spawnSync(process.execPath, [cli, 'build', ...args], { + cwd, encoding: 'utf8', timeout: 120_000, + env: { PATH: process.env['PATH'], FLOWS_BUILD_KEY: key, ...env }, + }); +} +async function seal(out: string, env: NodeJS.ProcessEnv = { FLOWS_BUILD_KEY: key }, warn = (_: string) => {}) { + return sealBundle({ name: 'example', repo: out, out, env, warn, files: [ + { path: 'spec.canonical.json', data: canonicalize({ name: 'example' }) }, + { path: 'preflight.json', data: '{}' }, { path: 'lockfile.json', data: '{}' }, + ] }); +} +afterEach(async () => { await Promise.all(temporary.splice(0).map(path => rm(path, { force: true, recursive: true }))); }); + +describe('immutable bundles', () => { + it('is byte-for-byte deterministic with a stable identity and reuses existing bundles', async () => { + const a = await seal(await temp()); + const b = await seal(await temp()); + expect(basename(a)).toBe(basename(b)); + for (const file of await readdir(a)) expect(await readFile(join(a, file))).toEqual(await readFile(join(b, file))); + const manifest = await readFile(join(a, 'manifest.json'), 'utf8'); + expect(basename(a)).toBe(`example@sha256:${sha256(manifest)}`); + const before = await stat(join(a, 'identity.json')); + expect(await seal(dirname(a), {})).toBe(a); + expect((await stat(join(a, 'identity.json'))).mtimeMs).toBe(before.mtimeMs); + }); + + it('keeps the payload digest deterministic with ephemeral signatures and warns', async () => { + const warnings: string[] = []; + const a = await seal(await temp(), {}, warning => warnings.push(warning)); + const b = await seal(await temp(), {}); + expect(warnings).toEqual(['identity_ephemeral: bundle can be verified but not attributed']); + expect(basename(a)).toBe(basename(b)); + expect(await readFile(join(a, 'identity.json'))).not.toEqual(await readFile(join(b, 'identity.json'))); + expect(await verifyBundle(a)).toBe(await verifyBundle(b)); + }); + + it('uses the local seed and lets the environment take precedence', async () => { + const out = await temp(); + await mkdir(join(out, '.flows')); + await writeFile(join(out, '.flows/build.key'), `${key}\n`); + const local = await seal(out, {}); + const env = await seal(await temp()); + expect(await readFile(join(local, 'identity.json'))).toEqual(await readFile(join(env, 'identity.json'))); + await expect(seal(await temp(), { FLOWS_BUILD_KEY: 'invalid' })).rejects.toThrow('32-byte seed'); + }); + + it('returns exit 2 naming a byte-flipped payload and refuses to reuse corruption', async () => { + const out = await temp(); + const bundle = await seal(out); + const path = join(bundle, 'spec.canonical.json'); + const bytes = await readFile(path); bytes[0] = bytes[0]! ^ 1; await writeFile(path, bytes); + const result = invoke(['--verify', bundle]); + expect(result.status).toBe(2); + expect(result.stderr).toContain('spec.canonical.json'); + await expect(seal(out)).rejects.toThrow('spec.canonical.json'); + }); + + it.each(['identity.json', 'manifest.json'])('detects tampered %s', async file => { + const bundle = await seal(await temp()); + await writeFile(join(bundle, file), '{}'); + await expect(verifyBundle(bundle)).rejects.toThrow(file); + }); + + it('rejects a renamed digest directory and a well-formed invalid signature', async () => { + const out = await temp(); + const bundle = await seal(out); + const moved = join(out, `example@sha256:${'0'.repeat(64)}`); + await rename(bundle, moved); + await expect(verifyBundle(moved)).rejects.toThrow('directory digest mismatch'); + await rename(moved, bundle); + const identity = JSON.parse(await readFile(join(bundle, 'identity.json'), 'utf8')); + identity.signature_hex = '0'.repeat(128); + await writeFile(join(bundle, 'identity.json'), canonicalize(identity)); + await expect(verifyBundle(bundle)).rejects.toThrow('identity.json: signature mismatch'); + }); + + it('rejects extra files, symlinks, and manifest traversal', async () => { + const bundle = await seal(await temp()); + await writeFile(join(bundle, 'extra'), 'extra'); + await expect(verifyBundle(bundle)).rejects.toThrow('extra'); + await rm(join(bundle, 'extra')); + await rm(join(bundle, 'lockfile.json')); + await symlink(join(bundle, 'preflight.json'), join(bundle, 'lockfile.json')); + await expect(verifyBundle(bundle)).rejects.toThrow('lockfile.json'); + await writeFile(join(bundle, 'manifest.json'), canonicalize([{ path: '../outside', sha256: 'a'.repeat(64), bytes: 0 }])); + await expect(verifyBundle(bundle)).rejects.toThrow('manifest.json'); + }); + + it('builds and verifies the canonical YAML fixture through the compiled CLI', async () => { + const out = await temp(); + const result = invoke(['--out', out, 'testdata/hello-deterministic.flow.yaml']); + expect(result.stderr).toBe(''); expect(result.status).toBe(0); + const bundle = result.stdout.trim(); + expect(await readFile(join(bundle, 'spec.canonical.json'), 'utf8')).toBe( + (await readFile(join(repo, 'testdata/hello-deterministic.spec.canonical.json'), 'utf8')).trim(), + ); + expect(await readdir(bundle)).not.toContain('flow'); + expect(invoke(['--verify', bundle]).status).toBe(0); + expect(invoke(['--out', out, 'testdata/hello-deterministic.flow.yaml']).stdout).toBe(result.stdout); + }); + + it('emits the ephemeral warning on CLI stderr and uses the default output directory', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'hello.yaml'), await readFile(join(repo, 'testdata/hello-deterministic.flow.yaml'))); + const result = invoke(['hello.yaml'], cwd, { FLOWS_BUILD_KEY: undefined }); + expect(result.status).toBe(0); + expect(result.stderr).toContain('identity_ephemeral: bundle can be verified but not attributed'); + expect(result.stdout).toContain(join(cwd, 'dist/flows')); + expect(invoke(['--verify', result.stdout.trim()]).status).toBe(0); + }); + + it('captures relative file references and refuses missing assets', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'script.sh'), '#!/bin/sh\necho hello\n'); + await writeFile(join(cwd, 'hello.yaml'), 'version: 0.1.0\nname: asset-flow\nsteps:\n - id: run\n type: deterministic\n command: ./script.sh\n'); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + expect(await readFile(join(bundle, 'assets/script.sh'), 'utf8')).toContain('echo hello'); + expect(await readFile(join(bundle, 'spec.canonical.json'), 'utf8')).toContain('./assets/script.sh'); + await rm(join(cwd, 'script.sh')); + await expect(buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {})).rejects.toThrow('script.sh'); + }); + + it('deduplicates named-agent CLI captures that share one relative path', async () => { + // Two named agents pointing at the same `./` CLI must both resolve to a + // single bundle entry. captureFiles runs the agents through Promise.all; + // if the capture cache holds resolved strings the two concurrent calls + // both miss the guard, push twice, and sealBundle refuses the whole flow. + const cwd = await temp(); + await writeFile(join(cwd, 'agent-cli'), '#!/bin/sh\necho ok\n'); + await chmod(join(cwd, 'agent-cli'), 0o755); + await writeFile(join(cwd, 'flows.json'), JSON.stringify({ + cli: './agent-cli', models: ['claude-sonnet-4-6'], + })); + await writeFile(join(cwd, 'twin.yaml'), JSON.stringify({ + version: '0.1.0', name: 'twin-agents', + agents: { + reviewer: { cli: './agent-cli', model: 'claude-sonnet-4-6' }, + drafter: { cli: './agent-cli', model: 'claude-sonnet-4-6' }, + }, + steps: [ + { id: 'first', type: 'agent', instruction: 'first', agent: 'reviewer' }, + { id: 'second', type: 'agent', instruction: 'second', agent: 'drafter' }, + ], + })); + const bundle = await buildFlow(join(cwd, 'twin.yaml'), join(cwd, 'out'), () => {}); + // toKernelSpec compiles named agents away, lowering each step's `cli` + // reference. Both steps should point at the same shared bundle path. + const spec = JSON.parse(await readFile(join(bundle, 'spec.canonical.json'), 'utf8')); + expect(spec.steps[0].cli).toBe('./assets/agent-cli'); + expect(spec.steps[1].cli).toBe('./assets/agent-cli'); + // Manifest must carry the CLI exactly once; a duplicate entry is what + // sealBundle would refuse if the capture cache lost the race. + const manifest = JSON.parse(await readFile(join(bundle, 'manifest.json'), 'utf8')); + const assetEntries = manifest.filter((entry: { path: string }) => entry.path === 'assets/agent-cli'); + expect(assetEntries).toHaveLength(1); + expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); + }); + + it('preserves quoted asset words and executable permissions', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'run script.sh'), '#!/bin/sh\ncat "$1"\n'); + await chmod(join(cwd, 'run script.sh'), 0o755); + await writeFile(join(cwd, 'data.txt'), 'bundled asset\n'); + await writeFile(join(cwd, 'hello.yaml'), JSON.stringify({ version: '0.1.0', name: 'assets', steps: [ + { id: 'run', type: 'deterministic', command: '"./run script.sh" ./data.txt' }, + ] })); + const bundle = await buildFlow(join(cwd, 'hello.yaml'), join(cwd, 'out'), () => {}); + const spec = JSON.parse(await readFile(join(bundle, 'spec.canonical.json'), 'utf8')); + expect(spec.steps[0].command).toBe('"./assets/run script.sh" ./assets/data.txt'); + const result = spawnSync('/bin/sh', ['-c', spec.steps[0].command], { cwd: bundle, encoding: 'utf8' }); + expect(result.status).toBe(0); + expect(result.stdout).toBe('bundled asset\n'); + expect(JSON.parse(await readFile(join(bundle, 'metadata.json'), 'utf8')).executables).toEqual(['assets/run script.sh']); + expect(await verifyBundle(bundle)).toBe(basename(bundle).split('@sha256:')[1]); + }); + + it('refuses build-provable CLI resolution errors without environment probes', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'invalid.yaml'), JSON.stringify({ version: '0.1.0', name: 'invalid', steps: [ + { id: 'ask', type: 'agent', instruction: 'hello' }, + ] })); + const result = invoke(['invalid.yaml'], cwd); + expect(result.status).toBe(2); + expect(result.stderr).toContain('cli_unresolved'); + }); + + it('builds a standalone TS fixture twice with identical executable hashes', async () => { + const result = invoke(['--out', await temp(), 'packages/sdk/tests/fixtures/build.flow.ts']); + expect(result.stderr).toBe(''); expect(result.status).toBe(0); + const bundle = result.stdout.trim(); + const second = invoke(['--out', await temp(), 'packages/sdk/tests/fixtures/build.flow.ts']); + expect(second.status).toBe(0); expect(basename(second.stdout.trim())).toBe(basename(bundle)); + for (const file of await readdir(bundle)) expect(sha256(await readFile(join(bundle, file)))).toBe(sha256(await readFile(join(second.stdout.trim(), file)))); + expect((await stat(join(bundle, 'flow'))).mode & 0o111).not.toBe(0); + expect(invoke(['--verify', bundle]).status).toBe(0); + expect(JSON.parse(await readFile(join(bundle, 'metadata.json'), 'utf8')).platform).toEqual({ os: process.platform, arch: process.arch }); + }, 120_000); + + it('refuses to label installed dependency drift with lockfile pins', async () => { + const cwd = await temp(); + await writeFile(join(cwd, 'hello.ts'), 'export default {};'); + await mkdir(join(cwd, 'node_modules/example'), { recursive: true }); + await writeFile(join(cwd, 'node_modules/example/package.json'), JSON.stringify({ version: '2.0.0' })); + await writeFile(join(cwd, 'package-lock.json'), JSON.stringify({ lockfileVersion: 3, packages: { + 'node_modules/example': { version: '1.0.0' }, + } })); + const result = invoke(['hello.ts'], cwd); + expect(result.status).toBe(2); + expect(result.stderr).toContain('node_modules/example does not match its pinned version'); + }); + + it.each([[], ['--out'], ['--verify'], ['--verify', 'x', 'y'], ['--out', 'x', '--out', 'y', 'flow.yaml']])('refuses invalid CLI arguments %j', async (...args) => { + expect(invoke(args).status).toBe(2); + }); +}); diff --git a/packages/sdk/tests/fixtures/build.flow.ts b/packages/sdk/tests/fixtures/build.flow.ts new file mode 100644 index 000000000..1d6e08653 --- /dev/null +++ b/packages/sdk/tests/fixtures/build.flow.ts @@ -0,0 +1,12 @@ +import { flow } from '@relayflows/surface'; + +export const spec = { + version: '0.1.0', name: 'hello-build-ts', + steps: [{ id: 'greet', type: 'deterministic', command: 'echo hello' }], +}; + +// Build retains this program without evaluating its body or requiring input. +export default flow('hello-build-ts', async f => { + await f.run('echo hello'); + f.done('success'); +}); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 9293212bf..1e5863c34 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -10,6 +10,8 @@ }, "include": [ "src/**/*.ts", + "tests/bundle.test.ts", + "tests/fixtures/build.flow.ts", "tests/typed-output.test.ts", "tests/mcp.test.ts", "tests/mcp-lifecycle.test.ts",