feat(sdk): flows build — immutable content-addressed bundle (#298)

.gitignore

@@ -20,3 +20,4 @@ dist/
 
 # Legacy drive-local snapshots; the preparing launcher no longer executes them.
 .drive-gate/
+.flows/build.key

docs/SURFACE.md

@@ -448,6 +448,35 @@ The herdr model: first-party helpers are just plugins that ship in the box; the
 
 `flows build` seals a flow into a content-addressed, immutable bundle: canonical spec JSON, compiled TS with pinned deps, helper/plugin lockfile, assets, preflight declaration, identity signature — `flow@sha256:…`, pushed to a bucket/registry. `flows deploy` points a trigger at a digest; `flows run flow@sha256:…` executes from the bucket on any cell, no checkout. Preflight runs at build time for everything build-provable and again at deploy time for environment facts (credentials, workers, MCP servers). The working tree is for authoring; **production only ever runs digests.**
 
+The local build and verification commands are available now:
+
+```text
+flows build [--out <dir>] <flow.yaml|flow.ts>
+flows build --verify <bundle-dir>
+```
+
+Output defaults to `dist/flows/<name>@sha256:<digest>/`. The canonical manifest
+hashes the payload files; `manifest.json` and `identity.json` are excluded from
+its entries to avoid circular hashing. A stable signing seed gives identical
+identity bytes across builds. Ephemeral keys preserve the payload digest but
+produce different identity signatures. An existing valid bundle is reused.
+Verification refuses changed, missing, unlisted, or symlinked files with exit 2.
+
+TypeScript builds require Bun on PATH and an installed npm workspace matching
+its `package-lock.json`. The full lockfile is retained in this first slice.
+A TS module can default-export a declarative spec, or default-export `flow()`
+with an additional exported `spec` declaration for build-time checks. The
+authored body is retained in the executable and is not run during build;
+nonempty authored headers are currently refused. Module initialization must
+be deterministic. `preflight.json` preserves the preflight report, including
+uncollected environment facts; `metadata.json` separately records the platform,
+compiler, executable asset paths, and checks deferred until deployment.
+
+Signing uses `FLOWS_BUILD_KEY` or the repository's `.flows/build.key`, each a
+base64 32-byte Ed25519 seed. Without either, stderr reports
+`identity_ephemeral: bundle can be verified but not attributed`.
+Deployment, remote upload, and execution by digest remain future slices.
+
 ## 5. Invocation: the gate-1 CLI
 
 Gate 1 ships three CLI verbs over the journal protocol, plus one out-of-band

packages/sdk/src/bundle-typescript.ts

@@ -0,0 +1,98 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { canonicalize } from './canonical.js';
+import { compileSpec } from './compile.js';
+import type { BundleFile } from './bundle.js';
+
+/** Bun is an explicit build dependency, never an implicit install. */
+function bun(args: string[], cwd: string): string {
+  const result = spawnSync('bun', ['--no-env-file', ...args], {
+    cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024,
+    env: { PATH: process.env['PATH'], NODE_ENV: 'production', TZ: 'UTC' },
+  });
+  if (result.error || result.status !== 0) {
+    throw new Error(`TypeScript build requires Bun: ${result.error?.message ?? result.stderr}`);
+  }
+  return result.stdout.trim();
+}
+
+export async function buildTypescript(input: string) {
+  const directory = dirname(input);
+  const lockPath = await findLock(directory);
+  const lock = JSON.parse(await readFile(lockPath, 'utf8'));
+  if (![2, 3].includes(lock.lockfileVersion) || !lock.packages) {
+    throw new Error('package-lock.json: expected npm lockfile version 2 or 3 with pinned packages');
+  }
+  await checkInstalledVersions(dirname(lockPath), lock.packages);
+  const compiler = `bun@${bun(['--version'], directory)}`;
+  const staging = await mkdtemp(join(tmpdir(), 'flows-ts-'));
+  try {
+    // Evaluate module declarations, never the authored body. Resolve the runtime
+    // beside the input, keeping flow()'s WeakMap identity in the same module.
+    const probe = `
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+const module = await import(${JSON.stringify(pathToFileURL(input).href)});
+let spec = module.default;
+let authored = false;
+if (!spec || !Array.isArray(spec.steps)) {
+  const require = createRequire(${JSON.stringify(pathToFileURL(input).href)});
+  const { getFlowDefinition } = await import(pathToFileURL(require.resolve('@relayflows/surface/runtime')).href);
+  const definition = getFlowDefinition(spec);
+  if (Object.keys(definition.header).length) throw new Error('unsupported authored flow header');
+  if (!module.spec) throw new Error('authored flow() requires an exported spec declaration for build-time preflight; the body is not executed during build');
+  spec = module.spec;
+  if (spec.name !== definition.name) throw new Error('exported spec name must match flow() name');
+  authored = true;
+}
+
+process.stdout.write(JSON.stringify({ spec, authored }));
+`;
+    const probePath = join(staging, 'inspect.ts');
+    await writeFile(probePath, probe);
+    const inspected = JSON.parse(bun([probePath], directory));
+    const spec = compileSpec(inspected.spec);
+    const executable = join(staging, 'flow');
+    bun(['build', '--compile', '--env=disable', '--no-compile-autoload-dotenv',
+      '--no-compile-autoload-bunfig', '--outfile', executable, input], directory);
+    const files: BundleFile[] = [
+      { path: 'flow', data: await readFile(executable) },
+      { path: 'lockfile.json', data: canonicalize(lock) },
+    ];
+    return { spec, authored: inspected.authored === true, compiler, files };
+  } finally { await rm(staging, { recursive: true, force: true }); }
+}
+
+/** Do not label a stale node_modules tree with a newer lockfile's pins. */
+async function checkInstalledVersions(root: string, packages: Record<string, {
+  version?: string; optional?: boolean; dev?: boolean; link?: boolean; resolved?: string;
+}>): Promise<void> {
+  for (const [path, entry] of Object.entries(packages)) {
+    if (path === '') continue;
+    if (path.startsWith('/') || path.includes('\\') || path.split('/').includes('..')
+      || (entry.resolved !== undefined && /^(?:file:|\/|[A-Za-z]:)/.test(entry.resolved))) {
+      throw new Error(`package-lock.json: ${path} must use portable, pinned dependency locations`);
+    }
+    if (entry.link) continue; // Workspace source is captured by the compiler.
+    let installed;
+    try { installed = JSON.parse(await readFile(join(root, path, 'package.json'), 'utf8')); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT' && (entry.optional || entry.dev)) continue;
+      throw new Error(`package-lock.json: ${path} is missing; run npm ci before building`);
+    }
+    if (typeof entry.version !== 'string' || installed.version !== entry.version) {
+      throw new Error(`package-lock.json: ${path} does not match its pinned version; run npm ci before building`);
+    }
+  }
+}
+async function findLock(start: string): Promise<string> {
+  for (let directory = start; ; directory = dirname(directory)) {
+    const candidate = join(directory, 'package-lock.json');
+    try { await readFile(candidate); return candidate; }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+    if (dirname(directory) === directory) throw new Error('package-lock.json: no workspace lockfile found for TS flow');
+  }
+}

packages/sdk/src/bundle.ts

@@ -0,0 +1,179 @@
+import { createHash, createPrivateKey, createPublicKey, randomBytes, sign, verify } from 'node:crypto';
+import { lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
+import { basename, dirname, join, resolve } from 'node:path';
+import { canonicalize } from './canonical.js';
+
+export interface BundleEntry { path: string; sha256: string; bytes: number }
+export interface BundleFile { path: string; data: Uint8Array | string; executable?: boolean }
+export interface BundleOptions {
+  name: string;
+  out: string;
+  repo: string;
+  files: BundleFile[];
+  env?: NodeJS.ProcessEnv;
+  warn(message: string): void;
+}
+
+export function sha256(data: Uint8Array | string): string {
+  return createHash('sha256').update(data).digest('hex');
+}
+
+function safePath(path: string): boolean {
+  return path.length > 0 && !path.includes('\\') && !path.includes('\0')
+    && path.split('/').every(part => part !== '' && part !== '.' && part !== '..')
+    && !path.includes(':');
+}
+
+/** Manifest and identity are envelopes, excluded to avoid circular hashing. */
+export async function sealBundle(options: BundleOptions): Promise<string> {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(options.name)) {
+    throw new Error('spec.canonical.json: name must be a safe single directory component');
+  }
+  const files = options.files.map(file => ({ ...file, data: Buffer.from(file.data) }))
+    .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+  const paths = new Set<string>();
+  for (const file of files) {
+    if (!safePath(file.path) || ['manifest.json', 'identity.json'].includes(file.path) || paths.has(file.path)) {
+      throw new Error(`${file.path}: invalid or duplicate bundle path`);
+    }
+    paths.add(file.path);
+  }
+  for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) {
+    if (!paths.has(required)) throw new Error(`${required}: missing bundle file`);
+  }
+  const manifest = canonicalize(files.map(file => ({
+    path: file.path, sha256: sha256(file.data), bytes: file.data.length,
+  })));
+  const digest = sha256(manifest);
+  const out = resolve(options.out);
+  const target = join(out, `${options.name}@sha256:${digest}`);
+  let exists = false;
+  try { await lstat(target); exists = true; }
+  catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+  if (exists) { await verifyBundle(target); return target; }
+  const identity = await signDigest(digest, options);
+  await mkdir(out, { recursive: true });
+  const staging = await mkdtemp(join(out, '.build-'));
+  try {
+    for (const file of files) {
+      await mkdir(dirname(join(staging, file.path)), { recursive: true });
+      await writeFile(join(staging, file.path), file.data, { mode: file.path === 'flow' || file.executable ? 0o755 : 0o644 });
+    }
+    await writeFile(join(staging, 'manifest.json'), manifest);
+    await writeFile(join(staging, 'identity.json'), canonicalize(identity));
+    try { await rename(staging, target); }
+    catch (error) {
+      if (!['EEXIST', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error;
+      await verifyBundle(target);
+    }
+    return target;
+  } finally { await rm(staging, { recursive: true, force: true }); }
+}
+
+async function signDigest(digest: string, options: BundleOptions) {
+  let seedText = (options.env ?? process.env)['FLOWS_BUILD_KEY'];
+  if (seedText === undefined) {
+    try { seedText = (await readFile(join(options.repo, '.flows/build.key'), 'utf8')).trim(); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+  }
+  let seed: Buffer;
+  if (seedText === undefined) {
+    options.warn('identity_ephemeral: bundle can be verified but not attributed');
+    seed = randomBytes(32);
+  } else {
+    seed = Buffer.from(seedText, 'base64');
+    if (seed.length !== 32 || seed.toString('base64') !== seedText) {
+      throw new Error('build key: expected a base64-encoded 32-byte seed');
+    }
+  }
+  const privateKey = createPrivateKey({
+    key: Buffer.concat([Buffer.from('302e020100300506032b657004220420', 'hex'), seed]),
+    format: 'der', type: 'pkcs8',
+  });
+  const publicKey = createPublicKey(privateKey).export({ format: 'der', type: 'spki' }).subarray(-32);
+  return {
+    algorithm: 'ed25519', keyid: sha256(publicKey).slice(0, 16), pubkey_b64: publicKey.toString('base64'),
+    signature_hex: sign(null, Buffer.from(digest, 'hex'), privateKey).toString('hex'),
+  };
+}
+
+async function regularFile(root: string, path: string): Promise<Buffer> {
+  try {
+    const parts = path.split('/');
+    for (let i = 1; i <= parts.length; i++) {
+      const stat = await lstat(join(root, ...parts.slice(0, i)));
+      if (i === parts.length ? !stat.isFile() : !stat.isDirectory()) {
+        throw new Error('expected a regular file, without symlinks');
+      }
+    }
+    return await readFile(join(root, path));
+  } catch (error) {
+    throw new Error(`${path}: ${error instanceof Error ? error.message : 'unreadable'}`);
+  }
+}
+
+/** Verify an untrusted directory without following symlinks or manifest traversal. */
+export async function verifyBundle(directory: string): Promise<string> {
+  const root = resolve(directory);
+  if (!(await lstat(root)).isDirectory()) throw new Error('manifest.json: bundle must be a directory, not a symlink');
+  const raw = (await regularFile(root, 'manifest.json')).toString('utf8');
+  let manifest: BundleEntry[];
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (!Array.isArray(parsed)) throw new Error('expected an array');
+    manifest = parsed;
+  } catch { throw new Error('manifest.json: invalid manifest'); }
+  const paths = new Set<string>();
+  let previous = '';
+  for (const entry of manifest) {
+    if (entry === null || typeof entry !== 'object' || typeof entry.path !== 'string'
+      || !safePath(entry.path) || entry.path <= previous
+      || ['manifest.json', 'identity.json'].includes(entry.path)
+      || Object.keys(entry).sort().join(',') !== 'bytes,path,sha256'
+      || !/^[a-f0-9]{64}$/.test(entry.sha256) || !Number.isSafeInteger(entry.bytes) || entry.bytes < 0) {
+      throw new Error('manifest.json: invalid, duplicate, or unsorted entry');
+    }
+    previous = entry.path;
+    paths.add(entry.path);
+    const bytes = await regularFile(root, entry.path);
+    if (bytes.length !== entry.bytes || sha256(bytes) !== entry.sha256) {
+      throw new Error(`${entry.path}: sha256 or byte count mismatch`);
+    }
+  }
+  for (const required of ['spec.canonical.json', 'preflight.json', 'lockfile.json']) {
+    if (!paths.has(required)) throw new Error(`${required}: missing manifest entry`);
+  }
+  const canonical = canonicalize(manifest);
+  if (raw !== canonical) throw new Error('manifest.json: noncanonical bytes');
+  const digest = sha256(canonical);
+  if (!basename(root).endsWith(`@sha256:${digest}`)) throw new Error('manifest.json: directory digest mismatch');
+  await rejectExtras(root, '', new Set([...paths, 'manifest.json', 'identity.json']));
+  try {
+    const identity = JSON.parse((await regularFile(root, 'identity.json')).toString('utf8'));
+    const publicKey = Buffer.from(identity.pubkey_b64, 'base64');
+    if (identity.algorithm !== 'ed25519' || publicKey.length !== 32
+      || publicKey.toString('base64') !== identity.pubkey_b64
+      || identity.keyid !== sha256(publicKey).slice(0, 16)
+      || !/^[a-f0-9]{128}$/.test(identity.signature_hex)) throw new Error('invalid identity');
+    const key = createPublicKey({
+      key: Buffer.concat([Buffer.from('302a300506032b6570032100', 'hex'), publicKey]), format: 'der', type: 'spki',
+    });
+    if (!verify(null, Buffer.from(digest, 'hex'), key, Buffer.from(identity.signature_hex, 'hex'))) {
+      throw new Error('signature mismatch');
+    }
+  } catch (error) {
+    throw new Error(`identity.json: ${error instanceof Error ? error.message : 'signature verification failed'}`);
+  }
+  return digest;
+}
+
+async function rejectExtras(root: string, prefix: string, paths: Set<string>): Promise<void> {
+  for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) {
+    const path = prefix + entry.name;
+    if (entry.isDirectory() && [...paths].some(file => file.startsWith(`${path}/`))) {
+      await rejectExtras(root, `${path}/`, paths);
+    } else if (!entry.isFile() || !paths.has(path)) {
+      throw new Error(`${path}: unlisted file or unsupported file type`);
+    }
+  }
+}

packages/sdk/src/cli.ts

@@ -21,6 +21,7 @@ import { parseReplayArgs, replayJournal, type ReplayArgs } from './cli/replay.js
 import { checkTypeScriptFlow } from './cli/check-typescript.js';
 import { runCloudCli } from './cli/cloud-run.js';
 import { isAuthoredFlowPath } from './direct-input.js';
+import { parseBuildArgs, runBuild, type BuildArgs } from './cli/build.js';
 import { runHnMonitor } from './cli/hn-monitor.js';
 import { runTickRunner } from './cli/tick-runner.js';
 import {
@@ -39,6 +40,7 @@ export interface CliIo {
 type CliExitCode = 0 | 1 | 2 | 3;
 type ParsedArgs =
   | ReplayArgs
+  | BuildArgs
   | { command: 'cloud-run'; value: string; json: boolean; wait: boolean }
   | { command: 'check'; json: boolean; value: string }
   | { command: 'run'; reuseFromRunId: string | undefined; localAgent: boolean; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; value: string }
@@ -52,6 +54,8 @@ type ParsedArgs =
 const DEFAULT_DATA_DIR = '.relayflowd';
 const USAGE = [
   'Usage:',
+  'flows build [--out <dir>] <flow.yaml|flow.ts>',
+  'flows build --verify <bundle-dir>',
   'flows check [--json] <flow.ts|flow.yaml|spec.json>',
   'flows run [--json] [--no-spawn] [--no-observer-link] [--data-dir <dir>] [--local-agent] [--reuse-from <run-id>] <flow.yaml|spec.json>',
   'flows run --cloud [--json] [--wait] <flow.yaml|spec.json>',
@@ -96,6 +100,7 @@ export async function runCli(
 
   if (parsed.command === 'cloud-run') return runCloudCli(parsed, io);
   if (parsed.command === 'replay') return replayJournal(parsed, io);
+  if (parsed.command === 'build') return runBuild(parsed, io);
 
   if (parsed.command === 'check') {
     // Deliberately daemon-free (kernel/DAEMON-LIFECYCLE.md §4). `checkFlow` is
@@ -385,6 +390,7 @@ function emitWait(
 function parseArgs(args: readonly string[]): ParsedArgs | undefined {
   const command = args[0];
   if (command === 'replay') return parseReplayArgs(args.slice(1));
+  if (command === 'build') return parseBuildArgs(args.slice(1));
   if (command === 'hn-monitor') return parseHnMonitorArgs(args.slice(1));
   if (command === 'tick') return parseTickArgs(args.slice(1));
   if (command === 'observer') return parseObserverArgs(args.slice(1));