From a2ec4a98ec1a63817c74a2abde2d000766685953 Mon Sep 17 00:00:00 2001 From: miguel Date: Sat, 22 Aug 2026 12:47:05 -0700 Subject: [PATCH 1/7] feat(integrations): add fx CLI harness session layer, tool adapter, and runner (phase 1) - New workspace package @browserbasehq/stagehand-integrations-fx-sdk: runFxSession spawns `fx ask --json` with an isolated HOME, tails the session events.jsonl for per-tool-call evidence, and normalizes status/stopReason/tokenUsage. - evals: fxToolAdapter (MCP-only mounts -> ~/.fx/mcp.json + settings.json + workspace .fx.json/AGENTS.md), fxRunner (mirrors codexRunner), harnesses/fxAdapter (tool_step events -> NormalizedToolCall), unit tests. - turbo/ci/vitest wiring for the new package. Registry/planner wiring lands in phase 2. --- .github/workflows/ci.yml | 1 + packages/evals/framework/fxRunner.ts | 212 ++++++ packages/evals/framework/fxToolAdapter.ts | 322 ++++++++ .../evals/framework/harnesses/fxAdapter.ts | 144 ++++ packages/evals/package.json | 3 + .../evals/tests/framework/fxAdapter.test.ts | 145 ++++ .../evals/tests/framework/fxRunner.test.ts | 95 +++ .../tests/framework/fxToolAdapter.test.ts | 101 +++ packages/integrations/fx-sdk/package.json | 34 + packages/integrations/fx-sdk/src/index.ts | 1 + packages/integrations/fx-sdk/src/session.ts | 645 ++++++++++++++++ .../integrations/fx-sdk/tests/session.test.ts | 261 +++++++ packages/integrations/fx-sdk/tsconfig.json | 13 + packages/integrations/fx-sdk/tsdown.config.ts | 15 + pnpm-lock.yaml | 720 +++++++----------- turbo.json | 18 + vitest.config.ts | 1 + 17 files changed, 2268 insertions(+), 463 deletions(-) create mode 100644 packages/evals/framework/fxRunner.ts create mode 100644 packages/evals/framework/fxToolAdapter.ts create mode 100644 packages/evals/framework/harnesses/fxAdapter.ts create mode 100644 packages/evals/tests/framework/fxAdapter.test.ts create mode 100644 packages/evals/tests/framework/fxRunner.test.ts create mode 100644 packages/evals/tests/framework/fxToolAdapter.test.ts create mode 100644 packages/integrations/fx-sdk/package.json create mode 100644 packages/integrations/fx-sdk/src/index.ts create mode 100644 packages/integrations/fx-sdk/src/session.ts create mode 100644 packages/integrations/fx-sdk/tests/session.test.ts create mode 100644 packages/integrations/fx-sdk/tsconfig.json create mode 100644 packages/integrations/fx-sdk/tsdown.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 284ded68d..b8a71dd57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,6 +188,7 @@ jobs: packages/integrations/pi-sdk/dist/** packages/integrations/eve-sdk/dist/** packages/integrations/deepagents-sdk/dist/** + packages/integrations/fx-sdk/dist/** packages/evals/dist/** retention-days: 1 diff --git a/packages/evals/framework/fxRunner.ts b/packages/evals/framework/fxRunner.ts new file mode 100644 index 000000000..4c3ce5e7e --- /dev/null +++ b/packages/evals/framework/fxRunner.ts @@ -0,0 +1,212 @@ +// Prompt builder and result parser are intentionally duplicated from codexRunner.ts for Phase 1; +// Phase 2 switches this to the shared externalRunner skeleton. +import { + buildFxTranscript, + normalizeFxModel, + runFxSession, + stringifyError, + toFiniteNumber, + type FxProcessRunner, + type FxSessionStore, + type FxTokenUsage, +} from "@browserbasehq/stagehand-integrations-fx-sdk"; +import type { AvailableModel } from "stagehand-v3"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import { datasetPromptGuidance, type ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import type { PreparedFxToolAdapter } from "./fxToolAdapter.js"; +import { readFxMaxAgentSteps } from "./fxToolAdapter.js"; +import { fxAdapter } from "./harnesses/fxAdapter.js"; +import type { TaskResult } from "./types.js"; +import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; + +export { + buildFxTranscript, + normalizeFxModel, + runFxSession, +} from "@browserbasehq/stagehand-integrations-fx-sdk"; + +type MetricValue = { count: number; value: number }; + +export interface FxRunnerInput { + plan: ExternalHarnessTaskPlan; + model: AvailableModel; + logger: EvalLogger; + toolAdapter?: PreparedFxToolAdapter; + signal?: AbortSignal; + verifier?: ExternalHarnessVerifierConfig; + runProcess?: FxProcessRunner; + store?: FxSessionStore; +} + +export interface ParsedFxResult { + success: boolean; + summary?: string; + finalAnswer?: string; + raw: string; +} + +export function buildFxPrompt(plan: ExternalHarnessTaskPlan, toolInstructions?: string): string { + return [ + "You are running a browser benchmark task.", + "", + `Dataset: ${plan.dataset}`, + plan.taskId ? `Task ID: ${plan.taskId}` : undefined, + `Start URL: ${plan.startUrl}`, + "", + "Instruction:", + plan.instruction, + "", + datasetPromptGuidance(plan.dataset), + toolInstructions ?? "Use the available browser/web tools to complete the task.", + "Do not edit repository files.", + "At the end, return compact JSON matching this schema:", + '{"success": boolean, "summary": string, "finalAnswer": string}', + ] + .filter(Boolean) + .join("\n"); +} + +export function parseFxResult(raw: string): ParsedFxResult { + const marker = "EVAL_RESULT:"; + const markerIndex = raw.lastIndexOf(marker); + const candidates = + markerIndex >= 0 + ? [ + raw.slice(markerIndex + marker.length).trim(), + raw + .slice(markerIndex + marker.length) + .trim() + .split(/\r?\n/u, 1)[0] + ?.trim(), + ] + : [raw.trim(), raw.trim().split(/\r?\n/u, 1)[0]?.trim()]; + + for (const candidate of candidates) { + if (!candidate) continue; + const parsed = tryParseFxJson(candidate); + if (parsed) return { ...parsed, raw }; + } + return { success: false, raw }; +} + +export async function runFxAgent({ + plan, + model, + logger, + toolAdapter, + signal, + verifier, + runProcess, + store, +}: FxRunnerInput): Promise { + if (!toolAdapter) throw new EvalsError("fx requires a prepared tool adapter."); + const prompt = buildFxPrompt(plan, toolAdapter.promptInstructions); + const sessionResult = await runFxSession({ + prompt, + model: normalizeFxModel(model), + cwd: toolAdapter.cwd, + home: toolAdapter.home, + env: toolAdapter.env, + permissionMode: process.env.EVAL_FX_PERMISSION_MODE === "yolo" ? "yolo" : "auto", + maxAgentSteps: readFxMaxAgentSteps(), + signal, + logger, + runProcess, + store, + onToolStep: toolAdapter.recordObservation + ? async () => toolAdapter.recordObservation?.() + : undefined, + observedTool: toolAdapter.observedToolMatcher, + }); + const { events, finalMessage, iterationError, status, stopReason, tokenUsage } = sessionResult; + const transcriptText = buildFxTranscript(events); + const iterationErrorMessage = stringifyError(iterationError); + const rawResult = [finalMessage, transcriptText, iterationErrorMessage] + .filter(Boolean) + .join("\n\n"); + const parsed = parseFxResult(rawResult); + const errorMessage = + parsed.summary ?? + stopReason ?? + (iterationErrorMessage || finalMessage || transcriptText || "fx did not report success"); + const baseResult: TaskResult = { + _success: parsed.success, + error: !parsed.success ? errorMessage : undefined, + reasoning: parsed.summary, + finalAnswer: parsed.finalAnswer, + rawResult: parsed.raw, + fxStatus: status, + ...(stopReason && { fxStopReason: stopReason }), + logs: logger.getLogs(), + metrics: buildFxMetrics(tokenUsage), + }; + if (!verifier) return baseResult; + + const finalObservation = await toolAdapter.captureEvidence?.().catch((): undefined => undefined); + const stepObservations = await toolAdapter.drainStepObservations?.(); + return gradeExternalTrajectory({ + buildTrajectory: () => + fxAdapter.fromHarnessResult( + { + events, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), + ...(toolAdapter.observedToolMatcher && { + observedToolName: toolAdapter.observedToolMatcher, + }), + finalAnswer: parsed.finalAnswer ?? finalMessage, + status: status === "completed" ? "complete" : "error", + usage: { + input_tokens: tokenUsage.input_tokens, + output_tokens: tokenUsage.output_tokens, + reasoning_tokens: tokenUsage.reasoning_output_tokens, + cached_input_tokens: tokenUsage.cached_input_tokens, + }, + }, + verifier.taskSpec, + ), + verifier, + baseResult, + errorMessage, + category: "fx", + logger, + }); +} + +function tryParseFxJson(candidate: string): Omit | undefined { + try { + const parsed = JSON.parse(candidate) as { + success?: unknown; + summary?: unknown; + finalAnswer?: unknown; + }; + return { + success: parsed.success === true, + summary: typeof parsed.summary === "string" ? parsed.summary : undefined, + finalAnswer: typeof parsed.finalAnswer === "string" ? parsed.finalAnswer : undefined, + }; + } catch { + return undefined; + } +} + +function buildFxMetrics(usage: FxTokenUsage): Record { + const inputTokens = toFiniteNumber(usage.input_tokens); + const cachedInputTokens = toFiniteNumber(usage.cached_input_tokens); + const outputTokens = toFiniteNumber(usage.output_tokens); + const reasoningOutputTokens = toFiniteNumber(usage.reasoning_output_tokens); + return { + fx_input_tokens: metricValue(inputTokens), + fx_cached_input_tokens: metricValue(cachedInputTokens), + fx_output_tokens: metricValue(outputTokens), + fx_reasoning_output_tokens: metricValue(reasoningOutputTokens), + fx_total_tokens: metricValue( + inputTokens + cachedInputTokens + outputTokens + reasoningOutputTokens, + ), + }; +} + +function metricValue(value: unknown): MetricValue { + return { count: 1, value: toFiniteNumber(value) }; +} diff --git a/packages/evals/framework/fxToolAdapter.ts b/packages/evals/framework/fxToolAdapter.ts new file mode 100644 index 000000000..b99aaa817 --- /dev/null +++ b/packages/evals/framework/fxToolAdapter.ts @@ -0,0 +1,322 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { ProbeEvidence } from "stagehand-v3"; +import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; + +export interface FxToolAdapterInput { + toolSurface?: ToolSurface; + startupProfile?: StartupProfile; + environment: "LOCAL" | "BROWSERBASE"; + plan: ExternalHarnessTaskPlan; + logger: EvalLogger; +} + +export interface PreparedFxToolAdapter { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + cwd: string; + home: string; + env: Record; + promptInstructions: string; + mcpServerNames: string[]; + captureEvidence?: () => Promise; + drainStepObservations?: () => Promise; + recordObservation?: () => void; + observedToolMatcher?: (name: string) => boolean; + cleanup: () => Promise; +} + +type FxMcpServerSpec = { + command: string; + args?: string[]; + env?: Record; +}; + +const FX_MCP_SURFACES = new Set([ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", +]); + +export function resolveFxToolSurface(requested?: ToolSurface): ToolSurface { + if (!requested) return "stagehand_facade"; + if (FX_MCP_SURFACES.has(requested)) return requested; + throw new EvalsError( + `fx harness supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "${requested}".`, + ); +} + +export function resolveFxStartupProfile( + toolSurface: ToolSurface, + environment: "LOCAL" | "BROWSERBASE", + requested?: StartupProfile, +): StartupProfile { + if (requested) return requested; + if (toolSurface === "stagehand_facade") { + return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; + } + if (toolSurface === "playwright_mcp" || toolSurface === "chrome_devtools_mcp") { + return environment === "BROWSERBASE" + ? "runner_provided_browserbase_cdp" + : "runner_provided_local_cdp"; + } + throw new EvalsError( + `No fx startup profile default for tool "${toolSurface}" in ${environment}.`, + ); +} + +export function buildFxMcpConfig( + mcpServers: Record, + options: { home: string; pathEnv: string }, +): { mcp: Record } { + const mcp: Record = {}; + for (const [serverName, rawSpec] of Object.entries(mcpServers)) { + if (!/^[A-Za-z0-9_-]+$/u.test(serverName)) { + throw new EvalsError(`Invalid fx MCP server name "${serverName}".`); + } + if (!isRecord(rawSpec) || typeof rawSpec.command !== "string") { + throw new EvalsError(`Invalid fx MCP launch spec for server "${serverName}".`); + } + const spec = rawSpec as FxMcpServerSpec; + const args = Array.isArray(spec.args) + ? spec.args.filter((arg): arg is string => typeof arg === "string") + : []; + const extraEnv = isStringRecord(spec.env) ? spec.env : {}; + mcp[serverName] = { + type: "stdio", + command: [spec.command, ...args], + environment: { PATH: options.pathEnv, HOME: options.home, ...extraEnv }, + required: true, + }; + } + return { mcp }; +} + +export function buildFxSettings( + _serverNames: string[], + toolSurface: ToolSurface, +): { permission: Record } { + return { + permission: { + run_command: "deny", + terminal: "deny", + write_file: "deny", + edit_file: "deny", + ...(toolSurface === "stagehand_facade" && { + mcp_stagehand_run: "allow", + mcp_stagehand_snapshot: "allow", + mcp_stagehand_screenshot: "allow", + }), + }, + }; +} + +export function buildFxAgentsMarkdown(promptInstructions: string, serverNames: string[]): string { + const prefixes = serverNames.map((server) => `mcp_${server}_`).join(", "); + const stagehandGuidance = serverNames.includes("stagehand") + ? "The Stagehand tools are exactly mcp_stagehand_run, mcp_stagehand_snapshot, and mcp_stagehand_screenshot." + : undefined; + return [ + "# Browser tool instructions for fx", + "", + `MCP tools use the fx name mcp__ (configured servers: ${serverNames.join(", ")}; patterns: ${prefixes}).`, + "Select tools with mcp_select_tool using their exact name. mcp_search_tools may return nothing.", + "Never invent tool names. Do not use the shell and do not edit files.", + stagehandGuidance, + "", + promptInstructions, + ] + .filter((line): line is string => line !== undefined) + .join("\n"); +} + +export async function prepareFxToolAdapter( + input: FxToolAdapterInput, +): Promise { + const toolSurface = resolveFxToolSurface(input.toolSurface); + const startupProfile = resolveFxStartupProfile( + toolSurface, + input.environment, + input.startupProfile, + ); + const runtime = await startAgentToolRuntime({ + toolSurface, + startupProfile, + environment: input.environment, + logger: input.logger, + }); + let root: string | undefined; + + try { + const mount = runtime.running.agentMount; + if (!mount) { + throw new EvalsError(`Tool surface "${toolSurface}" does not provide an agent mount.`); + } + if (mount.via !== "mcp") { + throw new EvalsError( + `fx does not support agent mounts delivered via "${mount.via}"; it can only host MCP servers.`, + ); + } + + root = await fsp.mkdtemp( + path.join(os.tmpdir(), `stagehand-evals-fx-${toolSurface.replace(/_/gu, "-")}-`), + ); + const home = path.join(root, "home"); + const workspace = path.join(root, "workspace"); + const fxHome = path.join(home, ".fx"); + await Promise.all([ + fsp.mkdir(fxHome, { recursive: true }), + fsp.mkdir(workspace, { recursive: true }), + ]); + + const serverNames = Object.keys(mount.mcpServers); + const pathEnv = process.env.PATH ?? ""; + const agentsMarkdown = buildFxAgentsMarkdown(mount.promptInstructions, serverNames); + await Promise.all([ + writeJson( + path.join(fxHome, "mcp.json"), + buildFxMcpConfig(mount.mcpServers, { home, pathEnv }), + ), + writeJson(path.join(fxHome, "settings.json"), buildFxSettings(serverNames, toolSurface)), + writeJson(path.join(workspace, ".fx.json"), { + max_agent_steps: readFxMaxAgentSteps(), + max_tool_result_bytes: 262_144, + }), + fsp.writeFile(path.join(workspace, "AGENTS.md"), agentsMarkdown), + ]); + + const recorder = runtime.running.captureEvidence + ? new ObservationRecorder(runtime.running.captureEvidence) + : undefined; + const capturedRoot = root; + let cleanupPromise: Promise | undefined; + input.logger.log({ + category: "fx", + message: `Initialized ${toolSurface} MCP mount for fx (servers: ${serverNames.join(", ")}).`, + level: 1, + auxiliary: { + startupProfile: { value: startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + return { + toolSurface, + startupProfile, + cwd: workspace, + home, + env: definedProcessEnv({ HOME: home }), + promptInstructions: agentsMarkdown, + mcpServerNames: serverNames, + ...(runtime.running.captureEvidence && { + captureEvidence: boundedCaptureEvidence(runtime.running.captureEvidence), + }), + ...(recorder && { + drainStepObservations: async () => { + await recorder.settle(); + return recorder.drain(); + }, + recordObservation: () => void recorder.record(), + }), + observedToolMatcher: (name: string) => + serverNames.some( + (server) => + name.startsWith(`mcp_${server}_`) || + name.startsWith(`mcp_${server.replace(/-/gu, "_")}_`), + ), + cleanup: async () => { + cleanupPromise ??= (async () => { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + "fx adapter cleanup", + ).catch((): undefined => undefined); + await fsp.rm(capturedRoot, { recursive: true, force: true }); + })(); + await cleanupPromise; + }, + }; + } catch (error) { + await withTimeout( + runtime.cleanup(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + "fx adapter cleanup", + ).catch((): undefined => undefined); + if (root) await fsp.rm(root, { recursive: true, force: true }); + throw error; + } +} + +export function readFxMaxAgentSteps(): number { + for (const key of ["EVAL_FX_MAX_STEPS", "AGENT_EVAL_MAX_STEPS"]) { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + return 60; +} + +function boundedCaptureEvidence( + capture: () => Promise, +): () => Promise { + return async () => { + try { + return await withTimeout( + capture(), + readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000), + "fx evidence capture", + ); + } catch { + return {}; + } + }; +} + +function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +function readPositiveIntEnv(key: string, fallback: number): number { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function definedProcessEnv(overrides: Record): Record { + const env: Record = {}; + for (const [key, value] of Object.entries({ ...process.env, ...overrides })) { + if (typeof value === "string") env[key] = value; + } + return env; +} + +async function writeJson(filePath: string, value: unknown): Promise { + await fsp.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isStringRecord(value: unknown): value is Record { + return isRecord(value) && Object.values(value).every((item) => typeof item === "string"); +} diff --git a/packages/evals/framework/harnesses/fxAdapter.ts b/packages/evals/framework/harnesses/fxAdapter.ts new file mode 100644 index 000000000..beef788a0 --- /dev/null +++ b/packages/evals/framework/harnesses/fxAdapter.ts @@ -0,0 +1,144 @@ +import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { + FxEvent, + FxToolCallRecord, + FxToolResultRecord, +} from "@browserbasehq/stagehand-integrations-fx-sdk"; +import type { StepObservation } from "../observationRecorder.js"; +import { + buildTrajectory, + type NormalizedToolCall, + type TrajectoryAdapter, +} from "./trajectoryAdapter.js"; + +export interface FxRunResult { + events: FxEvent[]; + finalAnswer?: string; + status?: Trajectory["status"]; + usage?: Partial; + finalObservation?: ProbeEvidence; + stepObservations?: StepObservation[]; + observedToolName?: (name: string) => boolean; +} + +export class FxTrajectoryAdapter implements TrajectoryAdapter { + fromHarnessResult(result: FxRunResult, taskSpec: TaskSpec): Trajectory { + const toolCalls: NormalizedToolCall[] = []; + let latestAgentMessage: string | undefined; + + for (const event of result.events) { + if (event.type === "assistant") { + latestAgentMessage = event.text; + continue; + } + if (event.type !== "tool_step") continue; + + const resultsById = new Map(); + for (const toolResult of event.tool_results) { + if (typeof toolResult.tool_call_id === "string") { + resultsById.set(toolResult.tool_call_id, toolResult); + } + } + event.tool_calls.forEach((call, index) => { + const toolResult = typeof call.id === "string" ? resultsById.get(call.id) : undefined; + toolCalls.push(normalizeFxToolCall(call, toolResult, index === 0 ? event.assistant : "")); + }); + } + + pairStepObservations(toolCalls, result); + + return buildTrajectory({ + taskSpec, + toolCalls, + finalAnswer: result.finalAnswer ?? latestAgentMessage, + status: result.status ?? "complete", + usage: result.usage, + ...(result.finalObservation?.screenshot && { + finalObservation: result.finalObservation, + }), + }); + } +} + +export const fxAdapter = new FxTrajectoryAdapter(); + +function normalizeFxToolCall( + call: FxToolCallRecord, + toolResult: FxToolResultRecord | undefined, + reasoning: string, +): NormalizedToolCall { + const args = parseArgs(call.arguments_json); + const output = typeof toolResult?.output === "string" ? toolResult.output : ""; + const images: Array<{ bytes: Buffer; mediaType: string }> = []; + const parsedOutput = tryParseJson(output); + const result = parsedOutput === undefined ? output : replaceImageBlocks(parsedOutput, images); + const ok = toolResult?.status === "success"; + return { + name: typeof call.name === "string" ? call.name : "unknown_tool", + args, + result, + ok, + ...(!ok && output && { error: clip(output, 500) }), + ...(reasoning && { reasoning }), + ...(images.length > 0 && { images }), + }; +} + +function parseArgs(value: unknown): Record { + if (typeof value !== "string") return {}; + const parsed = tryParseJson(value); + return isRecord(parsed) ? parsed : { raw: value }; +} + +function replaceImageBlocks( + value: unknown, + images: Array<{ bytes: Buffer; mediaType: string }>, +): unknown { + if (Array.isArray(value)) return value.map((item) => replaceImageBlocks(item, images)); + if (!isRecord(value)) return value; + if ( + value.type === "image" && + typeof value.data === "string" && + typeof value.mimeType === "string" + ) { + images.push({ bytes: Buffer.from(value.data, "base64"), mediaType: value.mimeType }); + return "[image]"; + } + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, replaceImageBlocks(item, images)]), + ); +} + +function pairStepObservations(toolCalls: NormalizedToolCall[], result: FxRunResult): void { + const observations = result.stepObservations ?? []; + if (observations.length === 0) return; + const observedCalls = toolCalls.filter((call) => + result.observedToolName ? result.observedToolName(call.name) : call.name.startsWith("mcp_"), + ); + const totalObservedRuns = + Math.max(...observations.map((observation) => observation.runIndex)) + 1; + if (observedCalls.length !== totalObservedRuns) return; + const observationsByRunIndex = new Map( + observations.map((observation) => [observation.runIndex, observation.evidence]), + ); + observedCalls.forEach((call, ordinal) => { + const observation = observationsByRunIndex.get(ordinal); + if (observation) call.probeEvidence = observation; + }); +} + +function tryParseJson(value: string): unknown { + try { + return JSON.parse(value); + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/evals/package.json b/packages/evals/package.json index ab445ece6..6adb30469 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -30,6 +30,9 @@ "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-mastra-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-pi-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-deepagents-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", "ai": "^5.0.133", "browse": "0.9.5", "dotenv": "^17.3.1", diff --git a/packages/evals/tests/framework/fxAdapter.test.ts b/packages/evals/tests/framework/fxAdapter.test.ts new file mode 100644 index 000000000..619e7de02 --- /dev/null +++ b/packages/evals/tests/framework/fxAdapter.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; +import type { FxEvent } from "@browserbasehq/stagehand-integrations-fx-sdk"; +import type { TaskSpec } from "stagehand-v3"; +import { fxAdapter } from "../../framework/harnesses/fxAdapter.js"; + +const taskSpec = { + id: "fx-test", + instruction: "Inspect the page", +} as TaskSpec; + +describe("fx trajectory adapter", () => { + it("normalizes calls, failures, reasoning, images, and the latest assistant", () => { + const imageData = Buffer.from("pixels").toString("base64"); + const events: FxEvent[] = [ + { + type: "tool_step", + assistant: "I will inspect and then click.", + tool_calls: [ + { + id: "call-1", + name: "mcp_stagehand_snapshot", + arguments_json: '{"depth":2}', + }, + { + id: "call-2", + name: "mcp_stagehand_run", + arguments_json: "not-json", + }, + ], + tool_results: [ + { + tool_call_id: "call-1", + status: "success", + output: JSON.stringify({ + content: [ + { type: "text", text: "snapshot" }, + { type: "image", data: imageData, mimeType: "image/jpeg" }, + ], + }), + }, + { + tool_call_id: "call-2", + status: "failure", + output: "click failed", + }, + ], + }, + { type: "assistant", text: "first" }, + { type: "assistant", text: "last" }, + ]; + + const trajectory = fxAdapter.fromHarnessResult( + { + events, + usage: { + input_tokens: 10, + output_tokens: 4, + cached_input_tokens: 3, + reasoning_tokens: 2, + }, + }, + taskSpec, + ); + + expect(trajectory.steps).toHaveLength(2); + expect(trajectory.steps[0]).toMatchObject({ + actionName: "mcp_stagehand_snapshot", + actionArgs: { depth: 2 }, + reasoning: "I will inspect and then click.", + toolOutput: { ok: true }, + }); + expect(trajectory.steps[0]?.toolOutput.result).toEqual({ + content: [{ type: "text", text: "snapshot" }, "[image]"], + }); + expect(trajectory.steps[0]?.agentEvidence.modalities).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "json" }), + expect.objectContaining({ type: "image", mediaType: "image/jpeg" }), + ]), + ); + expect(trajectory.steps[1]).toMatchObject({ + actionName: "mcp_stagehand_run", + actionArgs: { raw: "not-json" }, + reasoning: "", + toolOutput: { ok: false, result: "click failed", error: "click failed" }, + }); + expect(trajectory.finalAnswer).toBe("last"); + expect(trajectory.usage).toMatchObject({ + input_tokens: 10, + output_tokens: 4, + cached_input_tokens: 3, + reasoning_tokens: 2, + }); + }); + + it("pairs step observations only when observed call counts align", () => { + const events: FxEvent[] = [ + { + type: "tool_step", + assistant: "", + tool_calls: [ + { id: "one", name: "mcp_stagehand_run", arguments_json: "{}" }, + { id: "two", name: "read_file", arguments_json: "{}" }, + { id: "three", name: "mcp_stagehand_snapshot", arguments_json: "{}" }, + ], + tool_results: [ + { tool_call_id: "one", status: "success", output: "one" }, + { tool_call_id: "two", status: "success", output: "two" }, + { tool_call_id: "three", status: "success", output: "three" }, + ], + }, + ]; + const trajectory = fxAdapter.fromHarnessResult( + { + events, + observedToolName: (name) => name.startsWith("mcp_stagehand_"), + stepObservations: [ + { runIndex: 0, evidence: { url: "https://example.com/one" } }, + { runIndex: 1, evidence: { url: "https://example.com/two" } }, + ], + }, + taskSpec, + ); + expect(trajectory.steps[0]?.probeEvidence).toEqual({ url: "https://example.com/one" }); + expect(trajectory.steps[1]?.probeEvidence).toEqual({}); + expect(trajectory.steps[2]?.probeEvidence).toEqual({ url: "https://example.com/two" }); + }); + + it("includes a final observation only when it has a screenshot", () => { + const withoutScreenshot = fxAdapter.fromHarnessResult( + { events: [], finalObservation: { url: "https://example.com" } }, + taskSpec, + ); + const screenshot = Buffer.from("png"); + const withScreenshot = fxAdapter.fromHarnessResult( + { events: [], finalObservation: { url: "https://example.com", screenshot } }, + taskSpec, + ); + expect(withoutScreenshot.finalObservation).toBeUndefined(); + expect(withScreenshot.finalObservation).toEqual({ + url: "https://example.com", + screenshot, + }); + }); +}); diff --git a/packages/evals/tests/framework/fxRunner.test.ts b/packages/evals/tests/framework/fxRunner.test.ts new file mode 100644 index 000000000..4dcd0431b --- /dev/null +++ b/packages/evals/tests/framework/fxRunner.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import type { AvailableModel } from "stagehand-v3"; +import { EvalLogger } from "../../logger.js"; +import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; +import { buildFxPrompt, parseFxResult, runFxAgent } from "../../framework/fxRunner.js"; + +const plan: ExternalHarnessTaskPlan = { + dataset: "webvoyager", + taskId: "wv-fx-1", + startUrl: "https://example.com", + instruction: "Find the heading", +}; + +describe("fx runner helpers", () => { + it("builds a browser task prompt with structured result instructions", () => { + const prompt = buildFxPrompt(plan, "Use mcp_stagehand_snapshot."); + expect(prompt).toContain("Dataset: webvoyager"); + expect(prompt).toContain("Task ID: wv-fx-1"); + expect(prompt).toContain("Start URL: https://example.com"); + expect(prompt).toContain("Find the heading"); + expect(prompt).toContain("mcp_stagehand_snapshot"); + expect(prompt).toContain('"success": boolean'); + }); + + it("parses direct and marker JSON results", () => { + expect(parseFxResult('{"success":true,"summary":"done","finalAnswer":"Example"}')).toEqual({ + success: true, + summary: "done", + finalAnswer: "Example", + raw: '{"success":true,"summary":"done","finalAnswer":"Example"}', + }); + expect( + parseFxResult('assistant text\nEVAL_RESULT: {"success":true,"summary":"done"}'), + ).toMatchObject({ success: true, summary: "done" }); + }); + + it("runs a fake fx session into a successful task result", async () => { + const finalOutput = '{"success":true,"summary":"done","finalAnswer":"Example Domain"}'; + const events = JSON.stringify({ + kind: "history_turn_committed", + payload: { + total_input_tokens: 42, + total_output_tokens: 8, + turn: { + kind: "completed", + assistant: finalOutput, + terminal_reason: "completed", + execution: { schema_version: 3, tool_steps: [] }, + }, + }, + }); + const result = await runFxAgent({ + plan, + model: "openai/gpt-5.6-sol" as AvailableModel, + logger: new EvalLogger(false), + toolAdapter: { + toolSurface: "stagehand_facade", + startupProfile: "tool_launch_local", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: { PATH: "/bin" }, + promptInstructions: "Use mcp_stagehand_snapshot.", + mcpServerNames: ["stagehand"], + cleanup: async () => {}, + }, + runProcess: async ({ args, stdin }) => { + expect(args).toEqual(["ask", "--json", "--auto"]); + expect(stdin).toContain("Find the heading"); + return { + stdout: JSON.stringify({ output: finalOutput, exit_code: 0, session_id: "fx-1" }), + stderr: "", + exitCode: 0, + }; + }, + store: { + waitForSessionDir: async () => "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/fake/session", + readEventsJsonl: async () => events, + readUsageSnapshot: async () => ({ + snapshot: { + input_tokens: 42, + output_tokens: 8, + cache_read_tokens: 5, + reasoning_tokens: 2, + }, + }), + }, + }); + const metrics = result.metrics as Record; + expect(result._success).toBe(true); + expect(result.error).toBeUndefined(); + expect(result.fxStatus).toBe("completed"); + expect(result.finalAnswer).toBe("Example Domain"); + expect(metrics.fx_input_tokens.value).toBe(42); + }); +}); diff --git a/packages/evals/tests/framework/fxToolAdapter.test.ts b/packages/evals/tests/framework/fxToolAdapter.test.ts new file mode 100644 index 000000000..973f2d956 --- /dev/null +++ b/packages/evals/tests/framework/fxToolAdapter.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import type { ToolSurface } from "../../core/contracts/tool.js"; +import { + buildFxAgentsMarkdown, + buildFxMcpConfig, + buildFxSettings, + resolveFxStartupProfile, + resolveFxToolSurface, +} from "../../framework/fxToolAdapter.js"; + +describe("fx tool adapter helpers", () => { + it("defaults to the Stagehand facade and rejects unsupported surfaces", () => { + expect(resolveFxToolSurface()).toBe("stagehand_facade"); + expect(resolveFxToolSurface("playwright_mcp")).toBe("playwright_mcp"); + expect(resolveFxToolSurface("chrome_devtools_mcp")).toBe("chrome_devtools_mcp"); + expect(() => resolveFxToolSurface("browse_cli")).toThrow( + 'fx harness supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "browse_cli".', + ); + }); + + it("chooses surface-specific startup profiles", () => { + expect(resolveFxStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); + expect(resolveFxStartupProfile("stagehand_facade", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); + expect(resolveFxStartupProfile("playwright_mcp", "LOCAL")).toBe("runner_provided_local_cdp"); + expect(resolveFxStartupProfile("chrome_devtools_mcp", "BROWSERBASE")).toBe( + "runner_provided_browserbase_cdp", + ); + expect(resolveFxStartupProfile("playwright_mcp", "LOCAL", "tool_attach_local_cdp")).toBe( + "tool_attach_local_cdp", + ); + }); + + it("builds fx MCP launch specs with an explicit child environment", () => { + expect( + buildFxMcpConfig( + { + stagehand: { + command: "/usr/bin/node", + args: ["server.mjs", "--flag"], + env: { BROWSERBASE_API_KEY: "test" }, + }, + }, + { home: "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/isolated/home", pathEnv: "/usr/bin:/bin" }, + ), + ).toEqual({ + mcp: { + stagehand: { + type: "stdio", + command: ["/usr/bin/node", "server.mjs", "--flag"], + environment: { + PATH: "/usr/bin:/bin", + HOME: "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/isolated/home", + BROWSERBASE_API_KEY: "test", + }, + required: true, + }, + }, + }); + expect(() => + buildFxMcpConfig({ "bad server": { command: "node" } }, { home: "/home", pathEnv: "/bin" }), + ).toThrow(/Invalid fx MCP server name/u); + }); + + it("denies shell and editing tools and narrowly allows the facade", () => { + expect(buildFxSettings(["stagehand"], "stagehand_facade")).toEqual({ + permission: { + run_command: "deny", + terminal: "deny", + write_file: "deny", + edit_file: "deny", + mcp_stagehand_run: "allow", + mcp_stagehand_snapshot: "allow", + mcp_stagehand_screenshot: "allow", + }, + }); + expect(buildFxSettings(["playwright"], "playwright_mcp").permission).toEqual({ + run_command: "deny", + terminal: "deny", + write_file: "deny", + edit_file: "deny", + }); + }); + + it("teaches exact fx MCP selection and Stagehand names", () => { + const markdown = buildFxAgentsMarkdown("Use snapshots first.", ["stagehand"]); + expect(markdown).toContain("mcp_select_tool"); + expect(markdown).toContain("mcp_search_tools may return nothing"); + expect(markdown).toContain("mcp_stagehand_run"); + expect(markdown).toContain("mcp_stagehand_snapshot"); + expect(markdown).toContain("mcp_stagehand_screenshot"); + expect(markdown).toContain("Use snapshots first."); + }); + + it("rejects a startup default for an unrelated surface", () => { + expect(() => resolveFxStartupProfile("browse_cli" as ToolSurface, "LOCAL")).toThrow( + /No fx startup profile default/u, + ); + }); +}); diff --git a/packages/integrations/fx-sdk/package.json b/packages/integrations/fx-sdk/package.json new file mode 100644 index 000000000..491b712d7 --- /dev/null +++ b/packages/integrations/fx-sdk/package.json @@ -0,0 +1,34 @@ +{ + "name": "@browserbasehq/stagehand-integrations-fx-sdk", + "version": "4.0.1", + "private": true, + "description": "fx CLI harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/fx-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/fx-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@browserbasehq/stagehand-integrations": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=22.18.0" + } +} diff --git a/packages/integrations/fx-sdk/src/index.ts b/packages/integrations/fx-sdk/src/index.ts new file mode 100644 index 000000000..9df91886b --- /dev/null +++ b/packages/integrations/fx-sdk/src/index.ts @@ -0,0 +1 @@ +export * from "./session.js"; diff --git a/packages/integrations/fx-sdk/src/session.ts b/packages/integrations/fx-sdk/src/session.ts new file mode 100644 index 000000000..84b37c7ae --- /dev/null +++ b/packages/integrations/fx-sdk/src/session.ts @@ -0,0 +1,645 @@ +import { spawn } from "node:child_process"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; + +export type FxToolCallRecord = { + id?: string; + name?: string; + arguments_json?: string; + provider_result?: unknown; + [key: string]: unknown; +}; + +export type FxToolResultRecord = { + tool_call_id?: string; + tool_name?: string; + status?: string; + output?: string; + truncated?: boolean; + [key: string]: unknown; +}; + +export type FxAskOutput = { + output?: string; + exit_code?: number; + model?: string; + session_id?: string; + steps?: number; + tool_calls?: Array>; + error?: string; + terminal_reason?: string; + [key: string]: unknown; +}; + +export type FxToolStep = { + assistant: string; + tool_calls: FxToolCallRecord[]; + tool_results: FxToolResultRecord[]; +}; + +export type FxLogEvent = { + kind?: string; + payload?: Record; + [key: string]: unknown; +}; + +export type FxEvent = + | { + type: "tool_step"; + assistant: string; + tool_calls: FxToolCallRecord[]; + tool_results: FxToolResultRecord[]; + } + | { type: "assistant"; text: string } + | { type: "ask_result"; ask: FxAskOutput } + | { type: "stderr"; line: string } + | { type: "turn_committed"; terminal_reason?: string; turn_kind?: string }; + +export type FxTokenUsage = { + input_tokens: number; + cached_input_tokens: number; + output_tokens: number; + reasoning_output_tokens: number; + total_cost?: number; +}; + +export type FxSessionResult = { + events: FxEvent[]; + finalMessage: string; + status: "completed" | "max_turns" | "sdk_error"; + stopReason?: string; + tokenUsage: FxTokenUsage; + sessionId?: string; + exitCode?: number; + iterationError?: unknown; +}; + +export type FxProcessRunner = (input: { + bin: string; + args: string[]; + cwd: string; + env: Record; + stdin: string; + signal: AbortSignal; + onStderrLine?: (line: string) => void; +}) => Promise<{ + stdout: string; + stderr: string; + exitCode: number | null; + signal?: string | null; +}>; + +export type FxSessionStore = { + waitForSessionDir(home: string, signal: AbortSignal): Promise; + readEventsJsonl(sessionDir: string): Promise; + readUsageSnapshot?(sessionDir: string): Promise | undefined>; +}; + +export const FX_BIN_ENV = "EVAL_FX_PATH"; + +export function resolveFxBin(override?: string): string { + return override ?? process.env[FX_BIN_ENV] ?? "fx"; +} + +export function normalizeFxModel(model: string): string | undefined { + return model === "fx/default" ? undefined : model; +} + +const defaultProcessRunner: FxProcessRunner = async (input) => + new Promise((resolve) => { + let stdout = ""; + let stderr = ""; + let settled = false; + let stderrRemainder = ""; + let killTimer: NodeJS.Timeout | undefined; + let child: ReturnType; + + const finish = (exitCode: number | null, signal?: string | null): void => { + if (settled) return; + settled = true; + if (killTimer) clearTimeout(killTimer); + input.signal.removeEventListener("abort", abort); + if (stderrRemainder) input.onStderrLine?.(stderrRemainder); + resolve({ stdout, stderr, exitCode, signal }); + }; + const abort = (): void => { + if (!child || settled) return; + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), 2_000); + killTimer.unref(); + }; + + try { + child = spawn(input.bin, input.args, { + cwd: input.cwd, + env: input.env, + stdio: ["pipe", "pipe", "pipe"], + }); + child.stdout.on("data", (chunk: Buffer | string) => { + stdout += String(chunk); + }); + child.stderr.on("data", (chunk: Buffer | string) => { + const text = String(chunk); + stderr += text; + const lines = `${stderrRemainder}${text}`.split(/\r?\n/u); + stderrRemainder = lines.pop() ?? ""; + for (const line of lines) input.onStderrLine?.(line); + }); + child.once("error", (error) => { + stderr += `${stderr ? "\n" : ""}${stringifyError(error)}`; + finish(null); + }); + child.once("close", (exitCode, signal) => finish(exitCode, signal)); + input.signal.addEventListener("abort", abort, { once: true }); + if (input.signal.aborted) abort(); + child.stdin.end(input.stdin); + } catch (error) { + stderr += stringifyError(error); + finish(null); + } + }); + +const defaultSessionStore: FxSessionStore = { + async waitForSessionDir(home, signal) { + if (signal.aborted) return undefined; + const sessionsRoot = path.join(home, ".fx", "sessions"); + try { + const entries = await fsp.readdir(sessionsRoot, { withFileTypes: true }); + const candidates = entries + .filter( + (entry) => + entry.isDirectory() && + entry.name !== "latest" && + entry.name !== "index.pending" && + !entry.name.endsWith(".lock"), + ) + .map((entry) => entry.name) + .sort() + .reverse(); + return candidates[0] ? path.join(sessionsRoot, candidates[0]) : undefined; + } catch { + return undefined; + } + }, + async readEventsJsonl(sessionDir) { + try { + return await fsp.readFile(path.join(sessionDir, "events.jsonl"), "utf8"); + } catch { + return ""; + } + }, + async readUsageSnapshot(sessionDir) { + try { + const text = await fsp.readFile(path.join(sessionDir, "usage-v2.json"), "utf8"); + const parsed: unknown = JSON.parse(text); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } + }, +}; + +export async function runFxSession(input: { + prompt: string; + model?: string; + bin?: string; + cwd: string; + home: string; + env: Record; + permissionMode?: "auto" | "yolo"; + maxAgentSteps?: number; + signal?: AbortSignal; + logger: HarnessLogger; + runProcess?: FxProcessRunner; + store?: FxSessionStore; + onToolStep?: (call: FxToolCallRecord) => void | Promise; + observedTool?: (name: string) => boolean; + pollIntervalMs?: number; +}): Promise { + if (!input.cwd) throw new HarnessAdapterError("fx session requires cwd."); + if (!input.home) throw new HarnessAdapterError("fx session requires home."); + + const events: FxEvent[] = []; + const permissionMode = input.permissionMode ?? "auto"; + const args = ["ask", "--json", permissionMode === "yolo" ? "--yolo" : "--auto"]; + const model = input.model ? normalizeFxModel(input.model) : undefined; + const env: Record = { + ...input.env, + HOME: input.home, + ...(model && { FX_MODEL: model }), + ...(positiveInteger(input.maxAgentSteps) && { + FX_MAX_AGENT_STEPS: String(Math.floor(input.maxAgentSteps!)), + }), + FX_PERMISSION_MODE: permissionMode, + FX_SKIP_ONBOARDING: "1", + FX_AUTO_UPGRADE: "0", + FX_NO_OPEN_BROWSER: "1", + NO_COLOR: "1", + }; + const controller = new AbortController(); + const forwardAbort = (): void => controller.abort(input.signal?.reason); + if (input.signal) { + if (input.signal.aborted) controller.abort(input.signal.reason); + else input.signal.addEventListener("abort", forwardAbort, { once: true }); + } + + const store = input.store ?? defaultSessionStore; + const seenToolCalls = new Set(); + const observedTool = input.observedTool ?? ((name: string) => name.startsWith("mcp_")); + let sessionDir: string | undefined; + let processSettled = false; + let processResult: Awaited>; + + const notifyCalls = async (steps: FxToolStep[]): Promise => { + if (!input.onToolStep) return; + for (const step of steps) { + for (const call of step.tool_calls) { + const id = typeof call.id === "string" ? call.id : undefined; + const name = typeof call.name === "string" ? call.name : ""; + const key = id ?? `${name}:${call.arguments_json ?? ""}`; + if (!observedTool(name) || seenToolCalls.has(key)) continue; + seenToolCalls.add(key); + try { + await input.onToolStep(call); + } catch { + // Live observation is best-effort and must never fail an fx run. + } + } + } + }; + + try { + const processPromise = (input.runProcess ?? defaultProcessRunner)({ + bin: resolveFxBin(input.bin), + args, + cwd: input.cwd, + env, + stdin: input.prompt, + signal: controller.signal, + }).finally(() => { + processSettled = true; + }); + + if (input.onToolStep) { + // fx does not stream tool events. Tail its recovery checkpoints while + // the process is alive, then reconcile against the committed turn. + while (!processSettled && !controller.signal.aborted) { + sessionDir ??= await store + .waitForSessionDir(input.home, controller.signal) + .catch(() => undefined); + if (sessionDir) { + const liveText = await store.readEventsJsonl(sessionDir).catch(() => ""); + await notifyCalls(extractFxToolSteps(parseFxEventsJsonl(liveText))); + } + if (!processSettled) { + await delay(input.pollIntervalMs ?? 500, controller.signal); + } + } + } + processResult = await processPromise; + } catch (error) { + processResult = { stdout: "", stderr: stringifyError(error), exitCode: null }; + } finally { + input.signal?.removeEventListener("abort", forwardAbort); + } + + for (const rawLine of processResult.stderr.split(/\r?\n/u)) { + if (!rawLine) continue; + const line = sanitizeErrorMessage(rawLine); + const event: FxEvent = { type: "stderr", line }; + events.push(event); + logFxEvent(input.logger, event); + } + + const ask = parseFxAskOutput(processResult.stdout); + sessionDir ??= await store + .waitForSessionDir(input.home, controller.signal) + .catch(() => undefined); + const logEvents = sessionDir + ? parseFxEventsJsonl(await store.readEventsJsonl(sessionDir).catch(() => "")) + : []; + const toolSteps = extractFxToolSteps(logEvents); + await notifyCalls(toolSteps); + for (const step of toolSteps) { + const event: FxEvent = { type: "tool_step", ...step }; + events.push(event); + logFxEvent(input.logger, event); + } + + const committed = findLastCommittedTurn(logEvents); + const turn = committed?.turn; + const turnAssistant = typeof turn?.assistant === "string" ? turn.assistant : undefined; + const finalMessage = typeof ask?.output === "string" ? ask.output : (turnAssistant ?? ""); + if (turnAssistant || finalMessage) { + const event: FxEvent = { type: "assistant", text: turnAssistant ?? finalMessage }; + events.push(event); + logFxEvent(input.logger, event); + } + const terminalReason = + typeof turn?.terminal_reason === "string" + ? turn.terminal_reason + : typeof ask?.terminal_reason === "string" + ? ask.terminal_reason + : undefined; + if (committed) { + const event: FxEvent = { + type: "turn_committed", + ...(terminalReason && { terminal_reason: terminalReason }), + ...(typeof turn?.kind === "string" && { turn_kind: turn.kind }), + }; + events.push(event); + logFxEvent(input.logger, event); + } + if (ask) { + const event: FxEvent = { type: "ask_result", ask }; + events.push(event); + logFxEvent(input.logger, event); + } + + const usageSnapshot = + sessionDir && store.readUsageSnapshot + ? await store.readUsageSnapshot(sessionDir).catch(() => undefined) + : undefined; + const tokenUsage = extractFxTokenUsage(logEvents, usageSnapshot); + const aborted = input.signal?.aborted === true; + const resolution = resolveFxStatus({ + exitCode: processResult.exitCode, + signal: processResult.signal, + ask, + terminalReason, + aborted, + stderr: processResult.stderr, + }); + const stopReason = resolution.stopReason + ? sanitizeErrorMessage(resolution.stopReason) + : undefined; + let iterationError: unknown; + if (resolution.status !== "completed") { + iterationError = new Error(stopReason ?? "fx stopped before a normal result"); + input.logger.warn({ + category: "fx", + message: `fx stopped before a normal result: ${stopReason ?? "unknown error"}`, + level: 0, + auxiliary: { + error: { value: stopReason ?? "unknown error", type: "string" }, + }, + }); + } + + return { + events, + finalMessage, + status: resolution.status, + ...(stopReason && { stopReason }), + tokenUsage, + ...(typeof ask?.session_id === "string" && { sessionId: ask.session_id }), + ...(processResult.exitCode !== null && { exitCode: processResult.exitCode }), + ...(iterationError !== undefined && { iterationError }), + }; +} + +export function parseFxAskOutput(stdout: string): FxAskOutput | undefined { + if (!stdout.trim()) return undefined; + try { + const parsed: unknown = JSON.parse(stdout.trim()); + return isRecord(parsed) ? (parsed as FxAskOutput) : undefined; + } catch { + return undefined; + } +} + +export function parseFxEventsJsonl(text: string): FxLogEvent[] { + const events: FxLogEvent[] = []; + for (const line of text.split(/\r?\n/u)) { + if (!line.trim()) continue; + try { + const parsed: unknown = JSON.parse(line); + if (isRecord(parsed)) events.push(parsed as FxLogEvent); + } catch { + // A partially written final line is normal while tailing events.jsonl. + } + } + return events; +} + +export function extractFxToolSteps(events: FxLogEvent[]): FxToolStep[] { + let checkpointSteps: FxToolStep[] = []; + let committedSteps: FxToolStep[] | undefined; + for (const event of events) { + const payload = isRecord(event.payload) ? event.payload : undefined; + if (event.kind === "recovery_checkpoint_set") { + const checkpoint = isRecord(payload?.checkpoint) ? payload.checkpoint : undefined; + const execution = isRecord(checkpoint?.execution) ? checkpoint.execution : undefined; + checkpointSteps = readToolSteps(execution?.tool_steps); + } else if (event.kind === "history_turn_committed") { + const turn = isRecord(payload?.turn) ? payload.turn : undefined; + const execution = isRecord(turn?.execution) ? turn.execution : undefined; + committedSteps = readToolSteps(execution?.tool_steps); + } + } + return committedSteps ?? checkpointSteps; +} + +export function extractFxTokenUsage( + events: FxLogEvent[], + usageSnapshot?: Record, +): FxTokenUsage { + const snapshot = isRecord(usageSnapshot?.snapshot) ? usageSnapshot.snapshot : usageSnapshot; + if (snapshot && hasUsageFields(snapshot)) return usageFromRecord(snapshot); + + const committed = findLastCommittedTurn(events); + if (committed) { + return { + input_tokens: toFiniteNumber(committed.payload.total_input_tokens), + cached_input_tokens: 0, + output_tokens: toFiniteNumber(committed.payload.total_output_tokens), + reasoning_output_tokens: 0, + }; + } + + let lastUsage: Record | undefined; + for (const event of events) { + if (event.kind !== "usage_checkpointed" || !isRecord(event.payload)) continue; + if (isRecord(event.payload.usage)) lastUsage = event.payload.usage; + } + return usageFromRecord(lastUsage); +} + +export function resolveFxStatus(input: { + exitCode: number | null; + signal?: string | null; + ask?: FxAskOutput; + terminalReason?: string; + aborted?: boolean; + stderr?: string; +}): { status: "completed" | "max_turns" | "sdk_error"; stopReason?: string } { + if (input.aborted) return { status: "sdk_error", stopReason: "aborted" }; + if (input.exitCode === 130 || input.signal) { + return { status: "sdk_error", stopReason: "interrupted" }; + } + const error = typeof input.ask?.error === "string" ? input.ask.error : undefined; + if ( + input.terminalReason === "step_limit" || + input.terminalReason === "step_limit_reached" || + (error && /step.?limit/iu.test(error)) + ) { + return { status: "max_turns", stopReason: error ?? input.terminalReason }; + } + if (input.exitCode === 0 && !error) return { status: "completed" }; + if (!input.ask) { + const stderr = input.stderr?.trim(); + return { + status: "sdk_error", + stopReason: `fx produced no JSON output${stderr ? `: ${clip(stderr, 500)}` : ""}`, + }; + } + return { + status: "sdk_error", + stopReason: error ?? `fx exited with code ${input.exitCode ?? "unknown"}`, + }; +} + +export function buildFxTranscript(events: FxEvent[]): string { + return events + .map((event) => summarizeFxEvent(event).detail) + .filter((detail): detail is string => Boolean(detail)) + .join("\n"); +} + +export function logFxEvent(logger: HarnessLogger, event: FxEvent): void { + const summary = summarizeFxEvent(event); + logger.log({ + category: "fx", + message: summary.message, + level: 1, + auxiliary: { + type: { value: event.type, type: "string" }, + ...(summary.detail && { detail: { value: summary.detail, type: "string" } }), + }, + }); +} + +export function summarizeFxEvent(event: FxEvent): { message: string; detail?: string } { + if (event.type === "assistant") { + return { message: `assistant: ${clip(event.text, 500)}`, detail: event.text }; + } + if (event.type === "tool_step") { + const names = event.tool_calls.map((call) => String(call.name ?? "tool")).join(", "); + return { message: `tools: ${names}`, detail: safeJson(event) }; + } + if (event.type === "stderr") { + return { message: `stderr: ${clip(event.line, 500)}`, detail: event.line }; + } + if (event.type === "turn_committed") { + return { + message: `turn committed: ${event.terminal_reason ?? event.turn_kind ?? "unknown"}`, + detail: safeJson(event), + }; + } + return { message: "ask result", detail: safeJson(event.ask) }; +} + +function readToolSteps(value: unknown): FxToolStep[] { + if (!Array.isArray(value)) return []; + return value.filter(isRecord).map((step) => ({ + assistant: typeof step.assistant === "string" ? step.assistant : "", + tool_calls: Array.isArray(step.tool_calls) + ? step.tool_calls.filter(isRecord).map((call) => call as FxToolCallRecord) + : [], + tool_results: Array.isArray(step.tool_results) + ? step.tool_results.filter(isRecord).map((result) => result as FxToolResultRecord) + : [], + })); +} + +function findLastCommittedTurn( + events: FxLogEvent[], +): { payload: Record; turn?: Record } | undefined { + let found: { payload: Record; turn?: Record } | undefined; + for (const event of events) { + if (event.kind !== "history_turn_committed" || !isRecord(event.payload)) continue; + found = { + payload: event.payload, + ...(isRecord(event.payload.turn) && { turn: event.payload.turn }), + }; + } + return found; +} + +function hasUsageFields(record: Record): boolean { + return [ + "input_tokens", + "output_tokens", + "cache_read_tokens", + "reasoning_tokens", + "total_cost", + ].some((key) => key in record); +} + +function usageFromRecord(record?: Record): FxTokenUsage { + return { + input_tokens: toFiniteNumber(record?.input_tokens), + cached_input_tokens: toFiniteNumber(record?.cache_read_tokens), + output_tokens: toFiniteNumber(record?.output_tokens), + reasoning_output_tokens: toFiniteNumber(record?.reasoning_tokens), + ...(record && + "total_cost" in record && { + total_cost: toFiniteNumber(record.total_cost), + }), + }; +} + +function positiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value > 0; +} + +function delay(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, ms); + function done(): void { + clearTimeout(timer); + signal.removeEventListener("abort", done); + resolve(); + } + signal.addEventListener("abort", done, { once: true }); + }); +} + +export function toFiniteNumber(value: unknown): number { + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && value.trim() + ? Number(value) + : 0; + return Number.isFinite(parsed) ? parsed : 0; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +export function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +export function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + return safeJson(value) ?? Object.prototype.toString.call(value); +} + +export function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/integrations/fx-sdk/tests/session.test.ts b/packages/integrations/fx-sdk/tests/session.test.ts new file mode 100644 index 000000000..b7d3fd7fa --- /dev/null +++ b/packages/integrations/fx-sdk/tests/session.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it, vi } from "vitest"; +import { + normalizeFxModel, + runFxSession, + type FxProcessRunner, + type FxSessionStore, +} from "../src/index.js"; + +const logger = { log: () => {}, warn: () => {}, error: () => {} }; + +function jsonl(...events: unknown[]): string { + return events.map((event) => JSON.stringify(event)).join("\n"); +} + +function committedEvent(terminalReason = "completed") { + return { + kind: "history_turn_committed", + payload: { + total_input_tokens: 11, + total_output_tokens: 4, + turn: { + kind: "completed", + assistant: "turn answer", + terminal_reason: terminalReason, + execution: { + schema_version: 3, + tool_steps: [ + { + assistant: "I will inspect the page.", + tool_calls: [ + { + id: "call-1", + name: "mcp_stagehand_snapshot", + arguments_json: "{}", + provider_result: null, + }, + ], + tool_results: [ + { + tool_call_id: "call-1", + tool_name: "mcp_stagehand_snapshot", + status: "success", + output: "snapshot", + truncated: false, + }, + ], + }, + ], + }, + }, + }, + }; +} + +function fakeStore(events: string, usage?: Record): FxSessionStore { + return { + waitForSessionDir: async () => "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/fake/session", + readEventsJsonl: async () => events, + readUsageSnapshot: async () => usage, + }; +} + +describe("fx CLI session", () => { + it("runs ask through stdin and reconstructs the committed turn", async () => { + let captured: Parameters[0] | undefined; + const runProcess: FxProcessRunner = async (input) => { + captured = input; + return { + stdout: JSON.stringify({ + output: '{"success":true,"summary":"done","finalAnswer":"ok"}', + exit_code: 0, + session_id: "session-1", + }), + stderr: "progress", + exitCode: 0, + }; + }; + const result = await runFxSession({ + prompt: "do the task", + model: "openai/gpt-5.6-sol", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: { PATH: "/bin" }, + maxAgentSteps: 17, + logger, + runProcess, + store: fakeStore(jsonl(committedEvent()), { + snapshot: { + input_tokens: 100, + output_tokens: 20, + cache_read_tokens: 30, + reasoning_tokens: 5, + total_cost: 0.25, + }, + }), + }); + + expect(captured?.args).toEqual(["ask", "--json", "--auto"]); + expect(captured?.stdin).toBe("do the task"); + expect(captured?.env).toMatchObject({ + HOME: "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/fake/home", + FX_MODEL: "openai/gpt-5.6-sol", + FX_MAX_AGENT_STEPS: "17", + FX_PERMISSION_MODE: "auto", + }); + expect(result.events.map((event) => event.type)).toEqual([ + "stderr", + "tool_step", + "assistant", + "turn_committed", + "ask_result", + ]); + expect(result.tokenUsage).toEqual({ + input_tokens: 100, + cached_input_tokens: 30, + output_tokens: 20, + reasoning_output_tokens: 5, + total_cost: 0.25, + }); + expect(result.status).toBe("completed"); + expect(result.finalMessage).toContain('"success":true'); + }); + + it("reports missing credentials as an SDK error", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ error: "MissingCredentials", exit_code: 1 }), + stderr: "", + exitCode: 1, + }), + store: fakeStore(""), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toBe("MissingCredentials"); + }); + + it("maps a committed step limit to max_turns", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ output: "stopped", exit_code: 0 }), + stderr: "", + exitCode: 0, + }), + store: fakeStore(jsonl(committedEvent("step_limit_reached"))), + }); + expect(result.status).toBe("max_turns"); + }); + + it("reports output that is not JSON", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ stdout: "not json", stderr: "bad output", exitCode: 1 }), + store: fakeStore(""), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("fx produced no JSON output"); + }); + + it("deduplicates observed MCP tool calls and ignores built-in tools", async () => { + const onToolStep = vi.fn(); + const recovery = { + kind: "recovery_checkpoint_set", + payload: { + checkpoint: { + execution: { + tool_steps: [ + { + assistant: "inspect", + tool_calls: [ + { id: "call-1", name: "mcp_stagehand_snapshot", arguments_json: "{}" }, + { id: "call-2", name: "read_file", arguments_json: "{}" }, + ], + tool_results: [], + }, + ], + }, + }, + }, + }; + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + pollIntervalMs: 1, + onToolStep, + runProcess: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return { stdout: JSON.stringify({ output: "done" }), stderr: "", exitCode: 0 }; + }, + store: fakeStore(jsonl(recovery, committedEvent())), + }); + expect(result.status).toBe("completed"); + expect(onToolStep).toHaveBeenCalledTimes(1); + expect(onToolStep.mock.calls[0]?.[0]).toMatchObject({ id: "call-1" }); + }); + + it("forwards aborts to the process and reports aborted", async () => { + const controller = new AbortController(); + const runProcess: FxProcessRunner = async ({ signal }) => { + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }); + expect(signal.aborted).toBe(true); + return { stdout: "", stderr: "", exitCode: null, signal: "SIGTERM" }; + }; + const pending = runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + signal: controller.signal, + runProcess, + store: fakeStore(""), + }); + controller.abort(); + const result = await pending; + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toBe("aborted"); + }); + + it("normalizes only the fx default model", () => { + expect(normalizeFxModel("fx/default")).toBeUndefined(); + expect(normalizeFxModel("anthropic/claude-sonnet-4.5")).toBe("anthropic/claude-sonnet-4.5"); + }); + + it("sanitizes stop reasons", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ error: "failed with bb_live_abcd1234567890" }), + stderr: "", + exitCode: 1, + }), + store: fakeStore(""), + }); + expect(result.stopReason).not.toContain("1234567890"); + expect(result.stopReason).toContain("[redacted]"); + }); +}); diff --git a/packages/integrations/fx-sdk/tsconfig.json b/packages/integrations/fx-sdk/tsconfig.json new file mode 100644 index 000000000..6bddd941a --- /dev/null +++ b/packages/integrations/fx-sdk/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts", "vitest.config.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/integrations/fx-sdk/tsdown.config.ts b/packages/integrations/fx-sdk/tsdown.config.ts new file mode 100644 index 000000000..3b3b7c3bf --- /dev/null +++ b/packages/integrations/fx-sdk/tsdown.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + }, + format: ["esm"], + platform: "node", + target: "node22", + dts: { + sourcemap: true, + }, + sourcemap: true, + outDir: "dist", +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a229e5929..666ac4c96 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,3 +1,202 @@ +--- +lockfileVersion: '9.0' + +importers: + + .: + configDependencies: {} + packageManagerDependencies: + '@pnpm/exe': + specifier: 11.10.0 + version: 11.10.0 + pnpm: + specifier: 11.10.0 + version: 11.10.0 + +packages: + + '@pnpm/exe@11.10.0': + resolution: {integrity: sha512-mrmfi2C7LpZkyq0voKKye6MzrK8/K7tYQRiSB/jqOiwnFtQxxcA3xGTY9cEsrGpNl2ClPecQofLuj+BO8AIsxw==} + hasBin: true + + '@pnpm/linux-arm64@11.10.0': + resolution: {integrity: sha512-NbvDeUfs0SJuli9OPvgVvmnlbo2DvJ861XGXKrzgLu5AuTnLDLXgbZEEUd8mJ5I0YNrqOVXSWVfWEqiAazWzPA==} + cpu: [arm64] + os: [linux] + + '@pnpm/linux-x64@11.10.0': + resolution: {integrity: sha512-kdgb8BXZ/3XQ0x2cOmgLmsij7+SUIqd1bcV6OZhdGzQiDrOY6FAPrR+Y2Bp+NjrrhjzMHVm5pZrrmdeC67ymSQ==} + cpu: [x64] + os: [linux] + + '@pnpm/linuxstatic-arm64@11.10.0': + resolution: {integrity: sha512-JE1WrSyKGvqGQgWzqrMXn//ehedpiRax3hi1oP+v6mhvcnlJD1lpfXsjSfIFl9weTWC5KVtFbxSNKbKWfP+v6g==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@pnpm/linuxstatic-x64@11.10.0': + resolution: {integrity: sha512-1TBZVRkWb78GnsusIfVgwz20MOOW0fyehW+qLL3MxE9vT0IedNWsAhe3KWXgEvtC4M7NtMt55uQQJm+1egwBkA==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@pnpm/macos-arm64@11.10.0': + resolution: {integrity: sha512-94AVpPixBqyNT6SHYvIKFb1bfaHR5vxBzZsiFJahNSlGkgyPrgzmcqDiloZ/Jl+zxJd8L5PU1ddP0Q9PZnRqlA==} + cpu: [arm64] + os: [darwin] + + '@pnpm/win-arm64@11.10.0': + resolution: {integrity: sha512-g2Ymnq+LgVyZaWsGBQSlpIcBCOOxyLky2UX+kTwGiIXnj6k/xXqZR0ZJLuxeiGNh/CVmhftOqokpyJNzyj8kng==} + cpu: [arm64] + os: [win32] + + '@pnpm/win-x64@11.10.0': + resolution: {integrity: sha512-kCHYZudUEBjrEchgnJUnNHCdvLXpUZun2z0rMAeP5DymsaXwEVvVzGj220iR/NyhgDocJUmgtTKtwL/ejL785Q==} + cpu: [x64] + os: [win32] + + '@reflink/reflink-darwin-arm64@0.1.19': + resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@reflink/reflink-darwin-x64@0.1.19': + resolution: {integrity: sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@reflink/reflink-linux-arm64-gnu@0.1.19': + resolution: {integrity: sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@reflink/reflink-linux-arm64-musl@0.1.19': + resolution: {integrity: sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@reflink/reflink-linux-x64-gnu@0.1.19': + resolution: {integrity: sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@reflink/reflink-linux-x64-musl@0.1.19': + resolution: {integrity: sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@reflink/reflink-win32-arm64-msvc@0.1.19': + resolution: {integrity: sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@reflink/reflink-win32-x64-msvc@0.1.19': + resolution: {integrity: sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@reflink/reflink@0.1.19': + resolution: {integrity: sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA==} + engines: {node: '>= 10'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + pnpm@11.10.0: + resolution: {integrity: sha512-C3+LmAYAMZBMAX46QesYehbUDuuCm5XE+MsDaBdh/Eq1PdIZEVubRH9NzhoFohR2RGHn03AzkqnzL5URzoyGyA==} + engines: {node: '>=22.13'} + hasBin: true + +snapshots: + + '@pnpm/exe@11.10.0': + dependencies: + '@reflink/reflink': 0.1.19 + detect-libc: 2.1.2 + optionalDependencies: + '@pnpm/linux-arm64': 11.10.0 + '@pnpm/linux-x64': 11.10.0 + '@pnpm/linuxstatic-arm64': 11.10.0 + '@pnpm/linuxstatic-x64': 11.10.0 + '@pnpm/macos-arm64': 11.10.0 + '@pnpm/win-arm64': 11.10.0 + '@pnpm/win-x64': 11.10.0 + + '@pnpm/linux-arm64@11.10.0': + optional: true + + '@pnpm/linux-x64@11.10.0': + optional: true + + '@pnpm/linuxstatic-arm64@11.10.0': + optional: true + + '@pnpm/linuxstatic-x64@11.10.0': + optional: true + + '@pnpm/macos-arm64@11.10.0': + optional: true + + '@pnpm/win-arm64@11.10.0': + optional: true + + '@pnpm/win-x64@11.10.0': + optional: true + + '@reflink/reflink-darwin-arm64@0.1.19': + optional: true + + '@reflink/reflink-darwin-x64@0.1.19': + optional: true + + '@reflink/reflink-linux-arm64-gnu@0.1.19': + optional: true + + '@reflink/reflink-linux-arm64-musl@0.1.19': + optional: true + + '@reflink/reflink-linux-x64-gnu@0.1.19': + optional: true + + '@reflink/reflink-linux-x64-musl@0.1.19': + optional: true + + '@reflink/reflink-win32-arm64-msvc@0.1.19': + optional: true + + '@reflink/reflink-win32-x64-msvc@0.1.19': + optional: true + + '@reflink/reflink@0.1.19': + optionalDependencies: + '@reflink/reflink-darwin-arm64': 0.1.19 + '@reflink/reflink-darwin-x64': 0.1.19 + '@reflink/reflink-linux-arm64-gnu': 0.1.19 + '@reflink/reflink-linux-arm64-musl': 0.1.19 + '@reflink/reflink-linux-x64-gnu': 0.1.19 + '@reflink/reflink-linux-x64-musl': 0.1.19 + '@reflink/reflink-win32-arm64-msvc': 0.1.19 + '@reflink/reflink-win32-x64-msvc': 0.1.19 + + detect-libc@2.1.2: {} + + pnpm@11.10.0: {} + +--- lockfileVersion: '9.0' settings: @@ -244,7 +443,7 @@ importers: version: 3.1.1 mint: specifier: 'catalog:' - version: 4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) + version: 4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) packages/evals: dependencies: @@ -269,18 +468,9 @@ importers: '@browserbasehq/stagehand-integrations-codex-sdk': specifier: workspace:* version: link:../integrations/codex-sdk - '@browserbasehq/stagehand-integrations-deepagents-sdk': - specifier: workspace:* - version: link:../integrations/deepagents-sdk - '@browserbasehq/stagehand-integrations-eve-sdk': - specifier: workspace:* - version: link:../integrations/eve-sdk - '@browserbasehq/stagehand-integrations-mastra-sdk': + '@browserbasehq/stagehand-integrations-fx-sdk': specifier: workspace:* - version: link:../integrations/mastra-sdk - '@browserbasehq/stagehand-integrations-pi-sdk': - specifier: workspace:* - version: link:../integrations/pi-sdk + version: link:../integrations/fx-sdk ai: specifier: ^5.0.133 version: 5.0.220(zod@4.4.3) @@ -317,7 +507,7 @@ importers: version: 24.13.2 braintrust: specifier: ^0.4.10 - version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3) + version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(supports-color@8.1.1)(zod@4.4.3) chalk: specifier: ^5.4.1 version: 5.6.2 @@ -517,25 +707,6 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/deepagents-sdk: - dependencies: - '@browserbasehq/stagehand-integrations': - specifier: workspace:* - version: link:../core - devDependencies: - '@types/node': - specifier: 'catalog:' - version: 24.13.2 - tsdown: - specifier: 'catalog:' - version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) - typescript: - specifier: 'catalog:' - version: 5.9.3 - vitest: - specifier: 'catalog:' - version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/eve: dependencies: '@ai-sdk/openai': @@ -552,7 +723,7 @@ importers: version: 17.4.2 eve: specifier: 'catalog:' - version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) devDependencies: '@types/node': specifier: 'catalog:' @@ -564,32 +735,11 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/eve-sdk: + packages/integrations/fx-sdk: dependencies: - '@ai-sdk/anthropic': - specifier: 'catalog:' - version: 4.0.8(zod@4.4.3) - '@ai-sdk/google': - specifier: 'catalog:' - version: 4.0.8(zod@4.4.3) - '@ai-sdk/openai': - specifier: 'catalog:' - version: 4.0.8(zod@4.4.3) '@browserbasehq/stagehand-integrations': specifier: workspace:* version: link:../core - '@modelcontextprotocol/sdk': - specifier: 'catalog:' - version: 1.29.0(zod@4.4.3) - ai: - specifier: ^7.0.38 - version: 7.0.77(zod@4.4.3) - eve: - specifier: 'catalog:' - version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) - zod: - specifier: 'catalog:' - version: 4.4.3 devDependencies: '@types/node': specifier: 'catalog:' @@ -614,42 +764,14 @@ importers: version: link:../core '@mastra/core': specifier: 'catalog:' - version: 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) - '@mastra/mcp': - specifier: 'catalog:' - version: 1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) - devDependencies: - '@types/node': - specifier: 'catalog:' - version: 24.13.2 - typescript: - specifier: 'catalog:' - version: 5.9.3 - vitest: - specifier: 'catalog:' - version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - - packages/integrations/mastra-sdk: - dependencies: - '@browserbasehq/stagehand-integrations': - specifier: workspace:* - version: link:../core - '@mastra/core': - specifier: 'catalog:' - version: 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + version: 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) '@mastra/mcp': specifier: 'catalog:' - version: 1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) - zod: - specifier: 'catalog:' - version: 4.4.3 + version: 1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) devDependencies: '@types/node': specifier: 'catalog:' version: 24.13.2 - tsdown: - specifier: 'catalog:' - version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) typescript: specifier: 'catalog:' version: 5.9.3 @@ -682,34 +804,6 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/pi-sdk: - dependencies: - '@browserbasehq/stagehand-integrations': - specifier: workspace:* - version: link:../core - '@earendil-works/pi-coding-agent': - specifier: 'catalog:' - version: 0.84.2(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) - '@modelcontextprotocol/sdk': - specifier: 'catalog:' - version: 1.29.0(zod@4.4.3) - typebox: - specifier: 'catalog:' - version: 1.3.7 - devDependencies: - '@types/node': - specifier: 'catalog:' - version: 24.13.2 - tsdown: - specifier: 'catalog:' - version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) - typescript: - specifier: 'catalog:' - version: 5.9.3 - vitest: - specifier: 'catalog:' - version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/vercel-ai: dependencies: '@ai-sdk/mcp': @@ -872,12 +966,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/gateway@4.0.62': - resolution: {integrity: sha512-zR3pustGWhw5eUZHG+fJZx/V/PBe+LxdDpc5hDFWxozG/3MB/+eY62jn+YiR+9uOH+Hx63e5zJoeKLfZfPktWQ==} - engines: {node: '>=22'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/google-vertex@3.0.158': resolution: {integrity: sha512-Z9sY69vlrOR574Bb+3Tjp1P2W1QK4ut7fTUgA3F3lcvNxAvlAyxp24T4bodaBTWP6Q74pCB1IQGL5NaPDN1r2A==} engines: {node: '>=18'} @@ -974,12 +1062,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.29': - resolution: {integrity: sha512-7EIbwXiXKGa7EFk6tDZpuZBs6lxhEJpOuHeqrDb3Vd85uYdjwkdRuHnZDVDIIb2+QTSRmyph2NrXcbvuO/KAjQ==} - engines: {node: '>=22'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.5': resolution: {integrity: sha512-oI0t3dvCoqWNV1I8o1Rybi2DXDvHES5r/TrwtJW90tuFLVepgJlftPxrcjh8vaSvjqC2diTuA2vXyjKAyHJm4A==} engines: {node: '>=22'} @@ -1006,10 +1088,6 @@ packages: resolution: {integrity: sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==} engines: {node: '>=22'} - '@ai-sdk/provider@4.0.7': - resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} - engines: {node: '>=22'} - '@ai-sdk/togetherai@1.0.49': resolution: {integrity: sha512-g4BpEatN7flh3GZ0CN9KvAUX6uLPmIqGSrKKFvAmC3HZdnF940zl+ChXs3atdbtpr6+cwirxM5RACbUzr0uYhA==} engines: {node: '>=18'} @@ -1213,26 +1291,14 @@ packages: resolution: {integrity: sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.976.0': - resolution: {integrity: sha512-0cjRaEdlVoOrsNb9pP5q1Syyc8pXw5xSj2Np2ryReRTr9FppIIRVSdZK4lbnfmc2Hvgux/xBOUU6baB7z8//uA==} - engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.977.7': resolution: {integrity: sha512-I88Iov89NVmjSmJLKSv7Cn9M2J+a2942OkA8nZCbz+sl4ZeY4zEOcoLOrbt1GRfQ8zEQKnjAJdXixA3J/p1fDQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-env@3.972.60': - resolution: {integrity: sha512-BAkxdoe7tpDDqCghGpuOeHQRbm/2znVvOQm0AvpQbA2tbfMN46doN4zx65fv85ImP3KADwc2zQPmbrlI9MPfMg==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-env@3.972.68': resolution: {integrity: sha512-2a20A/IdNOwUvaDq91iqqS7BA0XlNMfW3iLGZGZLJv0EbUqhSxB0PIx4rQQqssvWj1uXImb3/UCCdHz/+1dOiA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-http@3.972.62': - resolution: {integrity: sha512-g/0fGqKTb9xpKdd9AtpmV5Eo3DFKbnkpA2+w0peISSlu7NfAoWOuYBFxsu+yWBtxU89ka55ezoZBCbFaS8pjYQ==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-http@3.972.70': resolution: {integrity: sha512-0yRem2Fs52r/Nn6UAqIlpjexfaYj8ziEozOe9tamtAVT/5bzFLKx8O2r7MaRqgS3hGKHIa1Jij9nKHSsNnb04A==} engines: {node: '>=20.0.0'} @@ -1241,30 +1307,14 @@ packages: resolution: {integrity: sha512-2M39DE02XpYYaSWYk/4AsImXYUU/1L2xmTMLUpMMWq7DfLv191/vCRy3baKtdr45AkJQyVgSjmuVOLm15SwrRQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-ini@3.973.5': - resolution: {integrity: sha512-ylubazcRfq2TVus/qXucSXeC42Qdjp5HQxTu68K/BsdMiZlcSLD1zkpoCgApXZX1Y6YJhtGGs7ZHhO/GuIgBlw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-login@3.972.67': - resolution: {integrity: sha512-CCygIKJ9YbI3n84OClSaSppkgKKHVj2TGT33c6FRORZrYNZQ1POmD+ip0FLYokiJAK7sSdc3YVkOsBm90oxWMQ==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-login@3.972.75': resolution: {integrity: sha512-jaTESuJlQsoUZ44f/i2puyPt8VlF/dMMJ9HM3cStYtk7eKX4N9UWi83OLixUkoOJH3BwWlPLCq9YIK9nfWhVBg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-node@3.972.71': - resolution: {integrity: sha512-HIg7Q2osBzajQwL+1Vkyh2E7Gim3eTNb9RHIsOxDGjW0eZg4oEKtRs5sioCnc73ilhaOm4gX2lHVF8J7+nt2rg==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-node@3.972.79': resolution: {integrity: sha512-RIw5dof1EHkWubrZzPC941CDtnFG1iAXsxbFgLkhdYZXHc4icU13c/uxSMI0J5eUx9bxa7LjfpdjfClBB1QsDA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-process@3.972.60': - resolution: {integrity: sha512-YIo3f99hM43QdYG8hDzwGemnR/pU95b0kramqSJUTleCqaB7+HwKf7YZFHqvOgTqZTPx/mRmNIqoDRr3U0Z3Tw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-process@3.972.68': resolution: {integrity: sha512-nLP3Pda2MQTFJ25hKBMmUuB9Uv+bTZQNlufbeCwklP549Vwnkd8bRLJoCKp5k6xjmdyptrPrOfGOhN0mKuca8A==} engines: {node: '>=20.0.0'} @@ -1273,14 +1323,6 @@ packages: resolution: {integrity: sha512-EmgyyHn+f9WCcelp3L/vci+LGbX8GigWaVphRArjVo5Pktkr9YnLy/mQ6VDkDyBD72dtfRNTgHmD2ts4rTDXKQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-sso@3.973.4': - resolution: {integrity: sha512-BPdmL8sSBOCv4ngZ+3LHxyc3CNqDCEK37CHioCk7zGrTMY5sUtkH8q+o6qA80nn6w3/fyBPGNE7OIRlmoOxRQA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-web-identity@3.972.66': - resolution: {integrity: sha512-kSAziJboOmZmsR9/MTbiNjowl2BPes1bQuJpne4qAZ62ubi8fjfr/aupJSQje6udBoYxXTQbsL0e0kby2la3ng==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-web-identity@3.972.74': resolution: {integrity: sha512-0YfczxGXF3RjGj8z7QG/Ho2HnLGKDHfPSHiTs47UU1U/+mmwISDN+rvGKt2zh+3FX8NdT4xd95LGBGyhQw2dgQ==} engines: {node: '>=20.0.0'} @@ -1297,18 +1339,10 @@ packages: resolution: {integrity: sha512-gdcWRbmIf1dWA/prf44Bnnzgqj+AbsXX2yfhZhOQLwSm7NfKIYPmkRlPqP0CTepHzjxMIBdWBDdtQB+Y/dFUeg==} engines: {node: '>= 14.0.0'} - '@aws-sdk/nested-clients@3.997.34': - resolution: {integrity: sha512-Y9REVrSwmLM+Qy6sZJ7ofMC2S3Hr3tPP/4CzL5U1olPP7OGoF+6+Px0E49cVQBtSxJtyeLJMf0UaBErfeSahAA==} - engines: {node: '>=20.0.0'} - '@aws-sdk/nested-clients@3.997.42': resolution: {integrity: sha512-XWRyon2MTHXD/zMoo0Mbge6Vwf+iE0qQaM/RyGO6NfZ9WukCFiQL27nQVZjYy2JwSIg+iXZxKOX95OBXqlSM4w==} engines: {node: '>=20.0.0'} - '@aws-sdk/signature-v4-multi-region@3.996.41': - resolution: {integrity: sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==} - engines: {node: '>=20.0.0'} - '@aws-sdk/signature-v4-multi-region@3.996.44': resolution: {integrity: sha512-ZSfQ35Qn4MhSY+A0Whyr+KBx+wJKZUyBsOrjB2pSHOafRzbFe47T8XcXM8hZqUAC69qnqIy0C9ArxTuud0CC2w==} engines: {node: '>=20.0.0'} @@ -1317,28 +1351,16 @@ packages: resolution: {integrity: sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==} engines: {node: '>=20.0.0'} - '@aws-sdk/token-providers@3.1092.0': - resolution: {integrity: sha512-hBYUAr6iBLNFcsiWTgtBb0stdSw39VOUq4Sp4A5caCNf66BAZplWN4FleKrVpJx5li2YgdnK2DqoFSMWC642FQ==} - engines: {node: '>=20.0.0'} - '@aws-sdk/token-providers@3.1108.0': resolution: {integrity: sha512-rI80zxDxGJ6904eC/YbjkdjY6JdaZvQ01kOmrMvw7cFQGIHo27fhnIVbMSVDS4T6foQImjxYSRoOu/uSJscXDw==} engines: {node: '>=20.0.0'} - '@aws-sdk/types@3.974.2': - resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} - engines: {node: '>=20.0.0'} - '@aws-sdk/types@3.974.3': resolution: {integrity: sha512-ECAqfpNsef+7MO8qtR0h9KcFIBAygaE7Cm6UOiQl+ft+uVap+1G7bNEjs4mdJE2OnA4m6k7i8peH8uGIAsOMGw==} engines: {node: '>=20.0.0'} - '@aws-sdk/util-locate-window@3.965.8': - resolution: {integrity: sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/xml-builder@3.972.36': - resolution: {integrity: sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==} + '@aws-sdk/util-locate-window@3.965.9': + resolution: {integrity: sha512-wB/ho7pTJKqWz3WYDt2ZWDWI8bxQpN/xwf+5ZQ1zWaj+HDY9B8Fn434i6qZ6j6ZG3aCiIJtZaQVqwajx5xYsQA==} engines: {node: '>=20.0.0'} '@aws-sdk/xml-builder@3.972.38': @@ -1883,8 +1905,8 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@hono/node-server@1.19.14': - resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} + '@hono/node-server@1.19.15': + resolution: {integrity: sha512-Za2ai6TLdKjUvnur+eenO6nuYYipVAEhyCAdaV8IRvmU9kK8crOZUSYvIXn72E4f8fJqyAbpcJuTsYYmZp9Deg==} engines: {node: '>=18.14.1'} peerDependencies: hono: ^4 @@ -3170,10 +3192,6 @@ packages: resolution: {integrity: sha512-doH1gimEu3A46VX6aVxpHTeHrytJAG6HgdxntYnCFiIFHEM/ZGpG8KiZGBChchjQmG0XFIBL552kBTjVcMZXwQ==} engines: {node: '>=12'} - '@smithy/core@3.29.8': - resolution: {integrity: sha512-rpCbCV+TimOBi3VLNBMmtTvgfOWcFIEAru3+TFlG87SL2F+te4jOnnNR+cf3uR4eJ5Qf4LnT80fqnBKgPRS6zA==} - engines: {node: '>=18.0.0'} - '@smithy/core@3.30.0': resolution: {integrity: sha512-dl2yRglDxfzH9uJ4fSo4zTaAHa0zH7+V7BZMRWy8hEYIKT1BiqMUK/CN6T3ADQ3kbA5N1tmUulroJ2UtONS7Kw==} engines: {node: '>=18.0.0'} @@ -3182,10 +3200,6 @@ packages: resolution: {integrity: sha512-NAiCSC78fzbNIEWoheoF74Ob5ZorLijCHpMY26Fqvqg/+9LuyIqMfHDg2p8Yk1rqOyowtiL3y7WX0AW+teL6zw==} engines: {node: '>=18.0.0'} - '@smithy/credential-provider-imds@4.4.13': - resolution: {integrity: sha512-X+2HNZhWi5i3rJsCas0LPf6fTQUaKyJ40zd8aTO/bwpRfpU3biYaqLr7C1WMibL7PVKJalpi1PyybjGPNoHC8Q==} - engines: {node: '>=18.0.0'} - '@smithy/credential-provider-imds@4.5.0': resolution: {integrity: sha512-2jsPi+7Zv2hSzD9IXR9D7DTqSn7mv4XalzRm+bESh53jiaUS3NKEUbpQFTJP0HhQy9qzZvluxQ3yS24zdRrqsA==} engines: {node: '>=18.0.0'} @@ -3194,10 +3208,6 @@ packages: resolution: {integrity: sha512-dk/8H8vjgsghRKq0Euout/Q5iQMGVw8pUVSs2gY7DFjZtg1+RB12Q/TNh4Uph6E9D46mKr161WobWFyAQWJdjQ==} engines: {node: '>=18.0.0'} - '@smithy/fetch-http-handler@5.6.10': - resolution: {integrity: sha512-5/Yj9mS2JjTsB3B8ZX7euh77mrY9aXW23ag1yAmFykSRmA6vldqBrgqmSeQ50EjY+5SB8+aE4w14B6LKbBVEhQ==} - engines: {node: '>=18.0.0'} - '@smithy/fetch-http-handler@5.7.0': resolution: {integrity: sha512-W/exA8T0LEzCQtJ02w4IzaEQPIspgarqZprb7W8FwnYiDowgCrjl2fTQ6FvuSSUnJORuepBF81abmBJwqh+0XQ==} engines: {node: '>=18.0.0'} @@ -3214,14 +3224,6 @@ packages: resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==} engines: {node: '>=18.0.0'} - '@smithy/node-http-handler@4.9.10': - resolution: {integrity: sha512-ETQz9v/Z+nTQc6fRWTXxUpxJqwpmzB3Tn3WKAdHwWkeT+m+HE5czs6GNG8vW+4vyxXSls65RVcvOZwk7Q/PS/Q==} - engines: {node: '>=18.0.0'} - - '@smithy/signature-v4@5.6.9': - resolution: {integrity: sha512-g5rnEii/mkT0mjVJmlsaOfyNBtHNTecD9Lo4NP8D5HzMUEnZNpz7/FbvBCjNcV4vteHFAxOGiLUYNxPkDZZAPw==} - engines: {node: '>=18.0.0'} - '@smithy/signature-v4@5.7.0': resolution: {integrity: sha512-hCynhm22wMJ8wTF9crcwu8mxggtUrSLLJgDcGUvYFBqpofxycYJCGKOMYg4xtPPFtgNiDJSYmhsWLTrcU/g59Q==} engines: {node: '>=18.0.0'} @@ -3582,12 +3584,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - ai@7.0.77: - resolution: {integrity: sha512-muLtBSTAUCreR77L16w4AFBiX2gK/RNt84EKp8m03SN9+MfNlC5EGqYYttRjYKV3xe0a33yj1Zawj1EnjejIWw==} - engines: {node: '>=22'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - ajv-draft-04@1.0.0: resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} peerDependencies: @@ -5153,8 +5149,8 @@ packages: highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} - hono@4.12.31: - resolution: {integrity: sha512-zJIHFrl6bq3RDd2YusFNCDlM8qUprxKswyi/OPzPyzKDdyBXDqWx8bZlZ7R+saTdSTatUmb3O7K4SspGPaEOQg==} + hono@4.12.32: + resolution: {integrity: sha512-XcuyW9qE2kJn07PkecMOBd5Vq/hMy7mmGw+idz1yblbg9N17ijJODrvPkn7/dwL3Kulj8LcRJ69DLOWf91dRUg==} engines: {node: '>=16.9.0'} hookable@6.1.1: @@ -5858,8 +5854,8 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} - media-typer@1.1.0: - resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + media-typer@1.1.1: + resolution: {integrity: sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==} engines: {node: '>= 0.8'} merge-descriptors@1.0.3: @@ -8279,13 +8275,6 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.4.3 - '@ai-sdk/gateway@4.0.62(zod@4.4.3)': - dependencies: - '@ai-sdk/provider': 4.0.7 - '@ai-sdk/provider-utils': 5.0.29(zod@4.4.3) - '@vercel/oidc': 3.2.0 - zod: 4.4.3 - '@ai-sdk/google-vertex@3.0.158(zod@4.4.3)': dependencies: '@ai-sdk/anthropic': 2.0.91(zod@4.4.3) @@ -8403,15 +8392,6 @@ snapshots: undici: 7.29.0 zod: 4.4.3 - '@ai-sdk/provider-utils@5.0.29(zod@4.4.3)': - dependencies: - '@ai-sdk/provider': 4.0.7 - '@standard-schema/spec': 1.1.0 - '@workflow/serde': 4.1.0 - eventsource-parser: 3.1.0 - undici: 7.29.0 - zod: 4.4.3 - '@ai-sdk/provider-utils@5.0.5(zod@4.4.3)': dependencies: '@ai-sdk/provider': 4.0.2 @@ -8440,10 +8420,6 @@ snapshots: dependencies: json-schema: 0.4.0 - '@ai-sdk/provider@4.0.7': - dependencies: - json-schema: 0.4.0 - '@ai-sdk/togetherai@1.0.49(zod@4.4.3)': dependencies: '@ai-sdk/openai-compatible': 1.0.46(zod@4.4.3) @@ -8618,15 +8594,15 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-crypto/supports-web-crypto': 5.2.0 '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.974.2 - '@aws-sdk/util-locate-window': 3.965.8 + '@aws-sdk/types': 3.974.3 + '@aws-sdk/util-locate-window': 3.965.9 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 '@aws-crypto/sha256-js@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.974.2 + '@aws-sdk/types': 3.974.3 tslib: 2.8.1 '@aws-crypto/supports-web-crypto@5.2.0': @@ -8635,7 +8611,7 @@ snapshots: '@aws-crypto/util@5.2.0': dependencies: - '@aws-sdk/types': 3.974.2 + '@aws-sdk/types': 3.974.3 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -8643,30 +8619,19 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.976.0 - '@aws-sdk/credential-provider-node': 3.972.71 + '@aws-sdk/core': 3.977.7 + '@aws-sdk/credential-provider-node': 3.972.79 '@aws-sdk/eventstream-handler-node': 3.972.32 '@aws-sdk/middleware-eventstream': 3.972.27 '@aws-sdk/middleware-websocket': 3.972.50 '@aws-sdk/token-providers': 3.1048.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/fetch-http-handler': 5.6.10 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.30.0 + '@smithy/fetch-http-handler': 5.7.0 '@smithy/node-http-handler': 4.7.3 '@smithy/types': 4.16.1 tslib: 2.8.1 - '@aws-sdk/core@3.976.0': - dependencies: - '@aws-sdk/types': 3.974.2 - '@aws-sdk/xml-builder': 3.972.36 - '@aws/lambda-invoke-store': 0.3.0 - '@smithy/core': 3.29.8 - '@smithy/signature-v4': 5.6.9 - '@smithy/types': 4.16.1 - bowser: 2.14.1 - tslib: 2.8.1 - '@aws-sdk/core@3.977.7': dependencies: '@aws-sdk/types': 3.974.3 @@ -8674,18 +8639,10 @@ snapshots: '@aws/lambda-invoke-store': 0.3.0 '@smithy/core': 3.32.0 '@smithy/signature-v4': 5.7.0 - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.0 bowser: 2.14.1 tslib: 2.8.1 - '@aws-sdk/credential-provider-env@3.972.60': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - '@aws-sdk/credential-provider-env@3.972.68': dependencies: '@aws-sdk/core': 3.977.7 @@ -8693,17 +8650,6 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/credential-provider-http@3.972.62': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/fetch-http-handler': 5.6.10 - '@smithy/node-http-handler': 4.9.10 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/credential-provider-http@3.972.70': dependencies: @@ -8714,7 +8660,6 @@ snapshots: '@smithy/node-http-handler': 4.10.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@aws-sdk/credential-provider-ini@3.973.13': dependencies: @@ -8731,32 +8676,6 @@ snapshots: '@smithy/credential-provider-imds': 4.5.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/credential-provider-ini@3.973.5': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/credential-provider-env': 3.972.60 - '@aws-sdk/credential-provider-http': 3.972.62 - '@aws-sdk/credential-provider-login': 3.972.67 - '@aws-sdk/credential-provider-process': 3.972.60 - '@aws-sdk/credential-provider-sso': 3.973.4 - '@aws-sdk/credential-provider-web-identity': 3.972.66 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/credential-provider-imds': 4.4.13 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-login@3.972.67': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/credential-provider-login@3.972.75': dependencies: @@ -8766,21 +8685,6 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/credential-provider-node@3.972.71': - dependencies: - '@aws-sdk/credential-provider-env': 3.972.60 - '@aws-sdk/credential-provider-http': 3.972.62 - '@aws-sdk/credential-provider-ini': 3.973.5 - '@aws-sdk/credential-provider-process': 3.972.60 - '@aws-sdk/credential-provider-sso': 3.973.4 - '@aws-sdk/credential-provider-web-identity': 3.972.66 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/credential-provider-imds': 4.4.13 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/credential-provider-node@3.972.79': dependencies: @@ -8795,15 +8699,6 @@ snapshots: '@smithy/credential-provider-imds': 4.5.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/credential-provider-process@3.972.60': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/credential-provider-process@3.972.68': dependencies: @@ -8812,7 +8707,6 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@aws-sdk/credential-provider-sso@3.973.12': dependencies: @@ -8823,26 +8717,6 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/credential-provider-sso@3.973.4': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/token-providers': 3.1092.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-web-identity@3.972.66': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/credential-provider-web-identity@3.972.74': dependencies: @@ -8852,7 +8726,6 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@aws-sdk/eventstream-handler-node@3.972.32': dependencies: @@ -8878,17 +8751,6 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 - '@aws-sdk/nested-clients@3.997.34': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/signature-v4-multi-region': 3.996.41 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/fetch-http-handler': 5.6.10 - '@smithy/node-http-handler': 4.9.10 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - '@aws-sdk/nested-clients@3.997.42': dependencies: '@aws-sdk/core': 3.977.7 @@ -8899,14 +8761,6 @@ snapshots: '@smithy/node-http-handler': 4.10.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/signature-v4-multi-region@3.996.41': - dependencies: - '@aws-sdk/types': 3.974.2 - '@smithy/signature-v4': 5.6.9 - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/signature-v4-multi-region@3.996.44': dependencies: @@ -8914,23 +8768,13 @@ snapshots: '@smithy/signature-v4': 5.7.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@aws-sdk/token-providers@3.1048.0': dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/token-providers@3.1092.0': - dependencies: - '@aws-sdk/core': 3.976.0 - '@aws-sdk/nested-clients': 3.997.34 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.8 + '@aws-sdk/core': 3.977.7 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.30.0 '@smithy/types': 4.16.1 tslib: 2.8.1 @@ -8942,25 +8786,14 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true - - '@aws-sdk/types@3.974.2': - dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 '@aws-sdk/types@3.974.3': dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/util-locate-window@3.965.8': - dependencies: + '@smithy/types': 4.17.0 tslib: 2.8.1 - '@aws-sdk/xml-builder@3.972.36': + '@aws-sdk/util-locate-window@3.965.9': dependencies: - '@smithy/types': 4.16.1 tslib: 2.8.1 '@aws-sdk/xml-builder@3.972.38': @@ -9566,9 +9399,9 @@ snapshots: - supports-color - utf-8-validate - '@hono/node-server@1.19.14(hono@4.12.31)': + '@hono/node-server@1.19.15(hono@4.12.32)': dependencies: - hono: 4.12.31 + hono: 4.12.32 '@img/colour@1.1.0': {} @@ -9914,7 +9747,7 @@ snapshots: dependencies: jsep: 1.4.0 - '@kwsites/file-exists@1.1.1': + '@kwsites/file-exists@1.1.1(supports-color@8.1.1)': dependencies: debug: 4.4.3(supports-color@8.1.1) transitivePeerDependencies: @@ -9990,7 +9823,7 @@ snapshots: '@mariozechner/clipboard-win32-x64-msvc': 0.3.9 optional: true - '@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3)': + '@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3)': dependencies: '@a2a-js/sdk': 0.3.14(express@5.2.1) '@ai-sdk/provider-utils-v5': '@ai-sdk/provider-utils@3.0.30(zod@4.4.3)' @@ -10006,7 +9839,7 @@ snapshots: '@sindresorhus/slugify': 2.2.1 '@standard-schema/spec': 1.1.0 ajv: 8.20.0 - chat: 4.37.0(ai@7.0.77(zod@4.4.3))(zod@4.4.3) + chat: 4.37.0(ai@7.0.16(zod@4.4.3))(zod@4.4.3) croner: 10.0.1 dotenv: 17.4.2 execa: 9.6.1 @@ -10036,9 +9869,9 @@ snapshots: - utf-8-validate - workflow - '@mastra/mcp@1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3)': + '@mastra/mcp@1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3)': dependencies: - '@mastra/core': 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + '@mastra/core': 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) '@modelcontextprotocol/ext-apps': 1.7.5(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) exit-hook: 5.1.0 @@ -10094,7 +9927,7 @@ snapshots: '@types/react': 19.2.17 react: 19.2.3 - '@mintlify/cli@4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3)': + '@mintlify/cli@4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@inquirer/prompts': 7.9.0(@types/node@25.9.4) '@mintlify/common': 1.0.1080(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -10106,7 +9939,7 @@ snapshots: adm-zip: 0.6.0 chalk: 5.2.0 color: 4.2.3 - detect-port: 1.5.1 + detect-port: 1.5.1(supports-color@8.1.1) fs-extra: 11.2.0 ink: 6.3.0(@types/react@19.2.17)(bufferutil@4.1.0)(react@19.2.3) inquirer: 12.3.0(@types/node@25.9.4) @@ -10413,7 +10246,7 @@ snapshots: '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: - '@hono/node-server': 1.19.14(hono@4.12.31) + '@hono/node-server': 1.19.15(hono@4.12.32) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -10423,7 +10256,7 @@ snapshots: eventsource-parser: 3.1.0 express: 5.2.1 express-rate-limit: 8.6.0(express@5.2.1) - hono: 4.12.31 + hono: 4.12.32 jose: 6.2.4 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -10915,34 +10748,21 @@ snapshots: dependencies: escape-string-regexp: 5.0.0 - '@smithy/core@3.29.8': - dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 - '@smithy/core@3.30.0': dependencies: '@smithy/types': 4.16.1 tslib: 2.8.1 - optional: true '@smithy/core@3.32.0': dependencies: '@smithy/types': 4.17.0 tslib: 2.8.1 - '@smithy/credential-provider-imds@4.4.13': - dependencies: - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - '@smithy/credential-provider-imds@4.5.0': dependencies: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@smithy/eventstream-codec@4.4.14': dependencies: @@ -10950,12 +10770,6 @@ snapshots: tslib: 2.8.1 optional: true - '@smithy/fetch-http-handler@5.6.10': - dependencies: - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - '@smithy/fetch-http-handler@5.7.0': dependencies: '@smithy/core': 3.32.0 @@ -10971,23 +10785,10 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/types': 4.17.0 tslib: 2.8.1 - optional: true '@smithy/node-http-handler@4.7.3': dependencies: - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@smithy/node-http-handler@4.9.10': - dependencies: - '@smithy/core': 3.29.8 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@smithy/signature-v4@5.6.9': - dependencies: - '@smithy/core': 3.29.8 + '@smithy/core': 3.30.0 '@smithy/types': 4.16.1 tslib: 2.8.1 @@ -11433,13 +11234,6 @@ snapshots: '@ai-sdk/provider-utils': 5.0.5(zod@4.4.3) zod: 4.4.3 - ai@7.0.77(zod@4.4.3): - dependencies: - '@ai-sdk/gateway': 4.0.62(zod@4.4.3) - '@ai-sdk/provider': 4.0.7 - '@ai-sdk/provider-utils': 5.0.29(zod@4.4.3) - zod: 4.4.3 - ajv-draft-04@1.0.0(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -11718,7 +11512,7 @@ snapshots: dependencies: fill-range: 7.1.1 - braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3): + braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(supports-color@8.1.1)(zod@4.4.3): dependencies: '@ai-sdk/provider': 1.1.3 '@next/env': 14.2.35 @@ -11736,7 +11530,7 @@ snapshots: minimatch: 9.0.9 mustache: 4.2.0 pluralize: 8.0.0 - simple-git: 3.36.0 + simple-git: 3.36.0(supports-color@8.1.1) slugify: 1.6.9 source-map: 0.7.6 uuid: 9.0.1 @@ -11871,7 +11665,7 @@ snapshots: chardet@2.2.0: {} - chat@4.37.0(ai@7.0.77(zod@4.4.3))(zod@4.4.3): + chat@4.37.0(ai@7.0.16(zod@4.4.3))(zod@4.4.3): dependencies: '@workflow/serde': 4.1.0-beta.2 mdast-util-to-string: 4.0.0 @@ -11881,7 +11675,7 @@ snapshots: remend: 1.3.0 unified: 11.0.5 optionalDependencies: - ai: 7.0.77(zod@4.4.3) + ai: 7.0.16(zod@4.4.3) zod: 4.4.3 transitivePeerDependencies: - supports-color @@ -12186,7 +11980,7 @@ snapshots: detect-libc@2.1.2: {} - detect-port@1.5.1: + detect-port@1.5.1(supports-color@8.1.1): dependencies: address: 1.2.2 debug: 4.4.3(supports-color@8.1.1) @@ -12556,9 +12350,9 @@ snapshots: etag@1.8.1: {} - eve@0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): + eve@0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): dependencies: - ai: 7.0.77(zod@4.4.3) + ai: 7.0.16(zod@4.4.3) nitro: 3.0.260610-beta(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) undici: 8.9.0 optionalDependencies: @@ -13345,7 +13139,7 @@ snapshots: highlight.js@10.7.3: {} - hono@4.12.31: {} + hono@4.12.32: {} hookable@6.1.1: {} @@ -14150,7 +13944,7 @@ snapshots: media-typer@0.3.0: {} - media-typer@1.1.0: {} + media-typer@1.1.1: {} merge-descriptors@1.0.3: {} @@ -14491,9 +14285,9 @@ snapshots: dependencies: minipass: 7.1.3 - mint@4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3): + mint@4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3): dependencies: - '@mintlify/cli': 4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) + '@mintlify/cli': 4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) transitivePeerDependencies: - '@base-ui/react' - '@types/node' @@ -15929,9 +15723,9 @@ snapshots: simple-concat: 1.0.1 optional: true - simple-git@3.36.0: + simple-git@3.36.0(supports-color@8.1.1): dependencies: - '@kwsites/file-exists': 1.1.1 + '@kwsites/file-exists': 1.1.1(supports-color@8.1.1) '@kwsites/promise-deferred': 1.1.1 '@simple-git/args-pathspec': 1.0.3 '@simple-git/argv-parser': 1.1.1 @@ -16404,7 +16198,7 @@ snapshots: type-is@2.1.0: dependencies: content-type: 2.0.0 - media-typer: 1.1.0 + media-typer: 1.1.1 mime-types: 3.0.2 typebox@1.3.7: {} diff --git a/turbo.json b/turbo.json index d3a547b21..df8dbd510 100644 --- a/turbo.json +++ b/turbo.json @@ -61,6 +61,11 @@ "inputs": ["$TURBO_DEFAULT$", "!dist/**"], "outputs": ["dist/**"] }, + "@browserbasehq/stagehand-integrations-fx-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, "@browserbasehq/stagehand-evals#build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "!dist/**"], @@ -186,6 +191,16 @@ "$TURBO_ROOT$/vitest.config.ts" ] }, + "@browserbasehq/stagehand-integrations-fx-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-fx-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] + }, "@browserbasehq/stagehand-integrations-example-mastra-facade#typecheck": { "dependsOn": ["^build"] }, @@ -207,6 +222,9 @@ "@browserbasehq/stagehand-integrations-deepagents-sdk#typecheck": { "dependsOn": ["^build"] }, + "@browserbasehq/stagehand-integrations-fx-sdk#typecheck": { + "dependsOn": ["^build"] + }, "@browserbasehq/stagehand-docs#typecheck": {}, "test:unit": { "dependsOn": ["^build"], diff --git a/vitest.config.ts b/vitest.config.ts index 1c98f8a1b..df095d81c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ "packages/integrations/pi-sdk/tests/**/*.test.ts", "packages/integrations/eve-sdk/tests/**/*.test.ts", "packages/integrations/deepagents-sdk/tests/**/*.test.ts", + "packages/integrations/fx-sdk/tests/**/*.test.ts", "packages/extension/tests/**/*.test.ts", "packages/sdk-ts/tests/**/*.test.ts", "packages/extension/understudy/**/*.test.ts", From 6f10941d0a67ad075411122eb17a51427cf3f6c3 Mon Sep 17 00:00:00 2001 From: miguel Date: Sat, 22 Aug 2026 13:27:45 -0700 Subject: [PATCH 2/7] feat(evals): register fx harness on the shared external-harness skeleton Define fxHarness with defineExternalHarness and add it to the bench harness registry so --harness fx plans, dry-runs, and executes like claude_code and codex. fxRunner now runs through runExternalHarnessTask (shared prompt, result parsing, normalized harness_* metrics and harnessStatus), and fxToolAdapter resolves surfaces/startup profiles through the shared registry helpers via FX_TOOL_SURFACES. EVAL_FX_MODELS overrides the default model list. Registry-derived guidance tests now include fx. --- packages/evals/framework/benchHarness.ts | 11 + packages/evals/framework/fxRunner.ts | 243 ++++----- packages/evals/framework/fxToolAdapter.ts | 42 +- .../tests/framework/benchHarness.test.ts | 21 +- .../evals/tests/framework/fxRunner.test.ts | 39 ++ .../tests/framework/fxToolAdapter.test.ts | 41 +- pnpm-lock.yaml | 472 +++++++++--------- 7 files changed, 419 insertions(+), 450 deletions(-) diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 0e522a6fa..ba4ee1dea 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -18,6 +18,8 @@ import { runEveAgent } from "./eveRunner.js"; import { EVE_TOOL_SURFACES, prepareEveToolAdapter } from "./eveToolAdapter.js"; import { runDeepagentsAgent } from "./deepagentsRunner.js"; import { DEEPAGENTS_TOOL_SURFACES, prepareDeepagentsToolAdapter } from "./deepagentsToolAdapter.js"; +import { runFxAgent } from "./fxRunner.js"; +import { FX_TOOL_SURFACES, prepareFxToolAdapter } from "./fxToolAdapter.js"; import { buildExternalHarnessTaskPlan, type ExternalHarnessTaskPlan, @@ -315,6 +317,14 @@ export const deepagentsHarness = defineExternalHarness({ runAgent: runDeepagentsAgent, }); +export const fxHarness = defineExternalHarness({ + harness: "fx", + supportedToolSurfaces: FX_TOOL_SURFACES, + defaultModels: ["openai/gpt-5.4-mini" as AvailableModel], + prepareToolAdapter: prepareFxToolAdapter, + runAgent: runFxAgent, +}); + const harnessRegistry = new Map([ ["stagehand", stagehandHarness], ["claude_code", claudeCodeHarness], @@ -323,6 +333,7 @@ const harnessRegistry = new Map([ ["pi", piHarness], ["eve", eveHarness], ["deepagents", deepagentsHarness], + ["fx", fxHarness], ]); export function registerBenchHarness(harness: BenchHarness): void { diff --git a/packages/evals/framework/fxRunner.ts b/packages/evals/framework/fxRunner.ts index 4c3ce5e7e..637c67f27 100644 --- a/packages/evals/framework/fxRunner.ts +++ b/packages/evals/framework/fxRunner.ts @@ -1,5 +1,3 @@ -// Prompt builder and result parser are intentionally duplicated from codexRunner.ts for Phase 1; -// Phase 2 switches this to the shared externalRunner skeleton. import { buildFxTranscript, normalizeFxModel, @@ -11,14 +9,21 @@ import { type FxTokenUsage, } from "@browserbasehq/stagehand-integrations-fx-sdk"; import type { AvailableModel } from "stagehand-v3"; -import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; -import { datasetPromptGuidance, type ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; -import type { PreparedFxToolAdapter } from "./fxToolAdapter.js"; -import { readFxMaxAgentSteps } from "./fxToolAdapter.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { readFxMaxAgentSteps, type PreparedFxToolAdapter } from "./fxToolAdapter.js"; +import { + buildExternalHarnessPrompt, + metricValue, + parseEvalResult, + runExternalHarnessTask, + type ExternalHarnessToolAdapterLike, + type MetricValue, + type ParsedEvalResult, +} from "./harnesses/externalRunner.js"; import { fxAdapter } from "./harnesses/fxAdapter.js"; import type { TaskResult } from "./types.js"; -import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; +import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; export { buildFxTranscript, @@ -26,68 +31,29 @@ export { runFxSession, } from "@browserbasehq/stagehand-integrations-fx-sdk"; -type MetricValue = { count: number; value: number }; - export interface FxRunnerInput { plan: ExternalHarnessTaskPlan; model: AvailableModel; logger: EvalLogger; - toolAdapter?: PreparedFxToolAdapter; + toolAdapter: PreparedFxToolAdapter; signal?: AbortSignal; verifier?: ExternalHarnessVerifierConfig; runProcess?: FxProcessRunner; store?: FxSessionStore; } -export interface ParsedFxResult { - success: boolean; - summary?: string; - finalAnswer?: string; - raw: string; -} +export interface ParsedFxResult extends ParsedEvalResult {} export function buildFxPrompt(plan: ExternalHarnessTaskPlan, toolInstructions?: string): string { - return [ - "You are running a browser benchmark task.", - "", - `Dataset: ${plan.dataset}`, - plan.taskId ? `Task ID: ${plan.taskId}` : undefined, - `Start URL: ${plan.startUrl}`, - "", - "Instruction:", - plan.instruction, - "", - datasetPromptGuidance(plan.dataset), - toolInstructions ?? "Use the available browser/web tools to complete the task.", - "Do not edit repository files.", - "At the end, return compact JSON matching this schema:", - '{"success": boolean, "summary": string, "finalAnswer": string}', - ] - .filter(Boolean) - .join("\n"); + return buildExternalHarnessPrompt({ + plan, + toolInstructions, + resultContract: "structured_output", + }); } export function parseFxResult(raw: string): ParsedFxResult { - const marker = "EVAL_RESULT:"; - const markerIndex = raw.lastIndexOf(marker); - const candidates = - markerIndex >= 0 - ? [ - raw.slice(markerIndex + marker.length).trim(), - raw - .slice(markerIndex + marker.length) - .trim() - .split(/\r?\n/u, 1)[0] - ?.trim(), - ] - : [raw.trim(), raw.trim().split(/\r?\n/u, 1)[0]?.trim()]; - - for (const candidate of candidates) { - if (!candidate) continue; - const parsed = tryParseFxJson(candidate); - if (parsed) return { ...parsed, raw }; - } - return { success: false, raw }; + return parseEvalResult(raw); } export async function runFxAgent({ @@ -100,113 +66,104 @@ export async function runFxAgent({ runProcess, store, }: FxRunnerInput): Promise { - if (!toolAdapter) throw new EvalsError("fx requires a prepared tool adapter."); - const prompt = buildFxPrompt(plan, toolAdapter.promptInstructions); - const sessionResult = await runFxSession({ - prompt, - model: normalizeFxModel(model), - cwd: toolAdapter.cwd, - home: toolAdapter.home, - env: toolAdapter.env, - permissionMode: process.env.EVAL_FX_PERMISSION_MODE === "yolo" ? "yolo" : "auto", - maxAgentSteps: readFxMaxAgentSteps(), - signal, - logger, - runProcess, - store, - onToolStep: toolAdapter.recordObservation - ? async () => toolAdapter.recordObservation?.() - : undefined, - observedTool: toolAdapter.observedToolMatcher, - }); - const { events, finalMessage, iterationError, status, stopReason, tokenUsage } = sessionResult; - const transcriptText = buildFxTranscript(events); - const iterationErrorMessage = stringifyError(iterationError); - const rawResult = [finalMessage, transcriptText, iterationErrorMessage] - .filter(Boolean) - .join("\n\n"); - const parsed = parseFxResult(rawResult); - const errorMessage = - parsed.summary ?? - stopReason ?? - (iterationErrorMessage || finalMessage || transcriptText || "fx did not report success"); - const baseResult: TaskResult = { - _success: parsed.success, - error: !parsed.success ? errorMessage : undefined, - reasoning: parsed.summary, - finalAnswer: parsed.finalAnswer, - rawResult: parsed.raw, - fxStatus: status, - ...(stopReason && { fxStopReason: stopReason }), - logs: logger.getLogs(), - metrics: buildFxMetrics(tokenUsage), + const adapterLike: ExternalHarnessToolAdapterLike = { + promptInstructions: toolAdapter.promptInstructions, + captureEvidence: toolAdapter.captureEvidence, + drainStepObservations: toolAdapter.drainStepObservations, + observedToolMatcher: toolAdapter.observedToolMatcher, }; - if (!verifier) return baseResult; - - const finalObservation = await toolAdapter.captureEvidence?.().catch((): undefined => undefined); - const stepObservations = await toolAdapter.drainStepObservations?.(); - return gradeExternalTrajectory({ - buildTrajectory: () => + return runExternalHarnessTask({ + harness: "fx", + plan, + logger, + toolAdapter: adapterLike, + verifier, + resultContract: "structured_output", + fallbackErrorMessage: "fx did not report success", + runSession: async (prompt) => { + const sessionResult = await runFxSession({ + prompt, + model: normalizeFxModel(model), + cwd: toolAdapter.cwd, + home: toolAdapter.home, + env: toolAdapter.env, + permissionMode: process.env.EVAL_FX_PERMISSION_MODE === "yolo" ? "yolo" : "auto", + maxAgentSteps: readFxMaxAgentSteps(), + signal, + logger, + runProcess, + store, + onToolStep: toolAdapter.recordObservation + ? async () => toolAdapter.recordObservation?.() + : undefined, + observedTool: toolAdapter.observedToolMatcher, + }); + const usage = normalizeFxUsage(sessionResult.tokenUsage); + return { + raw: sessionResult, + resultText: sessionResult.finalMessage, + transcriptText: buildFxTranscript(sessionResult.events), + iterationError: sessionResult.iterationError, + status: sessionResult.status, + stopReason: + sessionResult.stopReason || + (sessionResult.status === "sdk_error" + ? stringifyError(sessionResult.iterationError) || undefined + : undefined), + usage, + ...(typeof sessionResult.tokenUsage.total_cost === "number" && + Number.isFinite(sessionResult.tokenUsage.total_cost) && { + costUsd: sessionResult.tokenUsage.total_cost, + }), + metrics: buildFxMetrics(sessionResult.tokenUsage), + }; + }, + toTrajectory: ( + { raw, parsed, finalObservation, stepObservations, observedToolName, status }, + taskSpec, + ) => fxAdapter.fromHarnessResult( { - events, + events: raw.events, ...(finalObservation && { finalObservation }), ...(stepObservations?.length && { stepObservations }), - ...(toolAdapter.observedToolMatcher && { - observedToolName: toolAdapter.observedToolMatcher, - }), - finalAnswer: parsed.finalAnswer ?? finalMessage, - status: status === "completed" ? "complete" : "error", + ...(observedToolName && { observedToolName }), + finalAnswer: parsed.finalAnswer ?? raw.finalMessage, + status, usage: { - input_tokens: tokenUsage.input_tokens, - output_tokens: tokenUsage.output_tokens, - reasoning_tokens: tokenUsage.reasoning_output_tokens, - cached_input_tokens: tokenUsage.cached_input_tokens, + input_tokens: raw.tokenUsage.input_tokens, + output_tokens: raw.tokenUsage.output_tokens, + reasoning_tokens: raw.tokenUsage.reasoning_output_tokens, + cached_input_tokens: raw.tokenUsage.cached_input_tokens, }, }, - verifier.taskSpec, + taskSpec, ), - verifier, - baseResult, - errorMessage, - category: "fx", - logger, }); } -function tryParseFxJson(candidate: string): Omit | undefined { - try { - const parsed = JSON.parse(candidate) as { - success?: unknown; - summary?: unknown; - finalAnswer?: unknown; - }; - return { - success: parsed.success === true, - summary: typeof parsed.summary === "string" ? parsed.summary : undefined, - finalAnswer: typeof parsed.finalAnswer === "string" ? parsed.finalAnswer : undefined, - }; - } catch { - return undefined; - } -} - -function buildFxMetrics(usage: FxTokenUsage): Record { +function normalizeFxUsage(usage: FxTokenUsage) { const inputTokens = toFiniteNumber(usage.input_tokens); const cachedInputTokens = toFiniteNumber(usage.cached_input_tokens); const outputTokens = toFiniteNumber(usage.output_tokens); const reasoningOutputTokens = toFiniteNumber(usage.reasoning_output_tokens); + // fx reports cached input and reasoning output as separate token buckets. return { - fx_input_tokens: metricValue(inputTokens), - fx_cached_input_tokens: metricValue(cachedInputTokens), - fx_output_tokens: metricValue(outputTokens), - fx_reasoning_output_tokens: metricValue(reasoningOutputTokens), - fx_total_tokens: metricValue( - inputTokens + cachedInputTokens + outputTokens + reasoningOutputTokens, - ), + inputTokens, + cachedInputTokens, + outputTokens, + reasoningOutputTokens, + totalTokens: inputTokens + cachedInputTokens + outputTokens + reasoningOutputTokens, }; } -function metricValue(value: unknown): MetricValue { - return { count: 1, value: toFiniteNumber(value) }; +function buildFxMetrics(usage: FxTokenUsage): Record { + const normalized = normalizeFxUsage(usage); + return { + fx_input_tokens: metricValue(normalized.inputTokens), + fx_cached_input_tokens: metricValue(normalized.cachedInputTokens), + fx_output_tokens: metricValue(normalized.outputTokens), + fx_reasoning_output_tokens: metricValue(normalized.reasoningOutputTokens), + fx_total_tokens: metricValue(normalized.totalTokens), + }; } diff --git a/packages/evals/framework/fxToolAdapter.ts b/packages/evals/framework/fxToolAdapter.ts index b99aaa817..955eaaef9 100644 --- a/packages/evals/framework/fxToolAdapter.ts +++ b/packages/evals/framework/fxToolAdapter.ts @@ -7,6 +7,7 @@ import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; import { startAgentToolRuntime } from "./agentToolRuntime.js"; import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { resolveStartupProfile, resolveToolSurface } from "./harnesses/toolSurfaceResolution.js"; import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; export interface FxToolAdapterInput { @@ -38,38 +39,11 @@ type FxMcpServerSpec = { env?: Record; }; -const FX_MCP_SURFACES = new Set([ +export const FX_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", "playwright_mcp", "chrome_devtools_mcp", -]); - -export function resolveFxToolSurface(requested?: ToolSurface): ToolSurface { - if (!requested) return "stagehand_facade"; - if (FX_MCP_SURFACES.has(requested)) return requested; - throw new EvalsError( - `fx harness supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "${requested}".`, - ); -} - -export function resolveFxStartupProfile( - toolSurface: ToolSurface, - environment: "LOCAL" | "BROWSERBASE", - requested?: StartupProfile, -): StartupProfile { - if (requested) return requested; - if (toolSurface === "stagehand_facade") { - return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; - } - if (toolSurface === "playwright_mcp" || toolSurface === "chrome_devtools_mcp") { - return environment === "BROWSERBASE" - ? "runner_provided_browserbase_cdp" - : "runner_provided_local_cdp"; - } - throw new EvalsError( - `No fx startup profile default for tool "${toolSurface}" in ${environment}.`, - ); -} +]; export function buildFxMcpConfig( mcpServers: Record, @@ -139,8 +113,14 @@ export function buildFxAgentsMarkdown(promptInstructions: string, serverNames: s export async function prepareFxToolAdapter( input: FxToolAdapterInput, ): Promise { - const toolSurface = resolveFxToolSurface(input.toolSurface); - const startupProfile = resolveFxStartupProfile( + const toolSurface = resolveToolSurface( + { harness: "fx", supportedToolSurfaces: FX_TOOL_SURFACES }, + input.toolSurface, + ); + if (toolSurface === undefined) { + throw new EvalsError("fx harness requires a tool surface."); + } + const startupProfile = resolveStartupProfile( toolSurface, input.environment, input.startupProfile, diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index 256a1ac66..550dfe2df 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -12,6 +12,7 @@ import { listBenchHarnessesForToolSurface, mastraHarness, piHarness, + fxHarness, deepagentsHarness, eveHarness, registerBenchHarness, @@ -34,6 +35,7 @@ describe("bench harness registry", () => { "pi", "eve", "deepagents", + "fx", ]); }); @@ -41,7 +43,7 @@ describe("bench harness registry", () => { expect(parseBenchHarness(undefined)).toBe("stagehand"); expect(parseBenchHarness("codex")).toBe("codex"); expect(() => parseBenchHarness("nope")).toThrow( - /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents\./, + /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx\./, ); }); @@ -141,6 +143,23 @@ describe("bench harness registry", () => { expect(isExecutableBenchHarness("deepagents")).toBe(true); }); + it("registers fx as a concrete executable harness", () => { + const harness = getBenchHarness("fx"); + + expect(harness).toBe(fxHarness); + expect(harness.supportedTaskKinds).toEqual(["agent", "suite"]); + expect(harness.supportsApi).toBe(false); + expect(harness.execute).toBeDefined(); + expect(harness.start).toBeUndefined(); + expect(harness.supportedToolSurfaces).toEqual([ + "stagehand_facade", + "playwright_mcp", + "chrome_devtools_mcp", + ]); + expect(harness.defaultModels).toEqual(["openai/gpt-5.4-mini"]); + expect(isExecutableBenchHarness("fx")).toBe(true); + }); + it("registers a new harness and rejects duplicate ids", () => { const fakeHarness = { harness: "fake_harness", diff --git a/packages/evals/tests/framework/fxRunner.test.ts b/packages/evals/tests/framework/fxRunner.test.ts index 4dcd0431b..5c0328ef5 100644 --- a/packages/evals/tests/framework/fxRunner.test.ts +++ b/packages/evals/tests/framework/fxRunner.test.ts @@ -89,7 +89,46 @@ describe("fx runner helpers", () => { expect(result._success).toBe(true); expect(result.error).toBeUndefined(); expect(result.fxStatus).toBe("completed"); + expect(result.harnessStatus).toBe("completed"); expect(result.finalAnswer).toBe("Example Domain"); expect(metrics.fx_input_tokens.value).toBe(42); + expect(metrics.harness_input_tokens.value).toBe(42); + expect(metrics.harness_output_tokens.value).toBe(8); + expect(metrics.harness_cached_input_tokens.value).toBe(5); + expect(metrics.harness_reasoning_output_tokens.value).toBe(2); + expect(metrics.fx_total_tokens.value).toBe(57); + expect(metrics.harness_total_tokens.value).toBe(57); + expect(metrics.harness_cost_usd).toBeUndefined(); + }); + + it("returns a failed task result with sdk_error status when fx cannot start", async () => { + const result = await runFxAgent({ + plan, + model: "openai/gpt-5.6-sol" as AvailableModel, + logger: new EvalLogger(false), + toolAdapter: { + toolSurface: "stagehand_facade", + startupProfile: "tool_launch_local", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: { PATH: "/bin" }, + promptInstructions: "Use mcp_stagehand_snapshot.", + mcpServerNames: ["stagehand"], + cleanup: async () => {}, + }, + runProcess: async () => { + throw new Error("MissingCredentials"); + }, + store: { + waitForSessionDir: async () => undefined, + readEventsJsonl: async () => "", + }, + }); + + expect(result._success).toBe(false); + expect(result.fxStatus).toBe("sdk_error"); + expect(result.harnessStatus).toBe("sdk_error"); + expect(result.harnessStopReason).toBeDefined(); + expect(String(result.error)).not.toBe(""); }); }); diff --git a/packages/evals/tests/framework/fxToolAdapter.test.ts b/packages/evals/tests/framework/fxToolAdapter.test.ts index 973f2d956..733168aaf 100644 --- a/packages/evals/tests/framework/fxToolAdapter.test.ts +++ b/packages/evals/tests/framework/fxToolAdapter.test.ts @@ -1,35 +1,28 @@ import { describe, expect, it } from "vitest"; -import type { ToolSurface } from "../../core/contracts/tool.js"; +import { fxHarness } from "../../framework/benchHarness.js"; import { buildFxAgentsMarkdown, buildFxMcpConfig, buildFxSettings, - resolveFxStartupProfile, - resolveFxToolSurface, + FX_TOOL_SURFACES, } from "../../framework/fxToolAdapter.js"; +import { + resolveStartupProfile, + resolveToolSurface, +} from "../../framework/harnesses/toolSurfaceResolution.js"; describe("fx tool adapter helpers", () => { - it("defaults to the Stagehand facade and rejects unsupported surfaces", () => { - expect(resolveFxToolSurface()).toBe("stagehand_facade"); - expect(resolveFxToolSurface("playwright_mcp")).toBe("playwright_mcp"); - expect(resolveFxToolSurface("chrome_devtools_mcp")).toBe("chrome_devtools_mcp"); - expect(() => resolveFxToolSurface("browse_cli")).toThrow( - 'fx harness supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "browse_cli".', - ); - }); - - it("chooses surface-specific startup profiles", () => { - expect(resolveFxStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); - expect(resolveFxStartupProfile("stagehand_facade", "BROWSERBASE")).toBe( - "tool_create_browserbase", + it("resolves surfaces and startup profiles through the shared registry helpers", () => { + expect(FX_TOOL_SURFACES).toEqual(["stagehand_facade", "playwright_mcp", "chrome_devtools_mcp"]); + expect(resolveToolSurface(fxHarness)).toBe("stagehand_facade"); + expect(resolveToolSurface(fxHarness, "playwright_mcp")).toBe("playwright_mcp"); + expect(() => resolveToolSurface(fxHarness, "browse_cli")).toThrow( + 'Harness "fx" supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "browse_cli".', ); - expect(resolveFxStartupProfile("playwright_mcp", "LOCAL")).toBe("runner_provided_local_cdp"); - expect(resolveFxStartupProfile("chrome_devtools_mcp", "BROWSERBASE")).toBe( + expect(resolveStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); + expect(resolveStartupProfile("chrome_devtools_mcp", "BROWSERBASE")).toBe( "runner_provided_browserbase_cdp", ); - expect(resolveFxStartupProfile("playwright_mcp", "LOCAL", "tool_attach_local_cdp")).toBe( - "tool_attach_local_cdp", - ); }); it("builds fx MCP launch specs with an explicit child environment", () => { @@ -92,10 +85,4 @@ describe("fx tool adapter helpers", () => { expect(markdown).toContain("mcp_stagehand_screenshot"); expect(markdown).toContain("Use snapshots first."); }); - - it("rejects a startup default for an unrelated surface", () => { - expect(() => resolveFxStartupProfile("browse_cli" as ToolSurface, "LOCAL")).toThrow( - /No fx startup profile default/u, - ); - }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 666ac4c96..683061920 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,202 +1,3 @@ ---- -lockfileVersion: '9.0' - -importers: - - .: - configDependencies: {} - packageManagerDependencies: - '@pnpm/exe': - specifier: 11.10.0 - version: 11.10.0 - pnpm: - specifier: 11.10.0 - version: 11.10.0 - -packages: - - '@pnpm/exe@11.10.0': - resolution: {integrity: sha512-mrmfi2C7LpZkyq0voKKye6MzrK8/K7tYQRiSB/jqOiwnFtQxxcA3xGTY9cEsrGpNl2ClPecQofLuj+BO8AIsxw==} - hasBin: true - - '@pnpm/linux-arm64@11.10.0': - resolution: {integrity: sha512-NbvDeUfs0SJuli9OPvgVvmnlbo2DvJ861XGXKrzgLu5AuTnLDLXgbZEEUd8mJ5I0YNrqOVXSWVfWEqiAazWzPA==} - cpu: [arm64] - os: [linux] - - '@pnpm/linux-x64@11.10.0': - resolution: {integrity: sha512-kdgb8BXZ/3XQ0x2cOmgLmsij7+SUIqd1bcV6OZhdGzQiDrOY6FAPrR+Y2Bp+NjrrhjzMHVm5pZrrmdeC67ymSQ==} - cpu: [x64] - os: [linux] - - '@pnpm/linuxstatic-arm64@11.10.0': - resolution: {integrity: sha512-JE1WrSyKGvqGQgWzqrMXn//ehedpiRax3hi1oP+v6mhvcnlJD1lpfXsjSfIFl9weTWC5KVtFbxSNKbKWfP+v6g==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@pnpm/linuxstatic-x64@11.10.0': - resolution: {integrity: sha512-1TBZVRkWb78GnsusIfVgwz20MOOW0fyehW+qLL3MxE9vT0IedNWsAhe3KWXgEvtC4M7NtMt55uQQJm+1egwBkA==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@pnpm/macos-arm64@11.10.0': - resolution: {integrity: sha512-94AVpPixBqyNT6SHYvIKFb1bfaHR5vxBzZsiFJahNSlGkgyPrgzmcqDiloZ/Jl+zxJd8L5PU1ddP0Q9PZnRqlA==} - cpu: [arm64] - os: [darwin] - - '@pnpm/win-arm64@11.10.0': - resolution: {integrity: sha512-g2Ymnq+LgVyZaWsGBQSlpIcBCOOxyLky2UX+kTwGiIXnj6k/xXqZR0ZJLuxeiGNh/CVmhftOqokpyJNzyj8kng==} - cpu: [arm64] - os: [win32] - - '@pnpm/win-x64@11.10.0': - resolution: {integrity: sha512-kCHYZudUEBjrEchgnJUnNHCdvLXpUZun2z0rMAeP5DymsaXwEVvVzGj220iR/NyhgDocJUmgtTKtwL/ejL785Q==} - cpu: [x64] - os: [win32] - - '@reflink/reflink-darwin-arm64@0.1.19': - resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@reflink/reflink-darwin-x64@0.1.19': - resolution: {integrity: sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@reflink/reflink-linux-arm64-gnu@0.1.19': - resolution: {integrity: sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@reflink/reflink-linux-arm64-musl@0.1.19': - resolution: {integrity: sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@reflink/reflink-linux-x64-gnu@0.1.19': - resolution: {integrity: sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@reflink/reflink-linux-x64-musl@0.1.19': - resolution: {integrity: sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [musl] - - '@reflink/reflink-win32-arm64-msvc@0.1.19': - resolution: {integrity: sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [win32] - - '@reflink/reflink-win32-x64-msvc@0.1.19': - resolution: {integrity: sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@reflink/reflink@0.1.19': - resolution: {integrity: sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA==} - engines: {node: '>= 10'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - pnpm@11.10.0: - resolution: {integrity: sha512-C3+LmAYAMZBMAX46QesYehbUDuuCm5XE+MsDaBdh/Eq1PdIZEVubRH9NzhoFohR2RGHn03AzkqnzL5URzoyGyA==} - engines: {node: '>=22.13'} - hasBin: true - -snapshots: - - '@pnpm/exe@11.10.0': - dependencies: - '@reflink/reflink': 0.1.19 - detect-libc: 2.1.2 - optionalDependencies: - '@pnpm/linux-arm64': 11.10.0 - '@pnpm/linux-x64': 11.10.0 - '@pnpm/linuxstatic-arm64': 11.10.0 - '@pnpm/linuxstatic-x64': 11.10.0 - '@pnpm/macos-arm64': 11.10.0 - '@pnpm/win-arm64': 11.10.0 - '@pnpm/win-x64': 11.10.0 - - '@pnpm/linux-arm64@11.10.0': - optional: true - - '@pnpm/linux-x64@11.10.0': - optional: true - - '@pnpm/linuxstatic-arm64@11.10.0': - optional: true - - '@pnpm/linuxstatic-x64@11.10.0': - optional: true - - '@pnpm/macos-arm64@11.10.0': - optional: true - - '@pnpm/win-arm64@11.10.0': - optional: true - - '@pnpm/win-x64@11.10.0': - optional: true - - '@reflink/reflink-darwin-arm64@0.1.19': - optional: true - - '@reflink/reflink-darwin-x64@0.1.19': - optional: true - - '@reflink/reflink-linux-arm64-gnu@0.1.19': - optional: true - - '@reflink/reflink-linux-arm64-musl@0.1.19': - optional: true - - '@reflink/reflink-linux-x64-gnu@0.1.19': - optional: true - - '@reflink/reflink-linux-x64-musl@0.1.19': - optional: true - - '@reflink/reflink-win32-arm64-msvc@0.1.19': - optional: true - - '@reflink/reflink-win32-x64-msvc@0.1.19': - optional: true - - '@reflink/reflink@0.1.19': - optionalDependencies: - '@reflink/reflink-darwin-arm64': 0.1.19 - '@reflink/reflink-darwin-x64': 0.1.19 - '@reflink/reflink-linux-arm64-gnu': 0.1.19 - '@reflink/reflink-linux-arm64-musl': 0.1.19 - '@reflink/reflink-linux-x64-gnu': 0.1.19 - '@reflink/reflink-linux-x64-musl': 0.1.19 - '@reflink/reflink-win32-arm64-msvc': 0.1.19 - '@reflink/reflink-win32-x64-msvc': 0.1.19 - - detect-libc@2.1.2: {} - - pnpm@11.10.0: {} - ---- lockfileVersion: '9.0' settings: @@ -443,7 +244,7 @@ importers: version: 3.1.1 mint: specifier: 'catalog:' - version: 4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) + version: 4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) packages/evals: dependencies: @@ -468,9 +269,21 @@ importers: '@browserbasehq/stagehand-integrations-codex-sdk': specifier: workspace:* version: link:../integrations/codex-sdk + '@browserbasehq/stagehand-integrations-deepagents-sdk': + specifier: workspace:* + version: link:../integrations/deepagents-sdk + '@browserbasehq/stagehand-integrations-eve-sdk': + specifier: workspace:* + version: link:../integrations/eve-sdk '@browserbasehq/stagehand-integrations-fx-sdk': specifier: workspace:* version: link:../integrations/fx-sdk + '@browserbasehq/stagehand-integrations-mastra-sdk': + specifier: workspace:* + version: link:../integrations/mastra-sdk + '@browserbasehq/stagehand-integrations-pi-sdk': + specifier: workspace:* + version: link:../integrations/pi-sdk ai: specifier: ^5.0.133 version: 5.0.220(zod@4.4.3) @@ -507,7 +320,7 @@ importers: version: 24.13.2 braintrust: specifier: ^0.4.10 - version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(supports-color@8.1.1)(zod@4.4.3) + version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3) chalk: specifier: ^5.4.1 version: 5.6.2 @@ -707,6 +520,25 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/deepagents-sdk: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/eve: dependencies: '@ai-sdk/openai': @@ -735,6 +567,46 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/eve-sdk: + dependencies: + '@ai-sdk/anthropic': + specifier: 'catalog:' + version: 4.0.8(zod@4.4.3) + '@ai-sdk/google': + specifier: 'catalog:' + version: 4.0.8(zod@4.4.3) + '@ai-sdk/openai': + specifier: 'catalog:' + version: 4.0.8(zod@4.4.3) + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@modelcontextprotocol/sdk': + specifier: 'catalog:' + version: 1.29.0(zod@4.4.3) + ai: + specifier: 'catalog:' + version: 7.0.16(zod@4.4.3) + eve: + specifier: 'catalog:' + version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/fx-sdk: dependencies: '@browserbasehq/stagehand-integrations': @@ -779,6 +651,34 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/mastra-sdk: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@mastra/core': + specifier: 'catalog:' + version: 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + '@mastra/mcp': + specifier: 'catalog:' + version: 1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/pi: dependencies: '@browserbasehq/stagehand': @@ -804,6 +704,34 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/pi-sdk: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@earendil-works/pi-coding-agent': + specifier: 'catalog:' + version: 0.84.2(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) + '@modelcontextprotocol/sdk': + specifier: 'catalog:' + version: 1.29.0(zod@4.4.3) + typebox: + specifier: 'catalog:' + version: 1.3.7 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/vercel-ai: dependencies: '@ai-sdk/mcp': @@ -954,6 +882,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@2.0.118': + resolution: {integrity: sha512-9MZXPd1wiELUvAfHXdotmCp2tt6NWoT6OACkRvK1xQy9DG7XeOZ9ZPqFmCqcu5DcmN92BGpA/GtmilrOM+hVJQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@2.0.119': resolution: {integrity: sha512-LTbVThusUYSw6SxsRvdCKveVHCFf+3DBU3XY8+RgBYPBSjSfeKkVMirvGkcj7Hvwd5aTptioHpnreVXzhwjpVw==} engines: {node: '>=18'} @@ -1311,6 +1245,10 @@ packages: resolution: {integrity: sha512-jaTESuJlQsoUZ44f/i2puyPt8VlF/dMMJ9HM3cStYtk7eKX4N9UWi83OLixUkoOJH3BwWlPLCq9YIK9nfWhVBg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.71': + resolution: {integrity: sha512-HIg7Q2osBzajQwL+1Vkyh2E7Gim3eTNb9RHIsOxDGjW0eZg4oEKtRs5sioCnc73ilhaOm4gX2lHVF8J7+nt2rg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.79': resolution: {integrity: sha512-RIw5dof1EHkWubrZzPC941CDtnFG1iAXsxbFgLkhdYZXHc4icU13c/uxSMI0J5eUx9bxa7LjfpdjfClBB1QsDA==} engines: {node: '>=20.0.0'} @@ -1359,8 +1297,8 @@ packages: resolution: {integrity: sha512-ECAqfpNsef+7MO8qtR0h9KcFIBAygaE7Cm6UOiQl+ft+uVap+1G7bNEjs4mdJE2OnA4m6k7i8peH8uGIAsOMGw==} engines: {node: '>=20.0.0'} - '@aws-sdk/util-locate-window@3.965.9': - resolution: {integrity: sha512-wB/ho7pTJKqWz3WYDt2ZWDWI8bxQpN/xwf+5ZQ1zWaj+HDY9B8Fn434i6qZ6j6ZG3aCiIJtZaQVqwajx5xYsQA==} + '@aws-sdk/util-locate-window@3.965.8': + resolution: {integrity: sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==} engines: {node: '>=20.0.0'} '@aws-sdk/xml-builder@3.972.38': @@ -1905,8 +1843,8 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@hono/node-server@1.19.15': - resolution: {integrity: sha512-Za2ai6TLdKjUvnur+eenO6nuYYipVAEhyCAdaV8IRvmU9kK8crOZUSYvIXn72E4f8fJqyAbpcJuTsYYmZp9Deg==} + '@hono/node-server@1.19.14': + resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} engines: {node: '>=18.14.1'} peerDependencies: hono: ^4 @@ -2982,8 +2920,8 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} - '@posthog/core@1.46.9': - resolution: {integrity: sha512-EXO6y5ih+jBkTCpUCuYgQmajuDZuvy6vMvflkub6pLQyi0GPlCPWVSvZZkOeQw9e2MxoD5GteeGCt9R8+UJ/yQ==} + '@posthog/core@1.39.6': + resolution: {integrity: sha512-o6ajIwN5zXoNP0D4H/QPmOyibNTUkSyOR6ya7AG5U2ywXx4awo72L2KnCoiZPQM5x/bXv6jPBdimH8M18Ax0aw==} '@posthog/core@1.7.1': resolution: {integrity: sha512-kjK0eFMIpKo9GXIbts8VtAknsoZ18oZorANdtuTj1CbgS28t4ZVq//HAWhnxEuXRTrtkd+SUJ6Ux3j2Af8NCuA==} @@ -3572,6 +3510,12 @@ packages: resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==} engines: {node: '>=12'} + ai@5.0.219: + resolution: {integrity: sha512-bFjV5roRz/CqcSuFR+cfOU35O/7Z9U/2y5DyMIJUx6igkAdIMJ3HWNh+tOf9Gsrqiqbbjs7YMMidHi2HsW2PMg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ai@5.0.220: resolution: {integrity: sha512-8v7IFO+OMjVJeprLSNemO9GnDMBcoslid1SlxzhxlqPLnFS6o2uI+V5enEYZ3L0rLcu5aXeNilqP8VMccgyq6A==} engines: {node: '>=18'} @@ -5149,8 +5093,8 @@ packages: highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} - hono@4.12.32: - resolution: {integrity: sha512-XcuyW9qE2kJn07PkecMOBd5Vq/hMy7mmGw+idz1yblbg9N17ijJODrvPkn7/dwL3Kulj8LcRJ69DLOWf91dRUg==} + hono@4.12.31: + resolution: {integrity: sha512-zJIHFrl6bq3RDd2YusFNCDlM8qUprxKswyi/OPzPyzKDdyBXDqWx8bZlZ7R+saTdSTatUmb3O7K4SspGPaEOQg==} engines: {node: '>=16.9.0'} hookable@6.1.1: @@ -5854,8 +5798,8 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} - media-typer@1.1.1: - resolution: {integrity: sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==} + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} merge-descriptors@1.0.3: @@ -6756,8 +6700,8 @@ packages: resolution: {integrity: sha512-lz3YJOr0Nmiz0yHASaINEDHqoV+0bC3eD8aZAG+Ky292dAnVYul+ga/dMX8KCBXg8hHfKdxw0SztYD5j6dgUqQ==} engines: {node: '>=20'} - posthog-node@5.48.1: - resolution: {integrity: sha512-BxLX2SqGEQhPqCPTalpyo0RRv1NMbf7UaN7q9d/ED77ksD6XOmE7ko2vIKO8F0zPL1NtKxIi+DYXap9lvR0RaA==} + posthog-node@5.40.0: + resolution: {integrity: sha512-DrLfHuauO0W6qruF80iqr5JdmLysef74XzOB4eh36oRLRhxCySLraTqsi2Pj161LZnp9/JNdRDxwT8ei8VK2YA==} engines: {node: ^20.20.0 || >=22.22.0} peerDependencies: rxjs: ^7.0.0 @@ -8261,6 +8205,13 @@ snapshots: zod: 4.4.3 optional: true + '@ai-sdk/gateway@2.0.118(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 2.0.3 + '@ai-sdk/provider-utils': 3.0.30(zod@4.4.3) + '@vercel/oidc': 3.1.0 + zod: 4.4.3 + '@ai-sdk/gateway@2.0.119(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.3 @@ -8595,7 +8546,7 @@ snapshots: '@aws-crypto/supports-web-crypto': 5.2.0 '@aws-crypto/util': 5.2.0 '@aws-sdk/types': 3.974.3 - '@aws-sdk/util-locate-window': 3.965.9 + '@aws-sdk/util-locate-window': 3.965.8 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -8620,16 +8571,16 @@ snapshots: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/core': 3.977.7 - '@aws-sdk/credential-provider-node': 3.972.79 + '@aws-sdk/credential-provider-node': 3.972.71 '@aws-sdk/eventstream-handler-node': 3.972.32 '@aws-sdk/middleware-eventstream': 3.972.27 '@aws-sdk/middleware-websocket': 3.972.50 '@aws-sdk/token-providers': 3.1048.0 '@aws-sdk/types': 3.974.3 - '@smithy/core': 3.30.0 + '@smithy/core': 3.32.0 '@smithy/fetch-http-handler': 5.7.0 - '@smithy/node-http-handler': 4.7.3 - '@smithy/types': 4.16.1 + '@smithy/node-http-handler': 4.10.0 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@aws-sdk/core@3.977.7': @@ -8686,6 +8637,20 @@ snapshots: '@smithy/types': 4.17.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.71': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.68 + '@aws-sdk/credential-provider-http': 3.972.70 + '@aws-sdk/credential-provider-ini': 3.973.13 + '@aws-sdk/credential-provider-process': 3.972.68 + '@aws-sdk/credential-provider-sso': 3.973.12 + '@aws-sdk/credential-provider-web-identity': 3.972.74 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.32.0 + '@smithy/credential-provider-imds': 4.5.0 + '@smithy/types': 4.17.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.79': dependencies: '@aws-sdk/credential-provider-env': 3.972.68 @@ -8699,6 +8664,7 @@ snapshots: '@smithy/credential-provider-imds': 4.5.0 '@smithy/types': 4.17.0 tslib: 2.8.1 + optional: true '@aws-sdk/credential-provider-process@3.972.68': dependencies: @@ -8731,14 +8697,14 @@ snapshots: dependencies: '@aws-sdk/types': 3.974.3 '@smithy/core': 3.32.0 - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@aws-sdk/middleware-eventstream@3.972.27': dependencies: '@aws-sdk/types': 3.974.3 '@smithy/core': 3.32.0 - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@aws-sdk/middleware-websocket@3.972.50': @@ -8748,7 +8714,7 @@ snapshots: '@smithy/core': 3.32.0 '@smithy/fetch-http-handler': 5.7.0 '@smithy/signature-v4': 5.7.0 - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@aws-sdk/nested-clients@3.997.42': @@ -8774,8 +8740,8 @@ snapshots: '@aws-sdk/core': 3.977.7 '@aws-sdk/nested-clients': 3.997.42 '@aws-sdk/types': 3.974.3 - '@smithy/core': 3.30.0 - '@smithy/types': 4.16.1 + '@smithy/core': 3.32.0 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@aws-sdk/token-providers@3.1108.0': @@ -8792,7 +8758,7 @@ snapshots: '@smithy/types': 4.17.0 tslib: 2.8.1 - '@aws-sdk/util-locate-window@3.965.9': + '@aws-sdk/util-locate-window@3.965.8': dependencies: tslib: 2.8.1 @@ -8877,7 +8843,7 @@ snapshots: '@browserbasehq/sdk': 2.16.0 '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - ai: 5.0.220(zod@4.4.3) + ai: 5.0.219(zod@4.4.3) devtools-protocol: 0.0.1642743 fetch-cookie: 3.2.0 openai: 4.104.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) @@ -9399,9 +9365,9 @@ snapshots: - supports-color - utf-8-validate - '@hono/node-server@1.19.15(hono@4.12.32)': + '@hono/node-server@1.19.14(hono@4.12.31)': dependencies: - hono: 4.12.32 + hono: 4.12.31 '@img/colour@1.1.0': {} @@ -9747,7 +9713,7 @@ snapshots: dependencies: jsep: 1.4.0 - '@kwsites/file-exists@1.1.1(supports-color@8.1.1)': + '@kwsites/file-exists@1.1.1': dependencies: debug: 4.4.3(supports-color@8.1.1) transitivePeerDependencies: @@ -9852,7 +9818,7 @@ snapshots: p-map: 7.0.6 p-retry: 7.1.1 picomatch: 4.0.5 - posthog-node: 5.48.1(rxjs@7.8.2) + posthog-node: 5.40.0(rxjs@7.8.2) tokenx: 1.6.0 ws: 8.21.0(bufferutil@4.1.0) xxhash-wasm: 1.1.0 @@ -9927,7 +9893,7 @@ snapshots: '@types/react': 19.2.17 react: 19.2.3 - '@mintlify/cli@4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3)': + '@mintlify/cli@4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3)': dependencies: '@inquirer/prompts': 7.9.0(@types/node@25.9.4) '@mintlify/common': 1.0.1080(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -9939,7 +9905,7 @@ snapshots: adm-zip: 0.6.0 chalk: 5.2.0 color: 4.2.3 - detect-port: 1.5.1(supports-color@8.1.1) + detect-port: 1.5.1 fs-extra: 11.2.0 ink: 6.3.0(@types/react@19.2.17)(bufferutil@4.1.0)(react@19.2.3) inquirer: 12.3.0(@types/node@25.9.4) @@ -10246,7 +10212,7 @@ snapshots: '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: - '@hono/node-server': 1.19.15(hono@4.12.32) + '@hono/node-server': 1.19.14(hono@4.12.31) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -10256,7 +10222,7 @@ snapshots: eventsource-parser: 3.1.0 express: 5.2.1 express-rate-limit: 8.6.0(express@5.2.1) - hono: 4.12.32 + hono: 4.12.31 jose: 6.2.4 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -10567,7 +10533,7 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true - '@posthog/core@1.46.9': + '@posthog/core@1.39.6': dependencies: '@posthog/types': 1.402.2 @@ -10752,6 +10718,7 @@ snapshots: dependencies: '@smithy/types': 4.16.1 tslib: 2.8.1 + optional: true '@smithy/core@3.32.0': dependencies: @@ -10788,8 +10755,8 @@ snapshots: '@smithy/node-http-handler@4.7.3': dependencies: - '@smithy/core': 3.30.0 - '@smithy/types': 4.16.1 + '@smithy/core': 3.32.0 + '@smithy/types': 4.17.0 tslib: 2.8.1 '@smithy/signature-v4@5.7.0': @@ -10801,6 +10768,7 @@ snapshots: '@smithy/types@4.16.1': dependencies: tslib: 2.8.1 + optional: true '@smithy/types@4.17.0': dependencies: @@ -11219,6 +11187,14 @@ snapshots: clean-stack: 4.2.0 indent-string: 5.0.0 + ai@5.0.219(zod@4.4.3): + dependencies: + '@ai-sdk/gateway': 2.0.118(zod@4.4.3) + '@ai-sdk/provider': 2.0.3 + '@ai-sdk/provider-utils': 3.0.30(zod@4.4.3) + '@opentelemetry/api': 1.9.0 + zod: 4.4.3 + ai@5.0.220(zod@4.4.3): dependencies: '@ai-sdk/gateway': 2.0.119(zod@4.4.3) @@ -11512,7 +11488,7 @@ snapshots: dependencies: fill-range: 7.1.1 - braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(supports-color@8.1.1)(zod@4.4.3): + braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3): dependencies: '@ai-sdk/provider': 1.1.3 '@next/env': 14.2.35 @@ -11530,7 +11506,7 @@ snapshots: minimatch: 9.0.9 mustache: 4.2.0 pluralize: 8.0.0 - simple-git: 3.36.0(supports-color@8.1.1) + simple-git: 3.36.0 slugify: 1.6.9 source-map: 0.7.6 uuid: 9.0.1 @@ -11980,7 +11956,7 @@ snapshots: detect-libc@2.1.2: {} - detect-port@1.5.1(supports-color@8.1.1): + detect-port@1.5.1: dependencies: address: 1.2.2 debug: 4.4.3(supports-color@8.1.1) @@ -13139,7 +13115,7 @@ snapshots: highlight.js@10.7.3: {} - hono@4.12.32: {} + hono@4.12.31: {} hookable@6.1.1: {} @@ -13944,7 +13920,7 @@ snapshots: media-typer@0.3.0: {} - media-typer@1.1.1: {} + media-typer@1.1.0: {} merge-descriptors@1.0.3: {} @@ -14285,9 +14261,9 @@ snapshots: dependencies: minipass: 7.1.3 - mint@4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3): + mint@4.2.788(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3): dependencies: - '@mintlify/cli': 4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) + '@mintlify/cli': 4.0.1391(@base-ui/react@1.7.0(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@25.9.4)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) transitivePeerDependencies: - '@base-ui/react' - '@types/node' @@ -14901,9 +14877,9 @@ snapshots: dependencies: '@posthog/core': 1.7.1 - posthog-node@5.48.1(rxjs@7.8.2): + posthog-node@5.40.0(rxjs@7.8.2): dependencies: - '@posthog/core': 1.46.9 + '@posthog/core': 1.39.6 optionalDependencies: rxjs: 7.8.2 @@ -15723,9 +15699,9 @@ snapshots: simple-concat: 1.0.1 optional: true - simple-git@3.36.0(supports-color@8.1.1): + simple-git@3.36.0: dependencies: - '@kwsites/file-exists': 1.1.1(supports-color@8.1.1) + '@kwsites/file-exists': 1.1.1 '@kwsites/promise-deferred': 1.1.1 '@simple-git/args-pathspec': 1.0.3 '@simple-git/argv-parser': 1.1.1 @@ -16198,7 +16174,7 @@ snapshots: type-is@2.1.0: dependencies: content-type: 2.0.0 - media-typer: 1.1.1 + media-typer: 1.1.0 mime-types: 3.0.2 typebox@1.3.7: {} From 006d7ed8b2db48d122e32199c4069bedad64579b Mon Sep 17 00:00:00 2001 From: miguel Date: Sat, 22 Aug 2026 14:16:16 -0700 Subject: [PATCH 3/7] fix(evals): address fx harness review findings on observations, permissions, and status - fx-sdk session: emit tool-step observations only from the live events.jsonl tail (no post-exit replay against the final browser state) and return the observed call keys so the trajectory adapter pairs evidence by key - fx-sdk session: rewrite resolveFxStatus precedence so empty stdout, ask.exit_code != 0, and failed/cancelled turn kinds are sdk_error even on OS exit 0; sanitize event summaries/transcripts; signal the fx process group on abort - fxToolAdapter: deny every non-MCP fx built-in (web_search/web_fetch/file tools included), pre-allow every discovered mcp__ so --auto never adjudicates, pass the runner's HOME plus pnpm/XDG/proxy cache vars to MCP children, set startup_timeout_ms (EVAL_FX_MCP_STARTUP_TIMEOUT_MS), and log instead of swallowing cleanup timeouts while always removing the temp root --- packages/evals/framework/fxRunner.ts | 1 + packages/evals/framework/fxToolAdapter.ts | 217 +++++++++++++++--- .../evals/framework/harnesses/fxAdapter.ts | 26 ++- .../evals/tests/framework/fxAdapter.test.ts | 29 +++ .../tests/framework/fxToolAdapter.test.ts | 59 +++-- packages/integrations/fx-sdk/src/session.ts | 78 +++++-- .../integrations/fx-sdk/tests/session.test.ts | 135 ++++++++++- 7 files changed, 479 insertions(+), 66 deletions(-) diff --git a/packages/evals/framework/fxRunner.ts b/packages/evals/framework/fxRunner.ts index 637c67f27..e7eae7d96 100644 --- a/packages/evals/framework/fxRunner.ts +++ b/packages/evals/framework/fxRunner.ts @@ -128,6 +128,7 @@ export async function runFxAgent({ ...(finalObservation && { finalObservation }), ...(stepObservations?.length && { stepObservations }), ...(observedToolName && { observedToolName }), + observedToolCallKeys: raw.observedToolCallKeys, finalAnswer: parsed.finalAnswer ?? raw.finalMessage, status, usage: { diff --git a/packages/evals/framework/fxToolAdapter.ts b/packages/evals/framework/fxToolAdapter.ts index 955eaaef9..5b31199c7 100644 --- a/packages/evals/framework/fxToolAdapter.ts +++ b/packages/evals/framework/fxToolAdapter.ts @@ -1,7 +1,8 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import type { ProbeEvidence } from "stagehand-v3"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; +import { connectToMCPServer, type ProbeEvidence } from "stagehand-v3"; import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; @@ -16,6 +17,10 @@ export interface FxToolAdapterInput { environment: "LOCAL" | "BROWSERBASE"; plan: ExternalHarnessTaskPlan; logger: EvalLogger; + listMcpToolNames?: ( + serverName: string, + spec: { command: string; args: string[]; env: Record }, + ) => Promise; } export interface PreparedFxToolAdapter { @@ -45,9 +50,73 @@ export const FX_TOOL_SURFACES: ToolSurface[] = [ "chrome_devtools_mcp", ]; +export const FX_DENIED_TOOLS = [ + "run_command", + "terminal", + "write_file", + "edit_file", + "read_file", + "list_files", + "glob", + "grep", + "copy_file", + "delete_file", + "rename_file", + "background_command", + "web_search", + "web_fetch", + "skill", + "subagent", + "memory", +] as const; + +const MCP_CHILD_ENV_KEYS = [ + "PNPM_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "XDG_CONFIG_HOME", + "XDG_STATE_HOME", + "npm_config_cache", + "npm_config_store_dir", + "npm_config_prefix", + "NPM_CONFIG_CACHE", + "NPM_CONFIG_STORE_DIR", + "COREPACK_HOME", + "TMPDIR", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +type FxMcpOptions = { + home: string; + pathEnv: string; + parentEnv: Record; +}; + +export function buildFxMcpChildEnv( + specEnv: Record, + options: FxMcpOptions, +): Record { + const env: Record = { + PATH: options.pathEnv, + HOME: options.parentEnv.HOME ?? options.home, + }; + for (const key of MCP_CHILD_ENV_KEYS) { + const value = options.parentEnv[key]; + if (typeof value === "string") env[key] = value; + } + return { ...env, ...specEnv }; +} + export function buildFxMcpConfig( mcpServers: Record, - options: { home: string; pathEnv: string }, + options: FxMcpOptions & { startupTimeoutMs: number }, ): { mcp: Record } { const mcp: Record = {}; for (const [serverName, rawSpec] of Object.entries(mcpServers)) { @@ -65,29 +134,30 @@ export function buildFxMcpConfig( mcp[serverName] = { type: "stdio", command: [spec.command, ...args], - environment: { PATH: options.pathEnv, HOME: options.home, ...extraEnv }, + environment: buildFxMcpChildEnv(extraEnv, options), required: true, + startup_timeout_ms: options.startupTimeoutMs, }; } return { mcp }; } export function buildFxSettings( - _serverNames: string[], - toolSurface: ToolSurface, + mcpToolNames: Record, ): { permission: Record } { + const permission: Record = Object.fromEntries( + FX_DENIED_TOOLS.map((name) => [name, "deny" as const]), + ); + for (const [serverName, toolNames] of Object.entries(mcpToolNames)) { + for (const toolName of toolNames) { + permission[`mcp_${serverName}_${toolName}`] = "allow"; + if (serverName.includes("-")) { + permission[`mcp_${serverName.replace(/-/gu, "_")}_${toolName}`] = "allow"; + } + } + } return { - permission: { - run_command: "deny", - terminal: "deny", - write_file: "deny", - edit_file: "deny", - ...(toolSurface === "stagehand_facade" && { - mcp_stagehand_run: "allow", - mcp_stagehand_snapshot: "allow", - mcp_stagehand_screenshot: "allow", - }), - }, + permission, }; } @@ -101,7 +171,7 @@ export function buildFxAgentsMarkdown(promptInstructions: string, serverNames: s "", `MCP tools use the fx name mcp__ (configured servers: ${serverNames.join(", ")}; patterns: ${prefixes}).`, "Select tools with mcp_select_tool using their exact name. mcp_search_tools may return nothing.", - "Never invent tool names. Do not use the shell and do not edit files.", + "Never invent tool names. Do not use the shell, web search/fetch, or file tools.", stagehandGuidance, "", promptInstructions, @@ -157,13 +227,53 @@ export async function prepareFxToolAdapter( const serverNames = Object.keys(mount.mcpServers); const pathEnv = process.env.PATH ?? ""; + const mcpOptions: FxMcpOptions = { home, pathEnv, parentEnv: process.env }; + const mcpToolNames: Record = {}; + if (toolSurface === "stagehand_facade") { + mcpToolNames.stagehand = ["run", "snapshot", "screenshot"]; + } else { + const listMcpToolNames = input.listMcpToolNames ?? defaultListMcpToolNames; + for (const [serverName, rawSpec] of Object.entries(mount.mcpServers)) { + const spec = normalizeFxMcpServerSpec(serverName, rawSpec); + const childSpec = { + command: spec.command, + args: spec.args, + env: buildFxMcpChildEnv(spec.env, mcpOptions), + }; + try { + const toolNames = await withTimeout( + listMcpToolNames(serverName, childSpec), + readPositiveIntEnv("EVAL_FX_MCP_PROBE_TIMEOUT_MS", 60_000), + "fx MCP tool discovery", + ); + mcpToolNames[serverName] = toolNames; + input.logger.log({ + category: "fx", + message: `Discovered ${toolNames.length} MCP tools for fx server ${serverName}.`, + level: 1, + }); + } catch (error) { + const message = sanitizeErrorMessage(stringifyUnknown(error)); + mcpToolNames[serverName] = []; + input.logger.warn({ + category: "fx", + message: `fx MCP tool discovery failed for ${serverName}: ${message}`, + level: 0, + auxiliary: { error: { value: message, type: "string" } }, + }); + } + } + } const agentsMarkdown = buildFxAgentsMarkdown(mount.promptInstructions, serverNames); await Promise.all([ writeJson( path.join(fxHome, "mcp.json"), - buildFxMcpConfig(mount.mcpServers, { home, pathEnv }), + buildFxMcpConfig(mount.mcpServers, { + ...mcpOptions, + startupTimeoutMs: readPositiveIntEnv("EVAL_FX_MCP_STARTUP_TIMEOUT_MS", 120_000), + }), ), - writeJson(path.join(fxHome, "settings.json"), buildFxSettings(serverNames, toolSurface)), + writeJson(path.join(fxHome, "settings.json"), buildFxSettings(mcpToolNames)), writeJson(path.join(workspace, ".fx.json"), { max_agent_steps: readFxMaxAgentSteps(), max_tool_result_bytes: 262_144, @@ -212,27 +322,76 @@ export async function prepareFxToolAdapter( ), cleanup: async () => { cleanupPromise ??= (async () => { - await withTimeout( - runtime.cleanup(), - readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), - "fx adapter cleanup", - ).catch((): undefined => undefined); - await fsp.rm(capturedRoot, { recursive: true, force: true }); + try { + await cleanupFxRuntime(() => runtime.cleanup(), input.logger); + } finally { + await fsp.rm(capturedRoot, { recursive: true, force: true }); + } })(); await cleanupPromise; }, }; } catch (error) { + try { + await cleanupFxRuntime(() => runtime.cleanup(), input.logger); + } finally { + if (root) await fsp.rm(root, { recursive: true, force: true }); + } + throw error; + } +} + +async function defaultListMcpToolNames( + _serverName: string, + spec: { command: string; args: string[]; env: Record }, +): Promise { + const client = await connectToMCPServer(spec); + try { + const listed = await client.listTools(); + return listed.tools.map((tool) => tool.name); + } finally { + await client.close(); + } +} + +function normalizeFxMcpServerSpec( + serverName: string, + rawSpec: unknown, +): { command: string; args: string[]; env: Record } { + if (!isRecord(rawSpec) || typeof rawSpec.command !== "string") { + throw new EvalsError(`Invalid fx MCP launch spec for server "${serverName}".`); + } + return { + command: rawSpec.command, + args: Array.isArray(rawSpec.args) + ? rawSpec.args.filter((arg): arg is string => typeof arg === "string") + : [], + env: isStringRecord(rawSpec.env) ? rawSpec.env : {}, + }; +} + +async function cleanupFxRuntime(cleanup: () => Promise, logger: EvalLogger): Promise { + try { await withTimeout( - runtime.cleanup(), + cleanup(), readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), "fx adapter cleanup", - ).catch((): undefined => undefined); - if (root) await fsp.rm(root, { recursive: true, force: true }); - throw error; + ); + } catch (error) { + const message = stringifyUnknown(error); + logger.warn({ + category: "fx", + message: `fx adapter cleanup failed: ${message}`, + level: 0, + auxiliary: { error: { value: message, type: "string" } }, + }); } } +function stringifyUnknown(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + export function readFxMaxAgentSteps(): number { for (const key of ["EVAL_FX_MAX_STEPS", "AGENT_EVAL_MAX_STEPS"]) { const parsed = Number.parseInt(process.env[key] ?? "", 10); diff --git a/packages/evals/framework/harnesses/fxAdapter.ts b/packages/evals/framework/harnesses/fxAdapter.ts index beef788a0..f3d68e55f 100644 --- a/packages/evals/framework/harnesses/fxAdapter.ts +++ b/packages/evals/framework/harnesses/fxAdapter.ts @@ -19,11 +19,13 @@ export interface FxRunResult { finalObservation?: ProbeEvidence; stepObservations?: StepObservation[]; observedToolName?: (name: string) => boolean; + observedToolCallKeys?: string[]; } export class FxTrajectoryAdapter implements TrajectoryAdapter { fromHarnessResult(result: FxRunResult, taskSpec: TaskSpec): Trajectory { const toolCalls: NormalizedToolCall[] = []; + const toolCallKeys: string[] = []; let latestAgentMessage: string | undefined; for (const event of result.events) { @@ -42,10 +44,11 @@ export class FxTrajectoryAdapter implements TrajectoryAdapter { event.tool_calls.forEach((call, index) => { const toolResult = typeof call.id === "string" ? resultsById.get(call.id) : undefined; toolCalls.push(normalizeFxToolCall(call, toolResult, index === 0 ? event.assistant : "")); + toolCallKeys.push(fxToolCallKey(call)); }); } - pairStepObservations(toolCalls, result); + pairStepObservations(toolCalls, toolCallKeys, result); return buildTrajectory({ taskSpec, @@ -109,9 +112,28 @@ function replaceImageBlocks( ); } -function pairStepObservations(toolCalls: NormalizedToolCall[], result: FxRunResult): void { +function fxToolCallKey(call: FxToolCallRecord): string { + const name = typeof call.name === "string" ? call.name : ""; + return typeof call.id === "string" ? call.id : `${name}:${call.arguments_json ?? ""}`; +} + +function pairStepObservations( + toolCalls: NormalizedToolCall[], + toolCallKeys: string[], + result: FxRunResult, +): void { const observations = result.stepObservations ?? []; if (observations.length === 0) return; + if (result.observedToolCallKeys !== undefined) { + for (const observation of observations) { + const key = result.observedToolCallKeys[observation.runIndex]; + if (key === undefined) continue; + const callIndex = toolCallKeys.indexOf(key); + const call = callIndex >= 0 ? toolCalls[callIndex] : undefined; + if (call) call.probeEvidence = observation.evidence; + } + return; + } const observedCalls = toolCalls.filter((call) => result.observedToolName ? result.observedToolName(call.name) : call.name.startsWith("mcp_"), ); diff --git a/packages/evals/tests/framework/fxAdapter.test.ts b/packages/evals/tests/framework/fxAdapter.test.ts index 619e7de02..5072717c7 100644 --- a/packages/evals/tests/framework/fxAdapter.test.ts +++ b/packages/evals/tests/framework/fxAdapter.test.ts @@ -126,6 +126,35 @@ describe("fx trajectory adapter", () => { expect(trajectory.steps[2]?.probeEvidence).toEqual({ url: "https://example.com/two" }); }); + it("pairs observations by live call key when counts do not align", () => { + const events: FxEvent[] = [ + { + type: "tool_step", + assistant: "", + tool_calls: [ + { id: "one", name: "mcp_stagehand_run", arguments_json: "{}" }, + { id: "two", name: "mcp_stagehand_snapshot", arguments_json: "{}" }, + { id: "three", name: "mcp_stagehand_screenshot", arguments_json: "{}" }, + ], + tool_results: [], + }, + ]; + const trajectory = fxAdapter.fromHarnessResult( + { + events, + observedToolCallKeys: ["two"], + stepObservations: [ + { runIndex: 0, evidence: { url: "https://example.com/live" } }, + { runIndex: 1, evidence: { url: "https://example.com/unmatched" } }, + ], + }, + taskSpec, + ); + expect(trajectory.steps[0]?.probeEvidence).toEqual({}); + expect(trajectory.steps[1]?.probeEvidence).toEqual({ url: "https://example.com/live" }); + expect(trajectory.steps[2]?.probeEvidence).toEqual({}); + }); + it("includes a final observation only when it has a screenshot", () => { const withoutScreenshot = fxAdapter.fromHarnessResult( { events: [], finalObservation: { url: "https://example.com" } }, diff --git a/packages/evals/tests/framework/fxToolAdapter.test.ts b/packages/evals/tests/framework/fxToolAdapter.test.ts index 733168aaf..1372db6b1 100644 --- a/packages/evals/tests/framework/fxToolAdapter.test.ts +++ b/packages/evals/tests/framework/fxToolAdapter.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it } from "vitest"; import { fxHarness } from "../../framework/benchHarness.js"; import { buildFxAgentsMarkdown, + buildFxMcpChildEnv, buildFxMcpConfig, buildFxSettings, + FX_DENIED_TOOLS, FX_TOOL_SURFACES, } from "../../framework/fxToolAdapter.js"; import { @@ -35,7 +37,16 @@ describe("fx tool adapter helpers", () => { env: { BROWSERBASE_API_KEY: "test" }, }, }, - { home: "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/isolated/home", pathEnv: "/usr/bin:/bin" }, + { + home: "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/isolated/home", + pathEnv: "/usr/bin:/bin", + parentEnv: { + HOME: "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/runner/home", + PNPM_HOME: "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/runner/pnpm", + OPENAI_API_KEY: "must-not-leak", + }, + startupTimeoutMs: 123_000, + }, ), ).toEqual({ mcp: { @@ -44,36 +55,54 @@ describe("fx tool adapter helpers", () => { command: ["/usr/bin/node", "server.mjs", "--flag"], environment: { PATH: "/usr/bin:/bin", - HOME: "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/isolated/home", + HOME: "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/runner/home", + PNPM_HOME: "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/runner/pnpm", BROWSERBASE_API_KEY: "test", }, required: true, + startup_timeout_ms: 123_000, }, }, }); expect(() => - buildFxMcpConfig({ "bad server": { command: "node" } }, { home: "/home", pathEnv: "/bin" }), + buildFxMcpConfig( + { "bad server": { command: "node" } }, + { + home: "/home", + pathEnv: "/bin", + parentEnv: {}, + startupTimeoutMs: 120_000, + }, + ), ).toThrow(/Invalid fx MCP server name/u); + expect( + buildFxMcpChildEnv( + { HOME: "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/spec/home", PNPM_HOME: "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/spec/pnpm" }, + { + home: "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/fallback/home", + pathEnv: "/bin", + parentEnv: { HOME: "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/runner/home", PNPM_HOME: "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/runner/pnpm" }, + }, + ), + ).toMatchObject({ HOME: "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/spec/home", PNPM_HOME: "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/spec/pnpm" }); }); - it("denies shell and editing tools and narrowly allows the facade", () => { - expect(buildFxSettings(["stagehand"], "stagehand_facade")).toEqual({ + it("denies unsafe built-ins and allows exact discovered MCP tools", () => { + expect(buildFxSettings({ stagehand: ["run", "snapshot", "screenshot"] })).toEqual({ permission: { - run_command: "deny", - terminal: "deny", - write_file: "deny", - edit_file: "deny", + ...Object.fromEntries(FX_DENIED_TOOLS.map((name) => [name, "deny"])), mcp_stagehand_run: "allow", mcp_stagehand_snapshot: "allow", mcp_stagehand_screenshot: "allow", }, }); - expect(buildFxSettings(["playwright"], "playwright_mcp").permission).toEqual({ - run_command: "deny", - terminal: "deny", - write_file: "deny", - edit_file: "deny", + expect(buildFxSettings({ "chrome-devtools": ["click"] }).permission).toMatchObject({ + "mcp_chrome-devtools_click": "allow", + mcp_chrome_devtools_click: "allow", }); + expect(buildFxSettings({}).permission).toEqual( + Object.fromEntries(FX_DENIED_TOOLS.map((name) => [name, "deny"])), + ); }); it("teaches exact fx MCP selection and Stagehand names", () => { @@ -83,6 +112,8 @@ describe("fx tool adapter helpers", () => { expect(markdown).toContain("mcp_stagehand_run"); expect(markdown).toContain("mcp_stagehand_snapshot"); expect(markdown).toContain("mcp_stagehand_screenshot"); + expect(markdown).toContain("web search/fetch"); + expect(markdown).toContain("file tools"); expect(markdown).toContain("Use snapshots first."); }); }); diff --git a/packages/integrations/fx-sdk/src/session.ts b/packages/integrations/fx-sdk/src/session.ts index 84b37c7ae..5a5f347ae 100644 --- a/packages/integrations/fx-sdk/src/session.ts +++ b/packages/integrations/fx-sdk/src/session.ts @@ -77,6 +77,7 @@ export type FxSessionResult = { sessionId?: string; exitCode?: number; iterationError?: unknown; + observedToolCallKeys: string[]; }; export type FxProcessRunner = (input: { @@ -129,8 +130,8 @@ const defaultProcessRunner: FxProcessRunner = async (input) => }; const abort = (): void => { if (!child || settled) return; - child.kill("SIGTERM"); - killTimer = setTimeout(() => child.kill("SIGKILL"), 2_000); + signalFxProcess(child, "SIGTERM"); + killTimer = setTimeout(() => signalFxProcess(child, "SIGKILL"), 2_000); killTimer.unref(); }; @@ -139,6 +140,7 @@ const defaultProcessRunner: FxProcessRunner = async (input) => cwd: input.cwd, env: input.env, stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", }); child.stdout.on("data", (chunk: Buffer | string) => { stdout += String(chunk); @@ -164,6 +166,22 @@ const defaultProcessRunner: FxProcessRunner = async (input) => } }); +function signalFxProcess(child: ReturnType, signal: NodeJS.Signals): void { + if (process.platform !== "win32" && typeof child.pid === "number") { + try { + process.kill(-child.pid, signal); + return; + } catch { + // Fall back to the direct child if the process group is already gone. + } + } + try { + child.kill(signal); + } catch { + // The child may have exited between the abort check and the signal. + } +} + const defaultSessionStore: FxSessionStore = { async waitForSessionDir(home, signal) { if (signal.aborted) return undefined; @@ -250,6 +268,7 @@ export async function runFxSession(input: { const store = input.store ?? defaultSessionStore; const seenToolCalls = new Set(); + const observedToolCallKeys: string[] = []; const observedTool = input.observedTool ?? ((name: string) => name.startsWith("mcp_")); let sessionDir: string | undefined; let processSettled = false; @@ -262,8 +281,9 @@ export async function runFxSession(input: { const id = typeof call.id === "string" ? call.id : undefined; const name = typeof call.name === "string" ? call.name : ""; const key = id ?? `${name}:${call.arguments_json ?? ""}`; - if (!observedTool(name) || seenToolCalls.has(key)) continue; + if (processSettled || !observedTool(name) || seenToolCalls.has(key)) continue; seenToolCalls.add(key); + observedToolCallKeys.push(key); try { await input.onToolStep(call); } catch { @@ -286,8 +306,8 @@ export async function runFxSession(input: { }); if (input.onToolStep) { - // fx does not stream tool events. Tail its recovery checkpoints while - // the process is alive, then reconcile against the committed turn. + // fx does not stream tool events. Tail its recovery checkpoints only + // while the process is alive. while (!processSettled && !controller.signal.aborted) { sessionDir ??= await store .waitForSessionDir(input.home, controller.signal) @@ -324,7 +344,6 @@ export async function runFxSession(input: { ? parseFxEventsJsonl(await store.readEventsJsonl(sessionDir).catch(() => "")) : []; const toolSteps = extractFxToolSteps(logEvents); - await notifyCalls(toolSteps); for (const step of toolSteps) { const event: FxEvent = { type: "tool_step", ...step }; events.push(event); @@ -372,6 +391,7 @@ export async function runFxSession(input: { signal: processResult.signal, ask, terminalReason, + turnKind: typeof turn?.kind === "string" ? turn.kind : undefined, aborted, stderr: processResult.stderr, }); @@ -400,6 +420,7 @@ export async function runFxSession(input: { ...(typeof ask?.session_id === "string" && { sessionId: ask.session_id }), ...(processResult.exitCode !== null && { exitCode: processResult.exitCode }), ...(iterationError !== undefined && { iterationError }), + observedToolCallKeys, }; } @@ -475,6 +496,7 @@ export function resolveFxStatus(input: { signal?: string | null; ask?: FxAskOutput; terminalReason?: string; + turnKind?: string; aborted?: boolean; stderr?: string; }): { status: "completed" | "max_turns" | "sdk_error"; stopReason?: string } { @@ -490,7 +512,6 @@ export function resolveFxStatus(input: { ) { return { status: "max_turns", stopReason: error ?? input.terminalReason }; } - if (input.exitCode === 0 && !error) return { status: "completed" }; if (!input.ask) { const stderr = input.stderr?.trim(); return { @@ -498,9 +519,25 @@ export function resolveFxStatus(input: { stopReason: `fx produced no JSON output${stderr ? `: ${clip(stderr, 500)}` : ""}`, }; } + if (error) return { status: "sdk_error", stopReason: error }; + if (typeof input.ask.exit_code === "number" && input.ask.exit_code !== 0) { + return { + status: "sdk_error", + stopReason: `fx reported exit_code ${input.ask.exit_code}`, + }; + } + const failurePattern = + /^(cancel|interrupt|error|fail|abort|timeout|deadline|terminat|unreadable)/iu; + const failedTurn = [input.terminalReason, input.turnKind].find( + (value): value is string => typeof value === "string" && failurePattern.test(value), + ); + if (failedTurn) { + return { status: "sdk_error", stopReason: `fx turn ended: ${failedTurn}` }; + } + if (input.exitCode === 0) return { status: "completed" }; return { status: "sdk_error", - stopReason: error ?? `fx exited with code ${input.exitCode ?? "unknown"}`, + stopReason: `fx exited with code ${input.exitCode ?? "unknown"}`, }; } @@ -525,23 +562,26 @@ export function logFxEvent(logger: HarnessLogger, event: FxEvent): void { } export function summarizeFxEvent(event: FxEvent): { message: string; detail?: string } { + let summary: { message: string; detail?: string }; if (event.type === "assistant") { - return { message: `assistant: ${clip(event.text, 500)}`, detail: event.text }; - } - if (event.type === "tool_step") { + summary = { message: `assistant: ${clip(event.text, 500)}`, detail: event.text }; + } else if (event.type === "tool_step") { const names = event.tool_calls.map((call) => String(call.name ?? "tool")).join(", "); - return { message: `tools: ${names}`, detail: safeJson(event) }; - } - if (event.type === "stderr") { - return { message: `stderr: ${clip(event.line, 500)}`, detail: event.line }; - } - if (event.type === "turn_committed") { - return { + summary = { message: `tools: ${names}`, detail: safeJson(event) }; + } else if (event.type === "stderr") { + summary = { message: `stderr: ${clip(event.line, 500)}`, detail: event.line }; + } else if (event.type === "turn_committed") { + summary = { message: `turn committed: ${event.terminal_reason ?? event.turn_kind ?? "unknown"}`, detail: safeJson(event), }; + } else { + summary = { message: "ask result", detail: safeJson(event.ask) }; } - return { message: "ask result", detail: safeJson(event.ask) }; + return { + message: sanitizeErrorMessage(summary.message), + ...(summary.detail && { detail: sanitizeErrorMessage(summary.detail) }), + }; } function readToolSteps(value: unknown): FxToolStep[] { diff --git a/packages/integrations/fx-sdk/tests/session.test.ts b/packages/integrations/fx-sdk/tests/session.test.ts index b7d3fd7fa..a85caf341 100644 --- a/packages/integrations/fx-sdk/tests/session.test.ts +++ b/packages/integrations/fx-sdk/tests/session.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + buildFxTranscript, normalizeFxModel, runFxSession, type FxProcessRunner, @@ -119,6 +120,7 @@ describe("fx CLI session", () => { }); expect(result.status).toBe("completed"); expect(result.finalMessage).toContain('"success":true'); + expect(result.observedToolCallKeys).toEqual([]); }); it("reports missing credentials as an SDK error", async () => { @@ -170,8 +172,9 @@ describe("fx CLI session", () => { expect(result.stopReason).toContain("fx produced no JSON output"); }); - it("deduplicates observed MCP tool calls and ignores built-in tools", async () => { + it("records only MCP tool calls observed from a live recovery checkpoint", async () => { const onToolStep = vi.fn(); + let processExited = false; const recovery = { kind: "recovery_checkpoint_set", payload: { @@ -191,6 +194,13 @@ describe("fx CLI session", () => { }, }, }; + const committed = committedEvent(); + committed.payload.turn.execution.tool_steps[0]?.tool_calls.push({ + id: "call-2", + name: "mcp_stagehand_run", + arguments_json: "{}", + provider_result: null, + }); const result = await runFxSession({ prompt: "task", cwd: "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/fake/workspace", @@ -201,13 +211,54 @@ describe("fx CLI session", () => { onToolStep, runProcess: async () => { await new Promise((resolve) => setTimeout(resolve, 5)); + processExited = true; return { stdout: JSON.stringify({ output: "done" }), stderr: "", exitCode: 0 }; }, - store: fakeStore(jsonl(recovery, committedEvent())), + store: { + waitForSessionDir: async () => "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/fake/session", + readEventsJsonl: async () => jsonl(recovery, ...(processExited ? [committed] : [])), + }, }); expect(result.status).toBe("completed"); expect(onToolStep).toHaveBeenCalledTimes(1); expect(onToolStep.mock.calls[0]?.[0]).toMatchObject({ id: "call-1" }); + expect(result.observedToolCallKeys).toEqual(["call-1"]); + }); + + it("does not synthesize observations from a committed turn after exit", async () => { + const onToolStep = vi.fn(); + let processExited = false; + const committed = committedEvent(); + const toolCalls = committed.payload.turn.execution.tool_steps[0]?.tool_calls; + toolCalls?.push( + { id: "call-2", name: "mcp_stagehand_run", arguments_json: "{}", provider_result: null }, + { + id: "call-3", + name: "mcp_stagehand_screenshot", + arguments_json: "{}", + provider_result: null, + }, + ); + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + pollIntervalMs: 1, + onToolStep, + runProcess: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + processExited = true; + return { stdout: JSON.stringify({ output: "done" }), stderr: "", exitCode: 0 }; + }, + store: { + waitForSessionDir: async () => "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/fake/session", + readEventsJsonl: async () => (processExited ? jsonl(committed) : ""), + }, + }); + expect(onToolStep).not.toHaveBeenCalled(); + expect(result.observedToolCallKeys).toEqual([]); }); it("forwards aborts to the process and reports aborted", async () => { @@ -258,4 +309,84 @@ describe("fx CLI session", () => { expect(result.stopReason).not.toContain("1234567890"); expect(result.stopReason).toContain("[redacted]"); }); + + it("treats successful exits without JSON output as SDK errors", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ stdout: "", stderr: "", exitCode: 0 }), + store: fakeStore(""), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("no JSON output"); + }); + + it("honors fx ask exit codes even when the process exits zero", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ output: "x", exit_code: 2 }), + stderr: "", + exitCode: 0, + }), + store: fakeStore(""), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("exit_code 2"); + }); + + it("honors failed committed turn reasons even when the process exits zero", async () => { + const committed = committedEvent("cancelled"); + committed.payload.turn.kind = "interrupted"; + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ output: "x", exit_code: 0 }), + stderr: "", + exitCode: 0, + }), + store: fakeStore(jsonl(committed)), + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("cancelled"); + }); + + it("redacts sensitive event details in logs and transcripts", async () => { + const log = vi.fn(); + const secretEvent = committedEvent(); + const resultRecord = secretEvent.payload.turn.execution.tool_steps[0]?.tool_results[0]; + if (resultRecord) resultRecord.output = "token bb_live_abcd1234567890"; + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + logger: { log, warn: () => {}, error: () => {} }, + runProcess: async () => ({ + stdout: JSON.stringify({ output: "done", exit_code: 0 }), + stderr: "", + exitCode: 0, + }), + store: fakeStore(jsonl(secretEvent)), + }); + const details = log.mock.calls + .map((call) => call[0]?.auxiliary?.detail?.value) + .filter((value): value is string => typeof value === "string") + .join("\n"); + expect(details).toContain("[redacted]"); + expect(details).not.toContain("1234567890"); + expect(buildFxTranscript(result.events)).toContain("[redacted]"); + expect(buildFxTranscript(result.events)).not.toContain("1234567890"); + }); }); From 18a0bc7dcc4f998a27cb868223fa04b854533241 Mon Sep 17 00:00:00 2001 From: miguel Date: Sat, 22 Aug 2026 14:38:16 -0700 Subject: [PATCH 4/7] fix(evals): fx step-limit detection, real host-tool deny list, process-group cleanup, sanitized cleanup logs --- packages/evals/framework/fxToolAdapter.ts | 33 ++- .../tests/framework/fxToolAdapter.test.ts | 31 ++- packages/integrations/fx-sdk/src/session.ts | 234 +++++++++++++----- .../integrations/fx-sdk/tests/session.test.ts | 88 ++++++- 4 files changed, 309 insertions(+), 77 deletions(-) diff --git a/packages/evals/framework/fxToolAdapter.ts b/packages/evals/framework/fxToolAdapter.ts index 5b31199c7..8949d889e 100644 --- a/packages/evals/framework/fxToolAdapter.ts +++ b/packages/evals/framework/fxToolAdapter.ts @@ -51,14 +51,23 @@ export const FX_TOOL_SURFACES: ToolSurface[] = [ ]; export const FX_DENIED_TOOLS = [ + "*", "run_command", "terminal", "write_file", "edit_file", "read_file", "list_files", - "glob", - "grep", + "glob_files", + "grep_files", + "open_file", + "file_info", + "semantic_search", + "create_folder", + "install_skill", + "vision", + "ask_user_question", + "read_tool_result", "copy_file", "delete_file", "rename_file", @@ -142,13 +151,17 @@ export function buildFxMcpConfig( return { mcp }; } -export function buildFxSettings( - mcpToolNames: Record, -): { permission: Record } { +export function buildFxSettings(mcpToolNames: Record): { + permission: Record; +} { const permission: Record = Object.fromEntries( FX_DENIED_TOOLS.map((name) => [name, "deny" as const]), ); for (const [serverName, toolNames] of Object.entries(mcpToolNames)) { + permission[`mcp_${serverName}_*`] = "allow"; + if (serverName.includes("-")) { + permission[`mcp_${serverName.replace(/-/gu, "_")}_*`] = "allow"; + } for (const toolName of toolNames) { permission[`mcp_${serverName}_${toolName}`] = "allow"; if (serverName.includes("-")) { @@ -370,7 +383,10 @@ function normalizeFxMcpServerSpec( }; } -async function cleanupFxRuntime(cleanup: () => Promise, logger: EvalLogger): Promise { +export async function cleanupFxRuntime( + cleanup: () => Promise, + logger: EvalLogger, +): Promise { try { await withTimeout( cleanup(), @@ -378,7 +394,10 @@ async function cleanupFxRuntime(cleanup: () => Promise, logger: EvalLogger "fx adapter cleanup", ); } catch (error) { - const message = stringifyUnknown(error); + const message = sanitizeErrorMessage(stringifyUnknown(error)).replace( + /\b((?:apiKey|api_key|token|key)=)[^&\s"']+/giu, + "$1[redacted]", + ); logger.warn({ category: "fx", message: `fx adapter cleanup failed: ${message}`, diff --git a/packages/evals/tests/framework/fxToolAdapter.test.ts b/packages/evals/tests/framework/fxToolAdapter.test.ts index 1372db6b1..bb34a4f44 100644 --- a/packages/evals/tests/framework/fxToolAdapter.test.ts +++ b/packages/evals/tests/framework/fxToolAdapter.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { fxHarness } from "../../framework/benchHarness.js"; +import { EvalLogger } from "../../logger.js"; import { buildFxAgentsMarkdown, buildFxMcpChildEnv, buildFxMcpConfig, buildFxSettings, + cleanupFxRuntime, FX_DENIED_TOOLS, FX_TOOL_SURFACES, } from "../../framework/fxToolAdapter.js"; @@ -91,18 +93,45 @@ describe("fx tool adapter helpers", () => { expect(buildFxSettings({ stagehand: ["run", "snapshot", "screenshot"] })).toEqual({ permission: { ...Object.fromEntries(FX_DENIED_TOOLS.map((name) => [name, "deny"])), + "mcp_stagehand_*": "allow", mcp_stagehand_run: "allow", mcp_stagehand_snapshot: "allow", mcp_stagehand_screenshot: "allow", }, }); expect(buildFxSettings({ "chrome-devtools": ["click"] }).permission).toMatchObject({ + "mcp_chrome-devtools_*": "allow", + "mcp_chrome_devtools_*": "allow", "mcp_chrome-devtools_click": "allow", mcp_chrome_devtools_click: "allow", }); expect(buildFxSettings({}).permission).toEqual( Object.fromEntries(FX_DENIED_TOOLS.map((name) => [name, "deny"])), ); + const permission = buildFxSettings({ stagehand: ["run"] }).permission; + expect(permission).toMatchObject({ + "*": "deny", + grep_files: "deny", + open_file: "deny", + file_info: "deny", + semantic_search: "deny", + vision: "deny", + "mcp_stagehand_*": "allow", + mcp_stagehand_run: "allow", + }); + expect(FX_DENIED_TOOLS).not.toContain("glob" as never); + expect(FX_DENIED_TOOLS).not.toContain("grep" as never); + }); + + it("redacts cleanup failures before logging", async () => { + const logger = new EvalLogger(false); + const warn = vi.spyOn(logger, "warn"); + await cleanupFxRuntime(async () => { + throw new Error("cleanup failed apiKey=secret123"); + }, logger); + const logged = JSON.stringify(warn.mock.calls); + expect(logged).toContain("apiKey=[redacted]"); + expect(logged).not.toContain("secret123"); }); it("teaches exact fx MCP selection and Stagehand names", () => { diff --git a/packages/integrations/fx-sdk/src/session.ts b/packages/integrations/fx-sdk/src/session.ts index 5a5f347ae..e76530e4a 100644 --- a/packages/integrations/fx-sdk/src/session.ts +++ b/packages/integrations/fx-sdk/src/session.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import fsp from "node:fs/promises"; import path from "node:path"; import { @@ -95,6 +96,13 @@ export type FxProcessRunner = (input: { signal?: string | null; }>; +export type FxProcessRunnerOptions = { + spawnProcess?: typeof spawn; + killProcess?: typeof process.kill; + processHooks?: Pick; + killGraceMs?: number; +}; + export type FxSessionStore = { waitForSessionDir(home: string, signal: AbortSignal): Promise; readEventsJsonl(sessionDir: string): Promise; @@ -111,77 +119,140 @@ export function normalizeFxModel(model: string): string | undefined { return model === "fx/default" ? undefined : model; } -const defaultProcessRunner: FxProcessRunner = async (input) => - new Promise((resolve) => { - let stdout = ""; - let stderr = ""; - let settled = false; - let stderrRemainder = ""; - let killTimer: NodeJS.Timeout | undefined; - let child: ReturnType; - - const finish = (exitCode: number | null, signal?: string | null): void => { - if (settled) return; - settled = true; - if (killTimer) clearTimeout(killTimer); - input.signal.removeEventListener("abort", abort); - if (stderrRemainder) input.onStderrLine?.(stderrRemainder); - resolve({ stdout, stderr, exitCode, signal }); - }; - const abort = (): void => { - if (!child || settled) return; - signalFxProcess(child, "SIGTERM"); - killTimer = setTimeout(() => signalFxProcess(child, "SIGKILL"), 2_000); - killTimer.unref(); - }; - - try { - child = spawn(input.bin, input.args, { - cwd: input.cwd, - env: input.env, - stdio: ["pipe", "pipe", "pipe"], - detached: process.platform !== "win32", - }); - child.stdout.on("data", (chunk: Buffer | string) => { - stdout += String(chunk); - }); - child.stderr.on("data", (chunk: Buffer | string) => { - const text = String(chunk); - stderr += text; - const lines = `${stderrRemainder}${text}`.split(/\r?\n/u); - stderrRemainder = lines.pop() ?? ""; - for (const line of lines) input.onStderrLine?.(line); - }); - child.once("error", (error) => { - stderr += `${stderr ? "\n" : ""}${stringifyError(error)}`; - finish(null); - }); - child.once("close", (exitCode, signal) => finish(exitCode, signal)); - input.signal.addEventListener("abort", abort, { once: true }); - if (input.signal.aborted) abort(); - child.stdin.end(input.stdin); - } catch (error) { - stderr += stringifyError(error); - finish(null); +const liveFxChildren = new Set(); + +export function createFxProcessRunner( + options: FxProcessRunnerOptions = {}, + trackedChildren: Set = liveFxChildren, +): FxProcessRunner { + const spawnProcess = options.spawnProcess ?? spawn; + const killProcess = options.killProcess ?? process.kill.bind(process); + const processHooks = options.processHooks ?? process; + const killGraceMs = options.killGraceMs ?? 2_000; + const terminationTimers = new Map(); + let hooksRegistered = false; + + const signalChild = (child: ChildProcess, signal: NodeJS.Signals): void => { + if (process.platform !== "win32" && typeof child.pid === "number") { + try { + killProcess(-child.pid, signal); + return; + } catch { + // Fall back to the direct child if the process group is already gone. + } } - }); - -function signalFxProcess(child: ReturnType, signal: NodeJS.Signals): void { - if (process.platform !== "win32" && typeof child.pid === "number") { try { - process.kill(-child.pid, signal); - return; + child.kill(signal); } catch { - // Fall back to the direct child if the process group is already gone. + // The child may have exited between the abort check and the signal. } - } - try { - child.kill(signal); - } catch { - // The child may have exited between the abort check and the signal. - } + }; + + const terminateChild = ( + child: ChildProcess, + immediateKill = false, + ): NodeJS.Timeout | undefined => { + const existingTimer = terminationTimers.get(child); + if (existingTimer) { + if (!immediateKill) return existingTimer; + clearTimeout(existingTimer); + terminationTimers.delete(child); + } + signalChild(child, "SIGTERM"); + if (immediateKill) { + signalChild(child, "SIGKILL"); + return undefined; + } + const timer = setTimeout(() => { + terminationTimers.delete(child); + signalChild(child, "SIGKILL"); + }, killGraceMs); + terminationTimers.set(child, timer); + timer.unref(); + return timer; + }; + + const untrackChild = (child: ChildProcess): void => { + trackedChildren.delete(child); + const timer = terminationTimers.get(child); + if (timer) clearTimeout(timer); + terminationTimers.delete(child); + }; + + const registerHooks = (): void => { + if (hooksRegistered) return; + hooksRegistered = true; + const terminateAll = (immediateKill: boolean): void => { + for (const child of trackedChildren) terminateChild(child, immediateKill); + }; + // The exit event only permits synchronous work, so send both signals there. + processHooks.on("exit", () => terminateAll(true)); + processHooks.on("SIGINT", () => terminateAll(false)); + processHooks.on("SIGTERM", () => terminateAll(false)); + }; + + return async (input) => + new Promise((resolve) => { + let stdout = ""; + let stderr = ""; + let settled = false; + let stderrRemainder = ""; + let killTimer: NodeJS.Timeout | undefined; + let child: ReturnType; + + const finish = (exitCode: number | null, signal?: string | null): void => { + if (settled) return; + settled = true; + if (killTimer) clearTimeout(killTimer); + input.signal.removeEventListener("abort", abort); + if (stderrRemainder) input.onStderrLine?.(stderrRemainder); + resolve({ stdout, stderr, exitCode, signal }); + }; + const abort = (): void => { + if (!child || settled) return; + killTimer = terminateChild(child); + }; + + try { + registerHooks(); + child = spawnProcess(input.bin, input.args, { + cwd: input.cwd, + env: input.env, + stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + trackedChildren.add(child); + child.stdout.on("data", (chunk: Buffer | string) => { + stdout += String(chunk); + }); + child.stderr.on("data", (chunk: Buffer | string) => { + const text = String(chunk); + stderr += text; + const lines = `${stderrRemainder}${text}`.split(/\r?\n/u); + stderrRemainder = lines.pop() ?? ""; + for (const line of lines) input.onStderrLine?.(line); + }); + child.once("error", (error) => { + untrackChild(child); + stderr += `${stderr ? "\n" : ""}${stringifyError(error)}`; + finish(null); + }); + child.once("close", (exitCode, signal) => { + untrackChild(child); + finish(exitCode, signal); + }); + input.signal.addEventListener("abort", abort, { once: true }); + if (input.signal.aborted) abort(); + child.stdin.end(input.stdin); + } catch (error) { + stderr += stringifyError(error); + finish(null); + } + }); } +const defaultProcessRunner = createFxProcessRunner(); + const defaultSessionStore: FxSessionStore = { async waitForSessionDir(home, signal) { if (signal.aborted) return undefined; @@ -386,12 +457,17 @@ export async function runFxSession(input: { : undefined; const tokenUsage = extractFxTokenUsage(logEvents, usageSnapshot); const aborted = input.signal?.aborted === true; + const configuredMaxAgentSteps = toPositiveInteger(env.FX_MAX_AGENT_STEPS); + const observedToolSteps = Math.max(toolSteps.length, toPositiveInteger(ask?.steps) ?? 0); const resolution = resolveFxStatus({ exitCode: processResult.exitCode, signal: processResult.signal, ask, terminalReason, turnKind: typeof turn?.kind === "string" ? turn.kind : undefined, + assistantText: turnAssistant, + observedToolSteps, + maxAgentSteps: configuredMaxAgentSteps, aborted, stderr: processResult.stderr, }); @@ -497,6 +573,9 @@ export function resolveFxStatus(input: { ask?: FxAskOutput; terminalReason?: string; turnKind?: string; + assistantText?: string; + observedToolSteps?: number; + maxAgentSteps?: number; aborted?: boolean; stderr?: string; }): { status: "completed" | "max_turns" | "sdk_error"; stopReason?: string } { @@ -505,12 +584,30 @@ export function resolveFxStatus(input: { return { status: "sdk_error", stopReason: "interrupted" }; } const error = typeof input.ask?.error === "string" ? input.ask.error : undefined; + const stepLimitNotice = [input.assistantText, input.ask?.output].find( + (value): value is string => + typeof value === "string" && /agent step limit reached/iu.test(value), + ); + const reachedConfiguredStepLimit = + positiveInteger(input.maxAgentSteps) && + typeof input.observedToolSteps === "number" && + input.observedToolSteps >= input.maxAgentSteps; if ( + stepLimitNotice || + reachedConfiguredStepLimit || input.terminalReason === "step_limit" || input.terminalReason === "step_limit_reached" || (error && /step.?limit/iu.test(error)) ) { - return { status: "max_turns", stopReason: error ?? input.terminalReason }; + return { + status: "max_turns", + stopReason: + stepLimitNotice ?? + error ?? + (reachedConfiguredStepLimit + ? `fx reached the configured agent step limit (${input.maxAgentSteps} steps)` + : input.terminalReason), + }; } if (!input.ask) { const stderr = input.stderr?.trim(); @@ -638,6 +735,11 @@ function positiveInteger(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value > 0; } +function toPositiveInteger(value: unknown): number | undefined { + const parsed = typeof value === "number" ? value : Number.parseInt(String(value ?? ""), 10); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : undefined; +} + function delay(ms: number, signal: AbortSignal): Promise { if (signal.aborted) return Promise.resolve(); return new Promise((resolve) => { diff --git a/packages/integrations/fx-sdk/tests/session.test.ts b/packages/integrations/fx-sdk/tests/session.test.ts index a85caf341..fcdd76eef 100644 --- a/packages/integrations/fx-sdk/tests/session.test.ts +++ b/packages/integrations/fx-sdk/tests/session.test.ts @@ -1,6 +1,10 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import type { ChildProcess } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; import { buildFxTranscript, + createFxProcessRunner, normalizeFxModel, runFxSession, type FxProcessRunner, @@ -142,6 +146,7 @@ describe("fx CLI session", () => { }); it("maps a committed step limit to max_turns", async () => { + const notice = "Agent step limit reached; continue with a follow-up prompt if needed."; const result = await runFxSession({ prompt: "task", cwd: "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/fake/workspace", @@ -149,13 +154,90 @@ describe("fx CLI session", () => { env: {}, logger, runProcess: async () => ({ - stdout: JSON.stringify({ output: "stopped", exit_code: 0 }), + stdout: JSON.stringify({ output: notice, exit_code: 1 }), stderr: "", - exitCode: 0, + exitCode: 1, }), - store: fakeStore(jsonl(committedEvent("step_limit_reached"))), + store: fakeStore( + jsonl({ + kind: "history_turn_committed", + payload: { turn: { kind: "assistant", assistant: notice } }, + }), + ), }); expect(result.status).toBe("max_turns"); + expect(result.stopReason).toBe(notice); + }); + + it("maps the configured observed step count to max_turns", async () => { + const result = await runFxSession({ + prompt: "task", + cwd: "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/fake/workspace", + home: "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/fake/home", + env: {}, + maxAgentSteps: 1, + logger, + runProcess: async () => ({ + stdout: JSON.stringify({ output: "stopped", exit_code: 1 }), + stderr: "", + exitCode: 1, + }), + store: fakeStore(jsonl(committedEvent())), + }); + expect(result.status).toBe("max_turns"); + expect(result.stopReason).toContain("1 steps"); + }); + + it("tracks spawned process groups for abort and process shutdown", async () => { + const hooks = new EventEmitter(); + const killProcess = vi.fn(() => true); + const children: ChildProcess[] = []; + const spawnProcess = vi.fn(() => { + const child = new EventEmitter() as unknown as ChildProcess; + Object.assign(child, { + pid: 4321 + children.length, + stdout: new PassThrough(), + stderr: new PassThrough(), + stdin: new PassThrough(), + kill: vi.fn(() => true), + }); + children.push(child); + return child; + }); + const runner = createFxProcessRunner({ + spawnProcess: spawnProcess as unknown as typeof import("node:child_process").spawn, + killProcess: killProcess as unknown as typeof process.kill, + processHooks: hooks as unknown as Pick, + killGraceMs: 1, + }); + const run = (signal: AbortSignal) => + runner({ + bin: "fx", + args: ["ask"], + cwd: "/fake", + env: {}, + stdin: "task", + signal, + }); + + const first = run(new AbortController().signal); + expect(hooks.listenerCount("exit")).toBe(1); + hooks.emit("exit", 0); + expect(killProcess).toHaveBeenCalledWith(-4321, "SIGTERM"); + expect(killProcess).toHaveBeenCalledWith(-4321, "SIGKILL"); + children[0]?.emit("close", 0, null); + await first; + killProcess.mockClear(); + hooks.emit("exit", 0); + expect(killProcess).not.toHaveBeenCalled(); + + const controller = new AbortController(); + const second = run(controller.signal); + expect(hooks.listenerCount("exit")).toBe(1); + controller.abort(); + expect(killProcess).toHaveBeenCalledWith(-4322, "SIGTERM"); + children[1]?.emit("close", null, "SIGTERM"); + await second; }); it("reports output that is not JSON", async () => { From 6a9c5b7b167f7526e6dc2dbc400456e62650a9b1 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 24 Aug 2026 11:15:58 -0700 Subject: [PATCH 5/7] chore(evals): stack fx harness on deepagents (planner test merge, lockfile) --- .../evals/tests/framework/benchPlanner.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/evals/tests/framework/benchPlanner.test.ts b/packages/evals/tests/framework/benchPlanner.test.ts index 96bd2d5d8..cf1744f42 100644 --- a/packages/evals/tests/framework/benchPlanner.test.ts +++ b/packages/evals/tests/framework/benchPlanner.test.ts @@ -50,6 +50,20 @@ describe("benchPlanner", () => { }); }); + it("uses the registry-derived fx model override environment key", async () => { + expect(defaultModelsEnvKey("fx")).toBe("EVAL_FX_MODELS"); + await withEnvOverrides({ EVAL_FX_MODELS: "openai/custom-fx" }, async () => { + expect(resolveBenchModelEntries([makeTask()], { harness: "fx" }).modelEntries).toEqual([ + { modelName: "openai/custom-fx", mode: "hybrid", cua: false }, + ]); + }); + await withEnvOverrides({ EVAL_FX_MODELS: "" }, async () => { + expect(resolveBenchModelEntries([makeTask()], { harness: "fx" }).modelEntries).toEqual([ + { modelName: "openai/gpt-5.4-mini", mode: "hybrid", cua: false }, + ]); + }); + }); + it("plans registered pass-through harness rows generically", () => { registerBenchHarness({ harness: "fake_planner_harness", From 21e974711a2a1ddeb7494c5bd3daaf12b48459e9 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 24 Aug 2026 11:40:50 -0700 Subject: [PATCH 6/7] chore(evals): restack fx harness (lockfile, formatting) --- pnpm-lock.yaml | 110 +++++++++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 683061920..b03222c73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -555,7 +555,7 @@ importers: version: 17.4.2 eve: specifier: 'catalog:' - version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) devDependencies: '@types/node': specifier: 'catalog:' @@ -585,11 +585,11 @@ importers: specifier: 'catalog:' version: 1.29.0(zod@4.4.3) ai: - specifier: 'catalog:' - version: 7.0.16(zod@4.4.3) + specifier: ^7.0.38 + version: 7.0.77(zod@4.4.3) eve: specifier: 'catalog:' - version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) zod: specifier: 'catalog:' version: 4.4.3 @@ -636,10 +636,10 @@ importers: version: link:../core '@mastra/core': specifier: 'catalog:' - version: 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + version: 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) '@mastra/mcp': specifier: 'catalog:' - version: 1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) + version: 1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) devDependencies: '@types/node': specifier: 'catalog:' @@ -658,10 +658,10 @@ importers: version: link:../core '@mastra/core': specifier: 'catalog:' - version: 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + version: 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) '@mastra/mcp': specifier: 'catalog:' - version: 1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) + version: 1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) zod: specifier: 'catalog:' version: 4.4.3 @@ -882,12 +882,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/gateway@2.0.118': - resolution: {integrity: sha512-9MZXPd1wiELUvAfHXdotmCp2tt6NWoT6OACkRvK1xQy9DG7XeOZ9ZPqFmCqcu5DcmN92BGpA/GtmilrOM+hVJQ==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/gateway@2.0.119': resolution: {integrity: sha512-LTbVThusUYSw6SxsRvdCKveVHCFf+3DBU3XY8+RgBYPBSjSfeKkVMirvGkcj7Hvwd5aTptioHpnreVXzhwjpVw==} engines: {node: '>=18'} @@ -900,6 +894,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.62': + resolution: {integrity: sha512-zR3pustGWhw5eUZHG+fJZx/V/PBe+LxdDpc5hDFWxozG/3MB/+eY62jn+YiR+9uOH+Hx63e5zJoeKLfZfPktWQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/google-vertex@3.0.158': resolution: {integrity: sha512-Z9sY69vlrOR574Bb+3Tjp1P2W1QK4ut7fTUgA3F3lcvNxAvlAyxp24T4bodaBTWP6Q74pCB1IQGL5NaPDN1r2A==} engines: {node: '>=18'} @@ -996,6 +996,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.29': + resolution: {integrity: sha512-7EIbwXiXKGa7EFk6tDZpuZBs6lxhEJpOuHeqrDb3Vd85uYdjwkdRuHnZDVDIIb2+QTSRmyph2NrXcbvuO/KAjQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.5': resolution: {integrity: sha512-oI0t3dvCoqWNV1I8o1Rybi2DXDvHES5r/TrwtJW90tuFLVepgJlftPxrcjh8vaSvjqC2diTuA2vXyjKAyHJm4A==} engines: {node: '>=22'} @@ -1022,6 +1028,10 @@ packages: resolution: {integrity: sha512-tbHKNLirllUNF3ZlkCsXnwab2ZV1Sl4b1H/Cp9ruCce15IBmskE8Gwkk0yo9xDWY+jho2of7lVXtwSsyrq7cwQ==} engines: {node: '>=22'} + '@ai-sdk/provider@4.0.7': + resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} + engines: {node: '>=22'} + '@ai-sdk/togetherai@1.0.49': resolution: {integrity: sha512-g4BpEatN7flh3GZ0CN9KvAUX6uLPmIqGSrKKFvAmC3HZdnF940zl+ChXs3atdbtpr6+cwirxM5RACbUzr0uYhA==} engines: {node: '>=18'} @@ -3510,12 +3520,6 @@ packages: resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==} engines: {node: '>=12'} - ai@5.0.219: - resolution: {integrity: sha512-bFjV5roRz/CqcSuFR+cfOU35O/7Z9U/2y5DyMIJUx6igkAdIMJ3HWNh+tOf9Gsrqiqbbjs7YMMidHi2HsW2PMg==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - ai@5.0.220: resolution: {integrity: sha512-8v7IFO+OMjVJeprLSNemO9GnDMBcoslid1SlxzhxlqPLnFS6o2uI+V5enEYZ3L0rLcu5aXeNilqP8VMccgyq6A==} engines: {node: '>=18'} @@ -3528,6 +3532,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ai@7.0.77: + resolution: {integrity: sha512-muLtBSTAUCreR77L16w4AFBiX2gK/RNt84EKp8m03SN9+MfNlC5EGqYYttRjYKV3xe0a33yj1Zawj1EnjejIWw==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv-draft-04@1.0.0: resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} peerDependencies: @@ -8205,13 +8215,6 @@ snapshots: zod: 4.4.3 optional: true - '@ai-sdk/gateway@2.0.118(zod@4.4.3)': - dependencies: - '@ai-sdk/provider': 2.0.3 - '@ai-sdk/provider-utils': 3.0.30(zod@4.4.3) - '@vercel/oidc': 3.1.0 - zod: 4.4.3 - '@ai-sdk/gateway@2.0.119(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.3 @@ -8226,6 +8229,13 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.4.3 + '@ai-sdk/gateway@4.0.62(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.29(zod@4.4.3) + '@vercel/oidc': 3.2.0 + zod: 4.4.3 + '@ai-sdk/google-vertex@3.0.158(zod@4.4.3)': dependencies: '@ai-sdk/anthropic': 2.0.91(zod@4.4.3) @@ -8343,6 +8353,15 @@ snapshots: undici: 7.29.0 zod: 4.4.3 + '@ai-sdk/provider-utils@5.0.29(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.7 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.1.0 + undici: 7.29.0 + zod: 4.4.3 + '@ai-sdk/provider-utils@5.0.5(zod@4.4.3)': dependencies: '@ai-sdk/provider': 4.0.2 @@ -8371,6 +8390,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@4.0.7': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/togetherai@1.0.49(zod@4.4.3)': dependencies: '@ai-sdk/openai-compatible': 1.0.46(zod@4.4.3) @@ -8843,7 +8866,7 @@ snapshots: '@browserbasehq/sdk': 2.16.0 '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - ai: 5.0.219(zod@4.4.3) + ai: 5.0.220(zod@4.4.3) devtools-protocol: 0.0.1642743 fetch-cookie: 3.2.0 openai: 4.104.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) @@ -9789,7 +9812,7 @@ snapshots: '@mariozechner/clipboard-win32-x64-msvc': 0.3.9 optional: true - '@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3)': + '@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3)': dependencies: '@a2a-js/sdk': 0.3.14(express@5.2.1) '@ai-sdk/provider-utils-v5': '@ai-sdk/provider-utils@3.0.30(zod@4.4.3)' @@ -9805,7 +9828,7 @@ snapshots: '@sindresorhus/slugify': 2.2.1 '@standard-schema/spec': 1.1.0 ajv: 8.20.0 - chat: 4.37.0(ai@7.0.16(zod@4.4.3))(zod@4.4.3) + chat: 4.37.0(ai@7.0.77(zod@4.4.3))(zod@4.4.3) croner: 10.0.1 dotenv: 17.4.2 execa: 9.6.1 @@ -9835,9 +9858,9 @@ snapshots: - utf-8-validate - workflow - '@mastra/mcp@1.15.1(@mastra/core@1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3)': + '@mastra/mcp@1.15.1(@mastra/core@1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3)': dependencies: - '@mastra/core': 1.57.0(ai@7.0.16(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) + '@mastra/core': 1.57.0(ai@7.0.77(zod@4.4.3))(bufferutil@4.1.0)(express@5.2.1)(rxjs@7.8.2)(zod@4.4.3) '@modelcontextprotocol/ext-apps': 1.7.5(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(zod@4.4.3) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) exit-hook: 5.1.0 @@ -11187,14 +11210,6 @@ snapshots: clean-stack: 4.2.0 indent-string: 5.0.0 - ai@5.0.219(zod@4.4.3): - dependencies: - '@ai-sdk/gateway': 2.0.118(zod@4.4.3) - '@ai-sdk/provider': 2.0.3 - '@ai-sdk/provider-utils': 3.0.30(zod@4.4.3) - '@opentelemetry/api': 1.9.0 - zod: 4.4.3 - ai@5.0.220(zod@4.4.3): dependencies: '@ai-sdk/gateway': 2.0.119(zod@4.4.3) @@ -11210,6 +11225,13 @@ snapshots: '@ai-sdk/provider-utils': 5.0.5(zod@4.4.3) zod: 4.4.3 + ai@7.0.77(zod@4.4.3): + dependencies: + '@ai-sdk/gateway': 4.0.62(zod@4.4.3) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.29(zod@4.4.3) + zod: 4.4.3 + ajv-draft-04@1.0.0(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -11641,7 +11663,7 @@ snapshots: chardet@2.2.0: {} - chat@4.37.0(ai@7.0.16(zod@4.4.3))(zod@4.4.3): + chat@4.37.0(ai@7.0.77(zod@4.4.3))(zod@4.4.3): dependencies: '@workflow/serde': 4.1.0-beta.2 mdast-util-to-string: 4.0.0 @@ -11651,7 +11673,7 @@ snapshots: remend: 1.3.0 unified: 11.0.5 optionalDependencies: - ai: 7.0.16(zod@4.4.3) + ai: 7.0.77(zod@4.4.3) zod: 4.4.3 transitivePeerDependencies: - supports-color @@ -12326,9 +12348,9 @@ snapshots: etag@1.8.1: {} - eve@0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): + eve@0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.77(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): dependencies: - ai: 7.0.16(zod@4.4.3) + ai: 7.0.77(zod@4.4.3) nitro: 3.0.260610-beta(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@2.7.0)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) undici: 8.9.0 optionalDependencies: From 09b9b07d86e140643cdcc1b69ed30414173ec9aa Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 24 Aug 2026 11:53:38 -0700 Subject: [PATCH 7/7] chore(evals): restack fx harness (lockfile, formatting) --- packages/evals/package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/evals/package.json b/packages/evals/package.json index 6adb30469..0eb8b050a 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -28,11 +28,9 @@ "@browserbasehq/stagehand-integrations-codex-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-deepagents-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-mastra-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-pi-sdk": "workspace:*", - "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", - "@browserbasehq/stagehand-integrations-deepagents-sdk": "workspace:*", - "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", "ai": "^5.0.133", "browse": "0.9.5", "dotenv": "^17.3.1",