diff --git a/.changeset/tidy-planes-stop.md b/.changeset/tidy-planes-stop.md new file mode 100644 index 000000000..cc10c30d9 --- /dev/null +++ b/.changeset/tidy-planes-stop.md @@ -0,0 +1,6 @@ +--- +"@truefoundry/trueforge-core": patch +"@truefoundry/trueforge": patch +--- + +Abort sandbox exec work when a turn cancellation signal aborts. diff --git a/packages/trueforge-core/src/core/index.ts b/packages/trueforge-core/src/core/index.ts index 76538942f..7bdeecdb1 100644 --- a/packages/trueforge-core/src/core/index.ts +++ b/packages/trueforge-core/src/core/index.ts @@ -155,7 +155,7 @@ export type { CodeModeErrorSource, CodeModeReply, CodeModeRequest } from './sand export { DaytonaSandboxProvider } from './sandbox/provider/DaytonaProvider'; export type { DaytonaSandboxProviderOptions } from './sandbox/provider/DaytonaProvider'; export { absolutizeRelativeExecEnv } from './sandbox/provider/execEnv'; -export { ensureExecSuccess, shellEscape } from './sandbox/provider/Provider'; +export { SANDBOX_EXEC_ABORTED, ensureExecSuccess, shellEscape } from './sandbox/provider/Provider'; export type { ExecErrorResult, ExecResult, @@ -182,6 +182,7 @@ export { export { SANDBOX_IMAGE_URI } from './sandbox/sandboxImage'; export { existingSandboxIdForProvider, formatSandboxId, parseSandboxId, rawSandboxId } from './sandbox/sandboxRef'; export type { SandboxRefParts } from './sandbox/sandboxRef'; +export { onSignalAbort } from './util/abort'; // Web search export { ParallelWebSearchProvider } from './web-search/ParallelWebSearchProvider'; diff --git a/packages/trueforge-core/src/core/sandbox/Sandbox.ts b/packages/trueforge-core/src/core/sandbox/Sandbox.ts index 47eac9169..6bd44e6fa 100644 --- a/packages/trueforge-core/src/core/sandbox/Sandbox.ts +++ b/packages/trueforge-core/src/core/sandbox/Sandbox.ts @@ -87,6 +87,7 @@ export interface SandboxStoredFile { export interface SandboxOptions { provider: SandboxProvider; existingSandboxId?: string | undefined; + signal?: AbortSignal | undefined; skillMounter?: ISkillMounter | undefined; fileDownloadEnabled?: boolean | undefined; /** Pre-resolved credential-store file content (null = clear / no git auth). */ @@ -198,6 +199,7 @@ export class Sandbox extends LocalToolMCP { private readonly provider: SandboxProvider; private readonly existingSandboxId?: string | undefined; + private readonly signal?: AbortSignal | undefined; private existingSandboxInfo: SandboxInfo | undefined; // Cached promise to prevent concurrent sub-agents from creating duplicate sandboxes. private sandboxCreationPromise?: Promise | undefined; @@ -228,6 +230,7 @@ export class Sandbox extends LocalToolMCP { super({ tracing: options.tracing }); this.provider = options.provider; this.existingSandboxId = options.existingSandboxId; + this.signal = options.signal; this.skillMounter = options.skillMounter; this.fileDownloadEnabled = options.fileDownloadEnabled ?? false; const mcpBoundTimeoutMs = options.mcpRequestTimeoutMs + options.mcpConnectTimeoutMs; @@ -560,6 +563,7 @@ export class Sandbox extends LocalToolMCP { command: input.command, cwd: input.cwd, env, + signal: this.signal, }); return { @@ -595,6 +599,7 @@ export class Sandbox extends LocalToolMCP { command: input.command, cwd: input.cwd, env: retryEnv, + signal: this.signal, }); return { result: { diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 2332325c6..551ce0d8c 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -5,6 +5,7 @@ import { suppressTracing } from '@opentelemetry/core'; import { randomUUID } from 'node:crypto'; import { join } from 'node:path/posix'; import type { Logger } from 'winston'; +import { isSignalAborted, onSignalAbort } from '../../util/abort'; import { extractErrorLogFields } from '../../util/errorLogFields'; import { SandboxFileNotFoundError, @@ -16,7 +17,14 @@ import { import type { CodeModeTransport } from '../codeMode/CodeModeTransport'; import { CodeModeNatsTransport } from '../codeMode/nats/CodeModeNatsTransport'; import { DEFAULT_PREVIEW_URL_EXPIRY_SECONDS, DEFAULT_SANDBOX_NATS_WS_PORT } from '../constants'; -import type { ExecResult, SandboxBuild, SandboxExecParams, SandboxFileInfo, SandboxProvider } from './Provider'; +import { + SANDBOX_EXEC_ABORTED, + type ExecResult, + type SandboxBuild, + type SandboxExecParams, + type SandboxFileInfo, + type SandboxProvider, +} from './Provider'; const SANDBOX_NOT_FOUND_STATUS = 404; /** Another replica already registered this build name; its create is the one that counts. */ @@ -193,18 +201,22 @@ export class DaytonaSandboxProvider implements SandboxProvider { return recovery; } - private async executeWithSandboxRecovery(sandboxId: string, operation: () => Promise): Promise { + private async executeWithSandboxRecovery(params: { + sandboxId: string; + signal: AbortSignal | undefined; + operation: () => Promise; + }): Promise { try { - return await operation(); + return await params.operation(); } catch (originalError) { // TODO: Narrow to a specific Daytona error code once @daytona/sdk exposes one for "sandbox not running". - if (!(originalError instanceof DaytonaError)) { + if (!(originalError instanceof DaytonaError) || isSignalAborted(params.signal)) { throw originalError; } let recovered: boolean; try { - recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(sandboxId); + recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(params.sandboxId); } catch (recoveryError) { this.logger.error('Sandbox recovery failed', { ...extractErrorLogFields(recoveryError), @@ -218,7 +230,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { } try { - return await operation(); + return await params.operation(); } catch (retryError) { this.logger.error('Sandbox operation failed after successful recovery', { ...extractErrorLogFields(retryError), @@ -384,22 +396,56 @@ export class DaytonaSandboxProvider implements SandboxProvider { async exec(params: SandboxExecParams): Promise { return context.with(suppressTracing(context.active()), async (): Promise => { + if (isSignalAborted(params.signal)) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } + // Map the catch to abort only if this exec's handler called stop(); a later signal abort must not hide a Daytona error. + const execAbortRequested = { value: false }; try { - return await this.executeWithSandboxRecovery(params.sandboxId, async () => { - const { sandbox, defaultTimeoutMs } = await this.getOrCreateSandbox(params.sandboxId); - const response = await sandbox.process.executeCommand( - params.command, - params.cwd, - params.env ?? {}, - params.timeoutSeconds ?? defaultTimeoutMs / 1000, - ); - return { - success: true, - response: { exitCode: response.exitCode, result: response.result }, - }; + return await this.executeWithSandboxRecovery({ + sandboxId: params.sandboxId, + signal: params.signal, + operation: async () => { + const { sandbox, defaultTimeoutMs } = await this.getOrCreateSandbox(params.sandboxId); + const timeoutSeconds = params.timeoutSeconds ?? defaultTimeoutMs / 1000; + if (isSignalAborted(params.signal)) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } + const cleanupAbort = onSignalAbort(params.signal, () => { + execAbortRequested.value = true; + void sandbox.stop(timeoutSeconds, true).catch((error: unknown) => { + this.logger.error('Failed to stop Daytona sandbox after exec abort', { + ...extractErrorLogFields(error), + sandboxId: params.sandboxId, + }); + }); + }); + try { + const response = await sandbox.process.executeCommand( + params.command, + params.cwd, + params.env ?? {}, + timeoutSeconds, + ); + if (execAbortRequested.value) { + // Drop only the stale SDK object. The remote sandbox and filesystem stay under the same id. + DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } + return { + success: true, + response: { exitCode: response.exitCode, result: response.result }, + }; + } finally { + cleanupAbort(); + } + }, }); } catch (e: unknown) { DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + if (execAbortRequested.value) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } if (e instanceof SandboxNotAvailableError) { throw e; } @@ -418,18 +464,22 @@ export class DaytonaSandboxProvider implements SandboxProvider { async downloadFile(params: { sandboxId: string; path: string }): Promise { return context.with(suppressTracing(context.active()), async () => { try { - return await this.executeWithSandboxRecovery(params.sandboxId, async () => { - const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); - - const info = await this.getFileInfo(sandbox, params.path); - if (info.isDir) { - throw new SandboxPathIsDirectoryError(params.path); - } - if (info.size > this.fileMaxBytesForDownload) { - throw new SandboxFileTooLargeError(params.path, info.size, this.fileMaxBytesForDownload); - } - - return await sandbox.fs.downloadFile(params.path); + return await this.executeWithSandboxRecovery({ + sandboxId: params.sandboxId, + signal: undefined, + operation: async () => { + const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); + + const info = await this.getFileInfo(sandbox, params.path); + if (info.isDir) { + throw new SandboxPathIsDirectoryError(params.path); + } + if (info.size > this.fileMaxBytesForDownload) { + throw new SandboxFileTooLargeError(params.path, info.size, this.fileMaxBytesForDownload); + } + + return await sandbox.fs.downloadFile(params.path); + }, }); } catch (e: unknown) { if (e instanceof SandboxPathIsDirectoryError || e instanceof SandboxFileTooLargeError) { @@ -447,9 +497,13 @@ export class DaytonaSandboxProvider implements SandboxProvider { async uploadFile(params: { sandboxId: string; remotePath: string; content: Buffer }): Promise { return context.with(suppressTracing(context.active()), async () => { try { - await this.executeWithSandboxRecovery(params.sandboxId, async () => { - const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); - await sandbox.fs.uploadFile(params.content, params.remotePath); + await this.executeWithSandboxRecovery({ + sandboxId: params.sandboxId, + signal: undefined, + operation: async () => { + const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); + await sandbox.fs.uploadFile(params.content, params.remotePath); + }, }); } catch (e: unknown) { DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); @@ -462,10 +516,14 @@ export class DaytonaSandboxProvider implements SandboxProvider { private async getPreviewUrl(params: { sandboxId: string; port: number; expiresInSeconds: number }): Promise { return context.with(suppressTracing(context.active()), async () => { try { - return await this.executeWithSandboxRecovery(params.sandboxId, async () => { - const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); - const signed = await sandbox.getSignedPreviewUrl(params.port, params.expiresInSeconds); - return signed.url; + return await this.executeWithSandboxRecovery({ + sandboxId: params.sandboxId, + signal: undefined, + operation: async () => { + const { sandbox } = await this.getOrCreateSandbox(params.sandboxId); + const signed = await sandbox.getSignedPreviewUrl(params.port, params.expiresInSeconds); + return signed.url; + }, }); } catch (e: unknown) { DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); diff --git a/packages/trueforge-core/src/core/sandbox/provider/Provider.ts b/packages/trueforge-core/src/core/sandbox/provider/Provider.ts index 0a46e5477..80783ffaa 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/Provider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/Provider.ts @@ -14,6 +14,8 @@ export interface ExecErrorResult { export type ExecResult = ExecSuccessResult | ExecErrorResult; +export const SANDBOX_EXEC_ABORTED = 'Sandbox exec aborted'; + /** * Wraps a value in single quotes for safe use in a shell command. Inner single quotes are * escaped via the standard shell idiom: ' -> '\''. Use for any user-controlled value that is @@ -44,6 +46,7 @@ export interface SandboxExecParams { command: string; cwd?: string | undefined; env?: Record | undefined; + signal?: AbortSignal | undefined; /** Overrides the provider's default exec timeout (e.g. for long skill downloads). */ timeoutSeconds?: number | undefined; } diff --git a/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts index 7544eba18..6d978d853 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts @@ -4,6 +4,7 @@ import dedent from 'dedent'; import { randomUUID } from 'node:crypto'; import { join } from 'node:path/posix'; import type { Logger } from 'winston'; +import { isSignalAborted, onSignalAbort } from '../../util/abort'; import { extractErrorLogFields } from '../../util/errorLogFields'; import type { CodeModeTransport } from '../codeMode/CodeModeTransport'; import { CodeModeNatsTransport } from '../codeMode/nats/CodeModeNatsTransport'; @@ -17,6 +18,7 @@ import { import { absolutizeRelativeExecEnv } from './execEnv'; import { ensureExecSuccess, + SANDBOX_EXEC_ABORTED, shellEscape, type ExecResult, type SandboxBuild, @@ -152,9 +154,15 @@ export class TFYSandboxProvider implements SandboxProvider { const controller = new AbortController(); const clientTimeoutMs = (timeoutSeconds + CLIENT_TIMEOUT_BUFFER_SECONDS) * 1000; + const abortState: { source: 'timeout' | 'turn' | undefined } = { source: undefined }; const timer = setTimeout(() => { + abortState.source = 'timeout'; controller.abort(); }, clientTimeoutMs); + const cleanupAbort = onSignalAbort(params.signal, () => { + abortState.source = 'turn'; + controller.abort(); + }); try { const response = await fetch(`${this.serverUrl}/exec`, { @@ -174,6 +182,9 @@ export class TFYSandboxProvider implements SandboxProvider { return result; } catch (e: unknown) { if (e instanceof Error && e.name === 'AbortError') { + if (abortState.source === 'turn' && isSignalAborted(params.signal)) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } this.logger.error(`Sandbox exec timed out after ${String(timeoutSeconds)}s`, extractErrorLogFields(e)); return { success: false, error: `Sandbox exec timed out after ${String(timeoutSeconds)}s` }; } @@ -182,6 +193,7 @@ export class TFYSandboxProvider implements SandboxProvider { return { success: false, error: message }; } finally { clearTimeout(timer); + cleanupAbort(); } }); } diff --git a/packages/trueforge-core/src/core/util/abort.ts b/packages/trueforge-core/src/core/util/abort.ts index 7110de3f1..e9d698814 100644 --- a/packages/trueforge-core/src/core/util/abort.ts +++ b/packages/trueforge-core/src/core/util/abort.ts @@ -1,10 +1,17 @@ -export function onSignalAbort(signal: AbortSignal | undefined, callback: () => void): void { +export function isSignalAborted(signal: AbortSignal | undefined): boolean { + return signal?.aborted === true; +} + +export function onSignalAbort(signal: AbortSignal | undefined, callback: () => void): () => void { if (!signal) { - return; + return () => undefined; } if (signal.aborted) { callback(); - return; + return () => undefined; } signal.addEventListener('abort', callback, { once: true }); + return () => { + signal.removeEventListener('abort', callback); + }; } diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 2c94b868d..abd12f43a 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -1,5 +1,6 @@ -import { Daytona, DaytonaError } from '@daytona/sdk'; +import { Daytona, DaytonaError, type Sandbox } from '@daytona/sdk'; import { DaytonaSandboxProvider } from '../../../src/core/sandbox/provider/DaytonaProvider'; +import { SANDBOX_EXEC_ABORTED } from '../../../src/core/sandbox/provider/Provider'; import { SandboxNotAvailableError } from '../../../src/core/sandbox/SandboxErrors'; import { makeSilentLogger } from '../harnessMocks'; @@ -81,6 +82,69 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => { }); describe('DaytonaSandboxProvider exec', () => { + it('does not recover or retry when abort force-stops the exec sandbox', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + let rejectExecuteCommand: ((error: Error) => void) | undefined; + const execStarted = new Promise(resolve => { + const executeCommand = jest.fn( + ( + _command: string, + _cwd: string | undefined, + _env: Record, + _timeoutSeconds: number, + ): Promise<{ exitCode: number; result: string }> => { + resolve(); + return new Promise((_resolve, reject) => { + rejectExecuteCommand = reject; + }); + }, + ); + const sandbox: Sandbox = Object.assign(Object.create(null), { + name: 'test-tenant.abort', + state: 'started', + process: { executeCommand }, + refreshData: jest.fn(() => Promise.resolve()), + start: jest.fn(() => Promise.resolve()), + stop: jest.fn(() => Promise.resolve()), + }); + jest.spyOn(client, 'get').mockResolvedValue(sandbox); + }); + const provider = new DaytonaSandboxProvider({ + client, + apiKey: 'dtn-test', + apiUrl: API_URL, + tenantName: 'test-tenant', + sandboxImage: 'registry.example.com/sandbox:029ea5ff', + timeoutMs: 1000, + autoStopIntervalInMinutes: 5, + autoArchiveIntervalInMinutes: 60, + autoDeleteIntervalInMinutes: 7200, + fileMaxBytesForDownload: 1024, + logger: makeSilentLogger(), + }); + const controller = new AbortController(); + + const exec = provider.exec({ + sandboxId: 'test-tenant.abort', + command: 'sleep 120', + signal: controller.signal, + }); + await execStarted; + controller.abort(); + const reject = rejectExecuteCommand; + if (reject === undefined) { + throw new Error('executeCommand did not start'); + } + reject(new DaytonaError('sandbox stopped', 500)); + + await expect(exec).resolves.toEqual({ success: false, error: SANDBOX_EXEC_ABORTED }); + const sandbox = await client.get('test-tenant.abort'); + expect(sandbox.stop).toHaveBeenCalledWith(1, true); + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.refreshData).not.toHaveBeenCalled(); + expect(sandbox.process.executeCommand).toHaveBeenCalledTimes(1); + }); + it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => { const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS)); diff --git a/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts b/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts index 686077ea1..d65a78f7d 100644 --- a/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts @@ -1,7 +1,12 @@ import { absolutizeRelativeExecEnv } from '../../../../src/core/sandbox/provider/execEnv'; +import { SANDBOX_EXEC_ABORTED } from '../../../../src/core/sandbox/provider/Provider'; import { TFYSandboxProvider, withMcpClientOnPath } from '../../../../src/core/sandbox/provider/TFYSandboxProvider'; import { makeSilentLogger } from '../../harnessMocks'; +afterEach(() => { + jest.restoreAllMocks(); +}); + describe('TFYSandboxProvider layout', () => { it('keeps exec writes cwd-relative (no extra FS jail)', () => { const provider = new TFYSandboxProvider({ @@ -56,4 +61,52 @@ describe('TFYSandboxProvider layout', () => { expect(withMcpClientOnPath('/usr/bin:/bin')).toBe('mcp-client/bin:/usr/bin:/bin'); expect(withMcpClientOnPath('mcp-client/bin:/usr/bin')).toBe('mcp-client/bin:/usr/bin'); }); + + it('aborts the exec fetch when the turn signal aborts', async () => { + const provider = new TFYSandboxProvider({ + serverUrl: 'http://sandbox.example', + natsBridgeUrl: 'ws://nats.example', + tenantName: 'acme', + fileMaxBytesForDownload: 1024, + logger: makeSilentLogger(), + }); + const sandboxId = 'acme.00000000-0000-0000-0000-000000000001'; + const fetchMock = jest.spyOn(globalThis, 'fetch'); + fetchMock.mockResolvedValueOnce( + new Response( + JSON.stringify({ + success: true, + response: { exitCode: 0, result: '/sandbox/root\n/usr/bin\n' }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + const execFetchStarted = new Promise(resolve => { + fetchMock.mockImplementationOnce((_url, init) => { + const signal = init?.signal; + if (signal === undefined || signal === null) { + return Promise.reject(new Error('Expected fetch signal')); + } + resolve(); + return new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + const error = new Error('Aborted'); + error.name = 'AbortError'; + reject(error); + }, + { once: true }, + ); + }); + }); + }); + const controller = new AbortController(); + + const exec = provider.exec({ sandboxId, command: 'sleep 60', signal: controller.signal }); + await execFetchStarted; + controller.abort(); + + await expect(exec).resolves.toEqual({ success: false, error: SANDBOX_EXEC_ABORTED }); + }); }); diff --git a/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts b/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts index 1418e6b46..986826d32 100644 --- a/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts @@ -9,6 +9,7 @@ const AGENT_COMMAND = 'echo hello'; function makeSandbox(options: { mcpRequestTimeoutMs: number; mcpConnectTimeoutMs: number; + signal?: AbortSignal | undefined; transport?: CodeModeTransport; }): { sandbox: Sandbox; @@ -46,6 +47,7 @@ function makeSandbox(options: { blockDestructiveToolsInCodeMode: true, mcpRequestTimeoutMs: options.mcpRequestTimeoutMs, mcpConnectTimeoutMs: options.mcpConnectTimeoutMs, + signal: options.signal, logger: makeSilentLogger(), tracing: NOOP_AGENT_TRACING, }), @@ -113,4 +115,22 @@ describe('Code Mode timeouts', () => { expect(call.env?.['TFY_CM_REQUEST_TIMEOUT_SECONDS']).toBeUndefined(); expect(call.timeoutSeconds).toBeUndefined(); }); + + it('passes the turn signal only to the user exec command', async () => { + const controller = new AbortController(); + const { sandbox, execCalls } = makeSandbox({ + mcpRequestTimeoutMs: 90_000, + mcpConnectTimeoutMs: 30_000, + signal: controller.signal, + }); + + const call = await execAgentCommand(sandbox, execCalls); + + expect(call.signal).toBe(controller.signal); + const setupCalls = execCalls.filter(item => item.command !== AGENT_COMMAND); + expect(setupCalls.length).toBeGreaterThan(0); + for (const setupCall of setupCalls) { + expect(setupCall.signal).toBeUndefined(); + } + }); }); diff --git a/packages/trueforge/src/apis/turns.ts b/packages/trueforge/src/apis/turns.ts index 5d933410f..a20865051 100644 --- a/packages/trueforge/src/apis/turns.ts +++ b/packages/trueforge/src/apis/turns.ts @@ -219,7 +219,7 @@ function createTurnResolver(deps: { mcpRequestTimeoutMs: configuration.MCP_REQUEST_TIMEOUT_MS, mcpConnectTimeoutMs: configuration.MCP_CONNECT_TIMEOUT_MS, mcpMaxResponseBytes: configuration.MCP_TOOL_CALL_MAX_RESPONSE_BYTES, - sandboxProvider: async ({ spec, existingSandboxId, tracing }) => { + sandboxProvider: async ({ spec, existingSandboxId, signal, tracing }) => { const provider = await resolveSandboxProvider({ tenant_id, store: sandboxProviderStore, @@ -263,6 +263,7 @@ function createTurnResolver(deps: { skills: mountSkills, fileDownloadEnabled: spec.config.sandbox.file_downloads, existingSandboxId: carriedSandboxId, + signal, tracing, }); }, diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts index c1287be3e..05c42d97f 100644 --- a/packages/trueforge/src/runtime/sessionResources.ts +++ b/packages/trueforge/src/runtime/sessionResources.ts @@ -266,12 +266,14 @@ export function buildTurnSandbox(input: { skills?: readonly Skill[]; fileDownloadEnabled: boolean; existingSandboxId?: string | undefined; + signal: AbortSignal; tracing: AgentTracing; }): Sandbox { // Empty mounter still uploads requested-skills file so existing skills are cleaned up. return new Sandbox({ provider: input.provider, existingSandboxId: input.existingSandboxId, + signal: input.signal, fileDownloadEnabled: input.fileDownloadEnabled, blockDestructiveToolsInCodeMode: true, mcpRequestTimeoutMs: configuration.MCP_REQUEST_TIMEOUT_MS, diff --git a/packages/trueforge/src/sandbox/local/core/hostRun.ts b/packages/trueforge/src/sandbox/local/core/hostRun.ts index 2ca76071d..5802732e2 100644 --- a/packages/trueforge/src/sandbox/local/core/hostRun.ts +++ b/packages/trueforge/src/sandbox/local/core/hostRun.ts @@ -7,6 +7,7 @@ * {@link initSrt} — the same platform captured by LocalSandboxProvider.isSupported. */ import { getDefaultWritePaths, SandboxManager } from '@anthropic-ai/sandbox-runtime'; +import { onSignalAbort } from '@truefoundry/trueforge-core/core'; import { execFile, spawn, type ChildProcess } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { realpathSync } from 'node:fs'; @@ -217,6 +218,7 @@ export interface SessionResult { exitCode: number; protocolError: string | undefined; timedOut: boolean; + aborted: boolean; /** Process-group leader pid of the sandboxed command (Unix). */ childPid: number | undefined; } @@ -490,6 +492,7 @@ export async function runSupervisorSession(params: { onChildSpawn?: (pid: number) => void; /** Hard wall-clock limit for the sandboxed command; caller must choose deliberately. */ timeoutMs: number; + signal?: AbortSignal | undefined; }): Promise { const { sandboxRootPath, @@ -501,6 +504,7 @@ export async function runSupervisorSession(params: { stdin, onChildSpawn, timeoutMs, + signal, } = params; const command = wrapSandboxCommand({ platform, command: rawCommand }); @@ -565,6 +569,7 @@ export async function runSupervisorSession(params: { let bufferedOutput = 0; let protocolError: string | undefined; let timedOut = false; + let aborted = false; let closed = false; const ignoreStreamError = ( @@ -607,14 +612,22 @@ export async function runSupervisorSession(params: { timedOut = true; killExecTree(child); }, timeoutMs); + const cleanupAbort = onSignalAbort(signal, () => { + aborted = true; + killExecTree(child); + }); + const cleanup = (): void => { + clearTimeout(timer); + cleanupAbort(); + SandboxManager.cleanupAfterCommand(); + }; child.on('error', error => { if (closed) { return; } closed = true; - clearTimeout(timer); - SandboxManager.cleanupAfterCommand(); + cleanup(); reject(error); }); @@ -623,14 +636,20 @@ export async function runSupervisorSession(params: { return; } closed = true; - clearTimeout(timer); - SandboxManager.cleanupAfterCommand(); + cleanup(); + let exitCode = 0; + if (code !== null) { + exitCode = code; + } else if (timedOut || aborted) { + exitCode = 1; + } resolve({ stdoutText, stderrText, - exitCode: typeof code === 'number' ? code : timedOut ? 1 : 0, + exitCode, protocolError, timedOut, + aborted, childPid: child.pid, }); }); diff --git a/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts b/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts index b69625010..9d1f29c7f 100644 --- a/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts +++ b/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts @@ -10,6 +10,7 @@ import type { } from '@truefoundry/trueforge-core/core'; import { absolutizeRelativeExecEnv, + SANDBOX_EXEC_ABORTED, SandboxFileNotFoundError, SandboxFileTooLargeError, SandboxNotAvailableError, @@ -610,6 +611,9 @@ export class LocalSandboxProvider implements SandboxProvider { async exec(params: SandboxExecParams): Promise { this.ensureSandboxRoot(params.sandboxId); + if (params.signal?.aborted === true) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } try { await this.ensureSrt(); await this.ensureVenv(params.sandboxId); @@ -629,7 +633,11 @@ export class LocalSandboxProvider implements SandboxProvider { cwd, ...(env === undefined ? {} : { env }), timeoutMs: timeoutSeconds * 1000, + signal: params.signal, }); + if (session.aborted) { + return { success: false, error: SANDBOX_EXEC_ABORTED }; + } if (session.protocolError !== undefined) { return { success: false, error: session.protocolError }; } diff --git a/packages/trueforge/tests/sandbox/local/smoke.test.ts b/packages/trueforge/tests/sandbox/local/smoke.test.ts index 75d8c2547..bb0752e36 100644 --- a/packages/trueforge/tests/sandbox/local/smoke.test.ts +++ b/packages/trueforge/tests/sandbox/local/smoke.test.ts @@ -3,7 +3,7 @@ * plus Code Mode UDS and security probes. Run via `pnpm smoke`. */ import { getDefaultWritePaths, SandboxManager } from '@anthropic-ai/sandbox-runtime'; -import { CodeModeDispatcher, type IToolSet } from '@truefoundry/trueforge-core/core'; +import { CodeModeDispatcher, SANDBOX_EXEC_ABORTED, type IToolSet } from '@truefoundry/trueforge-core/core'; import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -129,6 +129,18 @@ function sleep(ms: number): Promise { }); } +async function waitForHostFile(path: string): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await access(path); + return; + } catch { + await sleep(50); + } + } + throw new Error(`timed out waiting for ${path}`); +} + /** * After initSrt grants the Code Mode parent, a sock under that parent is * connectable; a leftover host /tmp sock outside the parent is not. @@ -2020,6 +2032,33 @@ async function main(): Promise { assert.equal(persist2.response.result, 'persist-ok\n'); console.log('ok: sandbox persists across execs'); + const abortController = new AbortController(); + const abortStartedPath = join(sandboxId, 'abort-started.txt'); + const abortExecStartedAt = Date.now(); + const abortExec = provider.exec({ + sandboxId, + command: "printf 'abort-started\\n' > abort-started.txt; sleep 120", + timeoutSeconds: 120, + signal: abortController.signal, + }); + await waitForHostFile(abortStartedPath); + abortController.abort(); + const abortResult = await abortExec; + const abortElapsedMs = Date.now() - abortExecStartedAt; + assert.equal(abortResult.success, false); + if (abortResult.success) throw new Error('unreachable'); + assert.equal(abortResult.error, SANDBOX_EXEC_ABORTED); + assert.ok(abortElapsedMs < 10_000, `abort took ${String(abortElapsedMs)}ms`); + const afterAbort = await provider.exec({ + sandboxId, + command: "cat persist.txt && printf 'after-abort-ok\\n'", + }); + assert.equal(afterAbort.success, true); + if (!afterAbort.success) throw new Error('unreachable'); + assert.equal(afterAbort.response.exitCode, 0, afterAbort.response.result); + assert.equal(afterAbort.response.result, 'persist-ok\nafter-abort-ok\n'); + console.log('ok: abort stops current exec and same sandbox remains usable'); + const flood = await provider.exec({ sandboxId, command: `python3 -c 'import sys; sys.stdout.write("x" * ${String(MAX_OUTPUT_BYTES + 1)})'`,