Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions apps/server/scripts/record-codex-app-server-replay-fixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
SUBAGENT_CONTINUE_PARENT_PROMPT,
SUBAGENT_CONTINUE_PROMPT,
SUBAGENT_PROMPT,
SUBAGENT_V2_APPROVAL_PROMPT,
SUBAGENT_V2_PROMPT,
SUBAGENT_V2_NESTED_PROMPT,
THREAD_ROLLBACK_AFTER_PROMPT,
Expand Down Expand Up @@ -86,6 +87,7 @@ const SCENARIO_NAMES = [
"subagent",
"subagent_continue",
"subagent_v2",
"subagent_v2_approval",
"subagent_v2_nested",
"multi_turn",
"queued_turn",
Expand Down Expand Up @@ -469,6 +471,31 @@ function scenarios(): ReadonlyArray<ReplayScenario> {
},
],
},
{
name: "subagent_v2_approval",
fileName: "subagent_v2_approval.ndjson",
description:
"One root turn in approval-required mode whose multi-agent v2 subagent runs a command that needs approval.",
runs: [
{
name: "spawn-v2-subagent-needing-approval",
description:
"The child inherits the root's approval policy, so its write asks the client for approval on the child's native thread and turn.",
// The adapter's approval-required turn defaults.
turnDefaults: {
approvalPolicy: "untrusted",
sandboxPolicy: { type: "readOnly" },
},
steps: [
{
type: "turn",
label: "spawn-v2-subagent-needing-approval",
prompt: SUBAGENT_V2_APPROVAL_PROMPT,
},
],
},
],
},
{
name: "subagent_v2_nested",
fileName: "subagent_v2_nested.ndjson",
Expand Down
61 changes: 43 additions & 18 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1069,6 +1069,26 @@ interface CodexSubagentThreadContext {
task: OrchestrationV2Subagent;
}

/**
* The top-level turn a (possibly nested) subagent turn runs under, and the
* subagent on that turn's thread that leads to it. Native subagent threads are
* hidden from the sidebar, so their approvals are asked there instead.
*/
const approvalOwnerCodexTurn = (
context: ActiveCodexTurnContext,
): {
readonly owner: ActiveCodexTurnContext;
readonly subagent: CodexSubagentThreadContext | null;
} => {
let owner = context;
let subagent: CodexSubagentThreadContext | null = null;
while (owner.subagent !== null) {
subagent = owner.subagent;
owner = owner.subagent.parentContext;
}
return { owner, subagent };
};

const isDescendantCodexTurn = (
candidate: ActiveCodexTurnContext,
ancestor: ActiveCodexTurnContext,
Expand Down Expand Up @@ -3464,37 +3484,42 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi
}) =>
Effect.gen(function* () {
const createdAt = yield* DateTime.now;
const parentNodeId = idAllocator.derive.nodeFromProviderItem({
driver: CODEX_PROVIDER,
nativeItemId: input.nativeItemId,
});
// A subagent's approval is asked on the top-level thread and run,
// under the subagent that asked, where the user can see and answer it.
const { owner, subagent } = approvalOwnerCodexTurn(input.context);
const parentNodeId =
subagent?.subagentNodeId ??
idAllocator.derive.nodeFromProviderItem({
driver: CODEX_PROVIDER,
nativeItemId: input.nativeItemId,
});
const ordinal = yield* resolveItemOrdinal(
input.context,
owner,
`${input.nativeItemId}:approval:${input.nativeRequestId}`,
);
const requestId = yield* idAllocator.allocate.runtimeRequest({
driver: CODEX_PROVIDER,
providerTurnId: input.context.providerTurnId,
providerTurnId: owner.providerTurnId,
nativeRequestId: input.nativeRequestId,
});
const nodeId = idAllocator.derive.approvalNode({ requestId });
const providerSessionId = input.context.input.providerThread.providerSessionId;
const providerSessionId = owner.input.providerThread.providerSessionId;
if (providerSessionId === null) {
return yield* toProtocolError(
`Provider thread ${input.context.providerThread.id} is missing a provider session id.`,
`Provider thread ${owner.providerThread.id} is missing a provider session id.`,
);
}
const node: OrchestrationV2ExecutionNode = {
id: nodeId,
threadId: input.context.projectionThreadId,
runId: input.context.projectionRunId,
threadId: owner.projectionThreadId,
runId: owner.projectionRunId,
parentNodeId,
rootNodeId: input.context.rootNodeId,
rootNodeId: owner.rootNodeId,
kind: "approval_request",
status: "waiting",
countsForRun: false,
providerThreadId: input.context.providerThread.id,
providerTurnId: input.context.providerTurnId,
providerThreadId: owner.providerThread.id,
providerTurnId: owner.providerTurnId,
nativeItemRef: codexNativeItemRef(input.nativeItemId),
runtimeRequestId: requestId,
checkpointScopeId: null,
Expand All @@ -3504,7 +3529,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi
const request: OrchestrationV2RuntimeRequest = {
id: requestId,
nodeId,
providerTurnId: input.context.providerTurnId,
providerTurnId: owner.providerTurnId,
nativeRequestRef: {
driver: CODEX_PROVIDER,
nativeId: input.nativeRequestId,
Expand All @@ -3521,11 +3546,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi
};
const turnItem: OrchestrationV2TurnItem = {
id: idAllocator.derive.approvalTurnItem({ requestId }),
threadId: input.context.projectionThreadId,
runId: input.context.projectionRunId,
threadId: owner.projectionThreadId,
runId: owner.projectionRunId,
nodeId,
providerThreadId: input.context.providerThread.id,
providerTurnId: input.context.providerTurnId,
providerThreadId: owner.providerThread.id,
providerTurnId: owner.providerTurnId,
nativeItemRef: codexNativeItemRef(input.nativeItemId),
parentItemId: null,
ordinal,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,18 @@ const scenarioExpectations = {
turnCompletedCount: 2,
approvalRequestCount: 0,
},
subagent_v2_approval: {
outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
incoming: [
"item/commandExecution/requestApproval",
"serverRequest/resolved",
"item/completed",
"turn/completed",
],
turnStartCount: 1,
turnCompletedCount: 2,
approvalRequestCount: 1,
},
subagent_v2_nested: {
outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
incoming: ["turn/started", "item/completed", "item/agentMessage/delta", "turn/completed"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ export type OrchestratorV2ScenarioStep =
readonly commandId: CommandId;
readonly decision?: ProviderApprovalDecision;
readonly answers?: ProviderUserInputAnswers;
/** Captures the shell snapshot under this key while the request is pending. */
readonly shellSnapshotKeyWhilePending?: string;
};

export interface OrchestratorV2Scenario {
Expand Down Expand Up @@ -556,6 +558,12 @@ export function runOrchestratorV2Scenario(
break;
case "respond_to_next_runtime_request": {
const request = yield* waitForPendingRuntimeRequest(step.threadId);
if (step.shellSnapshotKeyWhilePending !== undefined) {
capturedShellSnapshots.set(
step.shellSnapshotKeyWhilePending,
yield* orchestrator.getShellSnapshot(),
);
}
const result = yield* orchestrator.dispatch({
type: "runtime-request.respond",
commandId: step.commandId,
Expand Down
18 changes: 18 additions & 0 deletions apps/server/src/orchestration-v2/testkit/fixtures/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ import { assertSubagentContinueOutput } from "./subagent_continue/codex_output.t
import { subagentContinueInput } from "./subagent_continue/input.ts";
import { assertSubagentV2Output } from "./subagent_v2/codex_output.ts";
import { subagentV2Input, subagentV2NestedInput } from "./subagent_v2/input.ts";
import { assertSubagentV2ApprovalOutput } from "./subagent_v2_approval/codex_output.ts";
import {
SUBAGENT_V2_APPROVAL_POLICY,
subagentV2ApprovalInput,
} from "./subagent_v2_approval/input.ts";
import { assertSubagentV2NestedOutput } from "./subagent_v2_nested/codex_output.ts";
import { assertClaudeThreadRollbackOutput } from "./thread_rollback/claude_output.ts";
import { assertThreadRollbackOutput } from "./thread_rollback/codex_output.ts";
Expand Down Expand Up @@ -577,6 +582,19 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray<OrchestratorReplayFixtu
},
],
},
{
name: "subagent_v2_approval",
buildInput: subagentV2ApprovalInput,
providers: [
{
driver: ProviderDriverKind.make("codex"),
transcriptFile: new URL("./subagent_v2_approval/codex_transcript.ndjson", import.meta.url),
modelSelection: CODEX_MODEL_SELECTION,
runtimePolicyOverride: SUBAGENT_V2_APPROVAL_POLICY,
assertOutput: assertSubagentV2ApprovalOutput,
},
],
},
{
name: "subagent_v2_nested",
buildInput: subagentV2NestedInput,
Expand Down
7 changes: 7 additions & 0 deletions apps/server/src/orchestration-v2/testkit/fixtures/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ export const SUBAGENT_V2_NESTED_PROMPT =
"Spawn one subagent and tell it to spawn its own subagent, which must in turn spawn one more subagent whose only task is to reply with exactly: Hello. Each agent waits for its child and replies with exactly what the child said. Wait for your subagent, then reply with exactly what it said.";
export const SUBAGENT_V2_PROMPT =
"Spawn one subagent whose only task is to reply with exactly: Hello. Wait for it to finish, then reply with exactly what it said.";
export const SUBAGENT_V2_APPROVAL_PROMPT =
"Do not run any commands yourself. Spawn one subagent whose only task is to run this exact shell command: printf 'subagent approval fixture' > subagent-approval.txt and then reply with exactly: Written. Wait for it to finish, then reply with exactly what it said.";
export const OPENCODE_SUBAGENT_PROMPT =
"Use the task tool exactly once. Delegate to the general subagent with this prompt: Respond exactly CHILD_OK. After the task completes, respond exactly PARENT_OK.";
export const SUBAGENT_CONTINUE_PROMPT =
Expand Down Expand Up @@ -225,6 +227,8 @@ export type OrchestratorFixtureInputStep =
OrchestrationV2Command,
{ readonly type: "runtime-request.respond" }
>["decision"];
/** Captures the shell snapshot under this key while the request is pending. */
readonly shellSnapshotKeyWhilePending?: string;
}
| {
readonly type: "answer_next_user_input_request";
Expand Down Expand Up @@ -660,6 +664,9 @@ export function materializeFixtureInput(input: {
threadId: ids.threadId,
commandId: commands.at(-1)!.commandId,
decision: step.decision ?? "accept",
...(step.shellSnapshotKeyWhilePending === undefined
? {}
: { shellSnapshotKeyWhilePending: step.shellSnapshotKeyWhilePending }),
};
steps.push({ type: "advance_clock", duration: "1 millis" });
steps.push({ type: "await_thread_idle", threadId: ids.threadId });
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { assert } from "@effect/vitest";
import type { ProviderReplayTranscript } from "@t3tools/contracts";

import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
import {
assertBaseProjection,
assertSemanticProjectionIntegrity,
assertUserMessagesInclude,
projectionFor,
SUBAGENT_V2_APPROVAL_PROMPT,
} from "../shared.ts";
import { SUBAGENT_APPROVAL_PENDING_SHELL_KEY } from "./input.ts";

/**
* A multi-agent v2 child inherits the root's approval policy, and Codex asks
* for its command approval on the child's own native thread and turn. The
* request is asked on the parent thread and run instead, under the subagent,
* because native subagent threads are hidden from the sidebar.
*/
export function assertSubagentV2ApprovalOutput(
result: OrchestratorV2ScenarioResult,
transcript: ProviderReplayTranscript,
) {
assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] });

const projection = projectionFor(result, transcript.scenario);
assertSemanticProjectionIntegrity(projection);
assertUserMessagesInclude(projection, [SUBAGENT_V2_APPROVAL_PROMPT]);
const run = projection.runs[0]!;

assert.lengthOf(projection.subagents, 1);
const subagent = projection.subagents[0]!;
assert.equal(subagent.origin, "provider_native");
assert.equal(subagent.status, "completed");
assert.equal(subagent.result, "Written.");
assert.isNotNull(subagent.childThreadId);

const requests = projection.runtimeRequests;
assert.lengthOf(requests, 1);
const request = requests[0]!;
assert.equal(request.kind, "command");
assert.equal(request.status, "resolved");
assert.equal(request.decision, "accept");
assert.equal(
projection.providerTurns.find((turn) => turn.id === request.providerTurnId)?.runAttemptId,
run.activeAttemptId,
"the request belongs to the parent's root provider turn",
);

const node = projection.nodes.find((candidate) => candidate.id === request.nodeId);
assert.isDefined(node);
assert.equal(node.kind, "approval_request");
assert.equal(node.runId, run.id);
assert.equal(node.parentNodeId, subagent.id, "the approval hangs off the subagent that asked");
assert.equal(node.status, "completed");

const approvalItems = projection.turnItems.filter((item) => item.type === "approval_request");
assert.lengthOf(approvalItems, 1);
const approval = approvalItems[0]!;
assert.equal(approval.runId, run.id);
assert.equal(approval.status, "completed");
if (approval.type !== "approval_request") throw new Error("expected an approval item");
assert.equal(approval.requestId, request.id);
assert.include(approval.prompt ?? "", "subagent-approval.txt");

// While Codex waited, the parent reported the approval and the child did not.
const pendingShell = result.capturedShellSnapshots.get(SUBAGENT_APPROVAL_PENDING_SHELL_KEY);
assert.isDefined(pendingShell);
const parentShell = pendingShell.threads.find((thread) => thread.id === projection.thread.id);
assert.equal(parentShell?.pendingRuntimeRequest?.id, request.id);
const childShell = pendingShell.threads.find((thread) => thread.id === subagent.childThreadId);
assert.isDefined(childShell, "the child thread exists while its approval is pending");
assert.isNull(childShell.pendingRuntimeRequest);

// The child keeps its own work and answer, and never holds a request.
const child = result.projections.get(subagent.childThreadId!);
assert.isDefined(child);
assert.lengthOf(child.runtimeRequests, 0);
assert.isFalse(child.turnItems.some((item) => item.type === "approval_request"));
const command = child.turnItems.find((item) => item.type === "command_execution");
assert.equal(command?.status, "completed");
assert.isTrue(
child.turnItems.some(
(item) => item.type === "assistant_message" && item.text.includes("Written."),
),
);
}
Loading
Loading