From dad26133e8645c040b91de4ea7768c0fbbf21f69 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 22 Sep 2026 17:55:21 -0400 Subject: [PATCH 01/15] feat: Codex voice conversations on web, desktop, and mobile Codex 0.156 voice mode for T3 Code: talk to a realtime voice model that hands work to Codex in the thread and reads its answers back. Audio runs over WebRTC between the client and OpenAI; the server relays only SDP and captions over provider.voice.session, and ending the stream stops the conversation. On orchestrator V2, Codex's self-started voice turns are interrupted and resubmitted as ordinary user runs, since V2 has no provider-initiated runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/desktop/src/window/DesktopWindow.test.ts | 4 + apps/desktop/src/window/DesktopWindow.ts | 5 + .../src/window/MainWindowPermissions.test.ts | 46 ++ .../src/window/MainWindowPermissions.ts | 63 +++ apps/mobile/app.config.ts | 22 +- apps/mobile/package.json | 2 + apps/mobile/src/components/AppSymbol.tsx | 4 + .../src/features/threads/ThreadComposer.tsx | 35 ++ .../features/voice-mode/ComposerVoiceMode.tsx | 170 +++++++ .../src/features/voice-mode/useVoiceMode.ts | 58 +++ .../src/features/voice-mode/voiceMode.ts | 117 +++++ .../voice-mode/voiceModePresentation.test.ts | 70 +++ .../voice-mode/voiceModePresentation.ts | 68 +++ apps/server/src/auth/RpcAuthorization.ts | 1 + .../Adapters/CodexAdapterV2.ts | 78 ++++ .../ProviderContinuationService.test.ts | 32 ++ .../ProviderContinuationService.ts | 54 ++- apps/server/src/provider/CodexProvider.ts | 1 + apps/server/src/provider/CodexVoice.test.ts | 273 +++++++++++ apps/server/src/provider/CodexVoice.ts | 292 ++++++++++++ .../provider/ProviderInstanceRegistry.test.ts | 1 + apps/server/src/ws.ts | 48 ++ apps/web/src/components/ChatView.tsx | 25 +- apps/web/src/components/CommandPalette.tsx | 66 +++ apps/web/src/components/chat/ChatComposer.tsx | 115 +++++ .../src/components/chat/VoiceModeStrip.tsx | 172 +++++++ apps/web/src/composer-logic.test.ts | 5 + apps/web/src/composer-logic.ts | 5 +- apps/web/src/keybindings.test.ts | 17 + apps/web/src/voice/voiceAvailability.test.ts | 40 ++ apps/web/src/voice/voiceAvailability.ts | 46 ++ apps/web/src/voice/voiceMode.ts | 144 ++++++ docs/internals/voice-conversations.md | 34 ++ docs/user/providers-codex.md | 23 + packages/client-runtime/package.json | 4 + packages/client-runtime/src/rpc/client.ts | 1 + .../src/voice-mode/controller.test.ts | 157 +++++++ .../src/voice-mode/controller.ts | 433 ++++++++++++++++++ .../client-runtime/src/voice-mode/index.ts | 19 + .../client-runtime/src/voice-mode/session.ts | 54 +++ packages/contracts/src/keybindings.ts | 2 + packages/contracts/src/provider.ts | 52 +++ packages/contracts/src/rpc.ts | 12 + packages/contracts/src/server.ts | 2 + packages/contracts/src/settings.ts | 29 +- .../src/server/ProviderAdapter.ts | 13 +- .../server/ProviderContinuationRequests.ts | 5 + .../provider-core/src/server/snapshotProbe.ts | 2 + packages/shared/src/composerTrigger.ts | 2 +- packages/shared/src/keybindings.ts | 1 + pnpm-lock.yaml | 62 +++ scripts/build-desktop-artifact.test.ts | 1 + scripts/build-desktop-artifact.ts | 4 + third-party-licenses.config.json | 15 + 54 files changed, 2972 insertions(+), 34 deletions(-) create mode 100644 apps/desktop/src/window/MainWindowPermissions.test.ts create mode 100644 apps/desktop/src/window/MainWindowPermissions.ts create mode 100644 apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx create mode 100644 apps/mobile/src/features/voice-mode/useVoiceMode.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceMode.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts create mode 100644 apps/mobile/src/features/voice-mode/voiceModePresentation.ts create mode 100644 apps/server/src/provider/CodexVoice.test.ts create mode 100644 apps/server/src/provider/CodexVoice.ts create mode 100644 apps/web/src/components/chat/VoiceModeStrip.tsx create mode 100644 apps/web/src/voice/voiceAvailability.test.ts create mode 100644 apps/web/src/voice/voiceAvailability.ts create mode 100644 apps/web/src/voice/voiceMode.ts create mode 100644 docs/internals/voice-conversations.md create mode 100644 packages/client-runtime/src/voice-mode/controller.test.ts create mode 100644 packages/client-runtime/src/voice-mode/controller.ts create mode 100644 packages/client-runtime/src/voice-mode/index.ts create mode 100644 packages/client-runtime/src/voice-mode/session.ts diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index c9a49b1d735b..c45f0fd1d818 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -94,6 +94,10 @@ function makeFakeBrowserWindow() { reloadIgnoringCache: vi.fn(), replaceMisspelling: vi.fn(), send: vi.fn(), + session: { + setPermissionCheckHandler: vi.fn(), + setPermissionRequestHandler: vi.fn(), + }, setBackgroundThrottling: vi.fn(), setWindowOpenHandler: vi.fn(), toggleDevTools: vi.fn(), diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 75393d54a1b2..c6ef8a074db0 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -32,6 +32,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts"; +import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts"; import { makeQuitShortcutHandler } from "./QuitHold.ts"; import { copyContextMenuImage } from "./ContextMenuImage.ts"; @@ -440,6 +441,10 @@ export const make = Effect.gen(function* () { if (environment.platform === "darwin") { window.setAutoHideCursor(false); } + installMainWindowPermissionHandlers(window.webContents.session, { + appUrl: applicationUrl, + platform: environment.platform, + }); let boundsPersistFiber: Fiber.Fiber | undefined; let pendingBoundsPersistFiber: Fiber.Fiber | undefined; let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds; diff --git a/apps/desktop/src/window/MainWindowPermissions.test.ts b/apps/desktop/src/window/MainWindowPermissions.test.ts new file mode 100644 index 000000000000..971bcb7a3320 --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { decideMainWindowMediaRequest, isAppUrl } from "./MainWindowPermissions.ts"; + +const APP_URL = "t3code://app/"; + +function decide(overrides: Partial[0]> = {}) { + return decideMainWindowMediaRequest({ + requestingUrl: "t3code://app/threads/abc", + appUrl: APP_URL, + mediaTypes: ["audio"], + platform: "darwin", + microphoneStatus: () => "granted", + ...overrides, + }); +} + +describe("isAppUrl", () => { + it("matches the app shell and nothing that merely shares its prefix", () => { + expect(isAppUrl("t3code://app", APP_URL)).toBe(true); + expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true); + expect(isAppUrl("t3code://application/", APP_URL)).toBe(false); + expect(isAppUrl("https://example.com/", APP_URL)).toBe(false); + expect(isAppUrl(undefined, APP_URL)).toBe(false); + }); +}); + +describe("decideMainWindowMediaRequest", () => { + it("denies media to frames outside the app shell", () => { + expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny"); + }); + + it("asks macOS for the microphone until the user has granted it", () => { + expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone"); + expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone"); + expect(decide()).toBe("grant"); + }); + + it("grants the microphone directly off macOS", () => { + expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant"); + }); + + it("keeps granting video-only capture such as preview tab recording", () => { + expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant"); + }); +}); diff --git a/apps/desktop/src/window/MainWindowPermissions.ts b/apps/desktop/src/window/MainWindowPermissions.ts new file mode 100644 index 000000000000..5b9ff65e1f65 --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.ts @@ -0,0 +1,63 @@ +import * as Electron from "electron"; + +export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone"; + +/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */ +export function isAppUrl(url: string | undefined, appUrl: string): boolean { + if (!url) return false; + // Custom schemes have an opaque WHATWG origin, so compare the serialized prefix. + const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl; + return url === appOrigin || url.startsWith(`${appOrigin}/`); +} + +/** + * Media requests come from the app shell (voice conversations use the + * microphone; preview tab recording routes `getDisplayMedia` through the same + * permission). Frames from any other origin never get media. On macOS the + * microphone also needs the system's consent, asked for on first use. + */ +export function decideMainWindowMediaRequest(input: { + readonly requestingUrl: string; + readonly appUrl: string; + readonly mediaTypes: ReadonlyArray<"audio" | "video">; + readonly platform: NodeJS.Platform; + readonly microphoneStatus: () => string; +}): MediaPermissionDecision { + if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny"; + if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant"; + return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone"; +} + +/** + * Installs the main window session's permission handlers. Only `media` is + * decided here; every other permission keeps Electron's default of allowing + * it, which is what this session did before it had handlers. + */ +export function installMainWindowPermissionHandlers( + session: Electron.Session, + options: { readonly appUrl: string; readonly platform: NodeJS.Platform }, +): void { + session.setPermissionRequestHandler((_webContents, permission, callback, details) => { + if (permission !== "media") { + callback(true); + return; + } + const decision = decideMainWindowMediaRequest({ + requestingUrl: details.requestingUrl, + appUrl: options.appUrl, + mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [], + platform: options.platform, + microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"), + }); + if (decision !== "ask-microphone") { + callback(decision === "grant"); + return; + } + Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => { + callback(false); + }); + }); + session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) => + permission === "media" ? isAppUrl(requestingOrigin, options.appUrl) : true, + ); +} diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 57717d2d8458..4ee5bd63acbd 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -220,6 +220,10 @@ const sharingPlugin: NonNullable[number] = [ }, ]; +const MICROPHONE_PERMISSION = + "Allow T3 Code to use your microphone for voice input and voice conversations."; +const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes."; + // These aliases match the fonts' PostScript names on iOS. Register the same // names on Android so React Native and the native composer use one set of // family names without waiting for runtime font loading. @@ -310,6 +314,8 @@ const config: ExpoConfig = { foregroundImage: variant.assets.androidAdaptiveForeground, monochromeImage: variant.assets.androidMonochromeIcon, }, + // react-native-webrtc's plugin requests overlay access for screen sharing, which voice does not use. + blockedPermissions: ["android.permission.SYSTEM_ALERT_WINDOW"], // Opts into OnBackInvokedCallback-based back dispatch (Android 13+). // JS back handling survives it via react-native's Android 16 shim plus // withAndroidPredictiveBackCompat on Android 13-15. @@ -382,22 +388,30 @@ const config: ExpoConfig = { [ "expo-audio", { - microphonePermission: "Allow T3 Code to use your microphone for voice input.", - recordAudioAndroid: false, + microphonePermission: MICROPHONE_PERMISSION, + recordAudioAndroid: true, enableBackgroundPlayback: false, enableBackgroundRecording: false, }, ], + // Voice conversations run over WebRTC. Pass both usage strings so its + // plugin cannot fall back to generic ones. + [ + "@config-plugins/react-native-webrtc", + { cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION }, + ], [ "expo-camera", { - cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.", + cameraPermission: CAMERA_PERMISSION, microphonePermission: false, barcodeScannerEnabled: true, recordAudioAndroid: false, }, ], - ["expo-image-picker", { photosPermission: false, microphonePermission: false }], + // `microphonePermission: false` would block RECORD_AUDIO on Android for every + // package, including voice conversations, so share the app's usage string. + ["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }], [ "expo-splash-screen", { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 16eccc711442..b23659ba52e9 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -129,6 +129,7 @@ "react-native-screens": "file:deps/react-native-screens-5.0.0-t3.27.tgz", "react-native-shiki-engine": "^0.3.12", "react-native-svg": "15.15.5", + "react-native-webrtc": "124.0.8", "react-native-webview": "14.0.1", "react-native-worklets": "0.13.0", "remark-gfm": "catalog:", @@ -139,6 +140,7 @@ "uniwind": "1.12.1" }, "devDependencies": { + "@config-plugins/react-native-webrtc": "15.0.2", "@effect/vitest": "catalog:", "@pierre/trees": "1.0.0-beta.6", "@types/mdast": "^4.0.4", diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 64b74b0e9fd5..9fb4e3c37d1a 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -82,6 +82,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar"; import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight"; import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing"; import IconMicrophone from "@tabler/icons-react-native/IconMicrophone"; +import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff"; import IconLink from "@tabler/icons-react-native/IconLink"; import IconListNumbers from "@tabler/icons-react-native/IconListNumbers"; import IconMenu2 from "@tabler/icons-react-native/IconMenu2"; @@ -127,6 +128,7 @@ import IconUserQuestion from "@tabler/icons-react-native/IconUserQuestion"; import IconUserPlus from "@tabler/icons-react-native/IconUserPlus"; import IconUsers from "@tabler/icons-react-native/IconUsers"; import IconWifi from "@tabler/icons-react-native/IconWifi"; +import IconWaveSine from "@tabler/icons-react-native/IconWaveSine"; import IconWifiOff from "@tabler/icons-react-native/IconWifiOff"; import IconWorld from "@tabler/icons-react-native/IconWorld"; import IconX from "@tabler/icons-react-native/IconX"; @@ -221,6 +223,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { macstudio: IconDeviceDesktop, magnifyingglass: IconSearch, mic: IconMicrophone, + "mic.slash": IconMicrophoneOff, paintbrush: IconPalette, pencil: IconPencil, "person.crop.circle": IconUserCircle, @@ -269,6 +272,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "tray.and.arrow.up": IconUpload, trash: IconTrash, wifi: IconWifi, + waveform: IconWaveSine, "wifi.slash": IconWifiOff, xmark: IconX, "xmark.circle.fill": IconCircleXFilled, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index f84ad7cbce01..fe7c1ed15358 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -117,6 +117,9 @@ import { } from "../voice-input/ComposerDictationControl"; import { useVoiceInputController } from "../voice-input/useVoiceInputController"; import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; +import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode"; +import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode"; +import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode"; import { rememberModelOptions, withRememberedModelOptions, @@ -515,15 +518,40 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer onUsageLimits: usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined, }); + const voiceTarget = useMemo( + () => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }), + [props.environmentId, props.selectedThread.id], + ); + const voicePhase = useVoiceModePhase(voiceTarget); + useVoiceModeThreadLifecycle(voiceTarget); + // Dictation and a voice conversation both need the microphone. + const voiceConversationLive = voicePhase !== "idle"; const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, + disabled: voiceConversationLive, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); + const offersVoiceConversation = + voiceModeSupported && selectedProviderStatus?.supportsVoice === true; + // The server attaches voice to the thread's provider session, which the first message creates. + const canToggleVoiceConversation = + voiceConversationLive || + (props.connectionState === "connected" && + props.selectedThread.latestUserMessageAt !== null && + !voiceInput.isBusy); + const toggleVoiceConversation = useCallback(() => voiceMode.toggle(voiceTarget), [voiceTarget]); + const voiceModeButton = offersVoiceConversation ? ( + + ) : null; const voicePresentation = resolveVoiceComposerPresentation( voiceInput.state, voiceInput.elapsedSeconds, @@ -833,6 +861,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer {selectedProviderStatus.compatibilityAdvisory.message} ) : null} + + {voiceConversationLive ? : null} + {modelUnavailable ? ( Model unavailable. Open model settings. @@ -1075,9 +1106,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer ) : null} {!isExpanded ? ( + {voiceModeButton} @@ -1168,10 +1201,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer )} + {isVoiceInputPresented ? null : voiceModeButton} void; +}) { + const live = props.phase !== "idle"; + return ( + + + + + + ); +} + +const LevelBar = memo(function LevelBar(props: { + readonly level: SharedValue; + readonly className: string; +}) { + const { level } = props; + const animatedStyle = useAnimatedStyle(() => ({ + transform: [{ scaleY: 0.15 + level.value * 0.85 }], + })); + return ( + + ); +}); + +function StripButton(props: { + readonly accessibilityLabel: string; + readonly icon: AppSymbolName; + readonly onPress: () => void; +}) { + return ( + + + + ); +} + +/** + * The live conversation's status, levels, controls, and newest captions. + * Render only while this thread's conversation is connecting or active. + */ +export function ComposerVoiceStrip() { + const state = useVoiceModeState(); + const microphoneLevel = useSharedValue(0); + const speakerLevel = useSharedValue(0); + const [speaking, setSpeaking] = useState(false); + + // Levels arrive at 10Hz; they drive shared values directly, and the + // speaking flag only re-renders when it flips. + useEffect(() => { + const isSpeaking = createSpeakingDetector(); + const unsubscribe = voiceMode.subscribeLevels((levels) => { + microphoneLevel.value = withTiming(voiceMeterLevel(levels.microphone), LEVEL_TIMING); + speakerLevel.value = withTiming(voiceMeterLevel(levels.speaker), LEVEL_TIMING); + setSpeaking(isSpeaking(levels.speaker, Date.now())); + }); + return () => { + unsubscribe(); + }; + }, [microphoneLevel, speakerLevel]); + + const status = resolveVoiceStripStatus({ phase: state.phase, muted: state.muted, speaking }); + const captions = state.captions.slice(-VISIBLE_CAPTIONS); + + return ( + + + + + + + + {VOICE_STRIP_STATUS_LABEL[status]} + + voiceMode.toggleMuted()} + /> + voiceMode.stop()} + /> + + {captions.map((caption, index) => ( + + {voiceCaptionTail(caption.text)} + + ))} + + ); +} diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts new file mode 100644 index 000000000000..f0adc61fd6f3 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -0,0 +1,58 @@ +import type { + VoiceModePhase, + VoiceModeState, + VoiceSessionTarget, +} from "@t3tools/client-runtime/voice-mode"; +import { useCallback, useEffect, useSyncExternalStore } from "react"; +import { Alert } from "react-native"; + +import { voiceMode } from "./voiceMode"; +import { resolveVoiceModeNotice } from "./voiceModePresentation"; + +export function useVoiceModeState(): VoiceModeState { + return useSyncExternalStore(voiceMode.subscribe, voiceMode.getState); +} + +/** This thread's conversation phase. A primitive, so captions do not re-render the caller. */ +export function useVoiceModePhase(target: VoiceSessionTarget): VoiceModePhase { + const { environmentId, threadId } = target; + const getSnapshot = useCallback( + () => + voiceMode.isActiveFor({ environmentId, threadId }) ? voiceMode.getState().phase : "idle", + [environmentId, threadId], + ); + return useSyncExternalStore(voiceMode.subscribe, getSnapshot); +} + +/** + * Ties a conversation to the screen showing its thread: leaving the thread + * ends it, so the microphone is never live without its controls on screen. + * Also reports failures and remote endings for this thread, once. + */ +export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { + const { environmentId, threadId } = target; + + useEffect( + () => () => { + if (voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop(); + }, + [environmentId, threadId], + ); + + useEffect(() => { + // Reads the live state: dismissing notifies listeners re-entrantly, so a + // listener's argument can already be stale. + const report = () => { + const state = voiceMode.getState(); + if (state.target?.environmentId !== environmentId || state.target.threadId !== threadId) { + return; + } + const notice = resolveVoiceModeNotice(state); + if (!notice) return; + voiceMode.dismiss(); + Alert.alert(notice.title, notice.message); + }; + report(); + return voiceMode.subscribe(report); + }, [environmentId, threadId]); +} diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts new file mode 100644 index 000000000000..f2eea23b8392 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -0,0 +1,117 @@ +import type { EnvironmentRegistry } from "@t3tools/client-runtime/connection"; +import { + VoiceModeController, + makeVoiceSessionOpener, + type VoiceModePhase, + type VoicePeerConnection, +} from "@t3tools/client-runtime/voice-mode"; +import * as Effect from "effect/Effect"; +import { AtomRegistry } from "effect/unstable/reactivity"; +import { setAudioModeAsync, setIsAudioActiveAsync } from "expo-audio"; +import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { AppState, NativeModules } from "react-native"; +import type * as WebRTC from "react-native-webrtc"; + +import { connectionAtomRuntime } from "../../connection/runtime"; +import { appAtomRegistry } from "../../state/atom-registry"; + +/** + * False on binaries built before react-native-webrtc was added. Importing the + * package there throws, so it only loads once a conversation starts. + */ +export const voiceModeSupported: boolean = NativeModules.WebRTCModule != null; + +let webrtc: typeof WebRTC | null = null; +function loadWebRTC(): typeof WebRTC { + webrtc ??= require("react-native-webrtc") as typeof WebRTC; + return webrtc; +} + +/** Runs the voice stream on the connection runtime, holding it for the conversation's lifetime. */ +function runOnConnectionRuntime(effect: Effect.Effect) { + return Effect.runFork( + Effect.gen(function* () { + yield* AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime); + const context = yield* AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime); + yield* Effect.provideContext(effect, context); + }).pipe(Effect.scoped, Effect.orDie), + ); +} + +/** + * react-native-webrtc's published declarations drop its EventTarget base + * (`lib/typescript` omits the vendored event-target-shim types), so the class + * types without `addEventListener`. The runtime class extends the shim and + * supports it, which is all the controller uses. + */ +function createPeerConnection(): VoicePeerConnection { + return new (loadWebRTC().RTCPeerConnection)() as unknown as VoicePeerConnection; +} + +export const voiceMode = new VoiceModeController({ + createPeerConnection, + getMicrophone: () => loadWebRTC().mediaDevices.getUserMedia({ audio: true }), + // Native WebRTC plays remote audio tracks through its own audio device; there is nothing to attach. + playRemoteAudio: () => () => undefined, + openSession: makeVoiceSessionOpener(runOnConnectionRuntime), +}); + +/* + * WebRTC's iOS audio session uses voice-chat mode, which plays through the + * earpiece. Once the call is live, re-apply play-and-record with the speaker + * as the default route (headphones and Bluetooth still win). WebRTC only sets + * its category when its audio unit starts, which has happened by then. On + * Android the same call turns the speakerphone on. + */ +async function routeVoiceToSpeaker(): Promise { + await setAudioModeAsync({ + allowsRecording: true, + playsInSilentMode: true, + interruptionMode: "doNotMix", + shouldPlayInBackground: false, + shouldRouteThroughEarpiece: false, + }); +} + +/** Mirrors dictation's release so interrupted audio in other apps resumes. */ +async function releaseVoiceAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + await setIsAudioActiveAsync(false); + } +} + +const KEEP_AWAKE_TAG = "voice-mode"; +let previousPhase: VoiceModePhase = "idle"; +let routedAudio = false; +let keepAwake: Promise | null = null; + +voiceMode.subscribe(({ phase }) => { + if (phase === previousPhase) return; + previousPhase = phase; + if (phase === "active" && !routedAudio) { + routedAudio = true; + routeVoiceToSpeaker().catch(() => undefined); + } + if (phase !== "idle" && !keepAwake) { + // A hands-free conversation must not auto-lock: backgrounding ends it. + keepAwake = activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch(() => undefined); + } + if (phase === "idle") { + if (routedAudio) { + routedAudio = false; + releaseVoiceAudio().catch(() => undefined); + } + const activation = keepAwake; + keepAwake = null; + activation?.then(() => deactivateKeepAwake(KEEP_AWAKE_TAG)).catch(() => undefined); + } +}); + +// iOS cuts the microphone in the background (the app has no background audio +// mode), and Android restricts background capture without a foreground +// service. End the conversation rather than leave it silently deaf. +AppState.addEventListener("change", (state) => { + if (state === "background") voiceMode.stop(); +}); diff --git a/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts b/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts new file mode 100644 index 000000000000..de2db045ec22 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vite-plus/test"; +import { INITIAL_VOICE_MODE_STATE } from "@t3tools/client-runtime/voice-mode"; + +import { + createSpeakingDetector, + resolveVoiceModeNotice, + resolveVoiceStripStatus, + voiceCaptionTail, +} from "./voiceModePresentation"; + +describe("resolveVoiceStripStatus", () => { + it("shows connecting until the conversation is live, then prefers the assistant's voice", () => { + expect(resolveVoiceStripStatus({ phase: "connecting", muted: true, speaking: true })).toBe( + "connecting", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: true, speaking: true })).toBe( + "speaking", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: true, speaking: false })).toBe( + "muted", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: false, speaking: false })).toBe( + "listening", + ); + }); +}); + +describe("createSpeakingDetector", () => { + it("holds through short pauses and releases after sustained silence", () => { + const isSpeaking = createSpeakingDetector(); + expect(isSpeaking(0, 0)).toBe(false); + expect(isSpeaking(0.3, 100)).toBe(true); + expect(isSpeaking(0, 300)).toBe(true); + expect(isSpeaking(0.001, 600)).toBe(true); + expect(isSpeaking(0, 700)).toBe(false); + }); +}); + +describe("voiceCaptionTail", () => { + it("keeps the newest words of long captions", () => { + expect(voiceCaptionTail("short")).toBe("short"); + expect(voiceCaptionTail("one two three four", 9)).toBe("…hree four"); + }); +}); + +describe("resolveVoiceModeNotice", () => { + const target = { environmentId: "env", threadId: "thread" }; + + it("stays quiet while live and after the user stops", () => { + expect(resolveVoiceModeNotice(INITIAL_VOICE_MODE_STATE)).toBeNull(); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, phase: "active", target }), + ).toBeNull(); + }); + + it("reports failures and endings the user did not choose", () => { + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, error: "Mic denied" }), + ).toEqual({ title: "Voice conversation failed", message: "Mic denied" }); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, endedReason: "ended" }), + ).toEqual({ title: "Voice conversation ended" }); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, endedReason: "replaced" }), + ).toEqual({ + title: "Voice conversation ended", + message: "Voice conversation moved to another window or device.", + }); + }); +}); diff --git a/apps/mobile/src/features/voice-mode/voiceModePresentation.ts b/apps/mobile/src/features/voice-mode/voiceModePresentation.ts new file mode 100644 index 000000000000..118d9257b1ce --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceModePresentation.ts @@ -0,0 +1,68 @@ +import { + describeVoiceEnd, + type VoiceModePhase, + type VoiceModeState, +} from "@t3tools/client-runtime/voice-mode"; + +export type VoiceStripStatus = "connecting" | "listening" | "speaking" | "muted"; + +export const VOICE_STRIP_STATUS_LABEL: Record = { + connecting: "Connecting…", + listening: "Listening", + speaking: "Speaking", + muted: "Muted", +}; + +/** The assistant's voice outranks mute: while it talks, that is what the user hears. */ +export function resolveVoiceStripStatus(input: { + readonly phase: VoiceModePhase; + readonly muted: boolean; + readonly speaking: boolean; +}): VoiceStripStatus { + if (input.phase !== "active") return "connecting"; + if (input.speaking) return "speaking"; + return input.muted ? "muted" : "listening"; +} + +const SPEAKING_LEVEL = 0.02; +const SPEAKING_HOLD_MS = 600; + +/** + * Turns sampled speaker levels into a steady "speaking" flag. The flag holds + * through the short gaps between words so the status label does not flicker. + */ +export function createSpeakingDetector(): (level: number, nowMs: number) => boolean { + let lastLoudAtMs: number | null = null; + return (level, nowMs) => { + if (level >= SPEAKING_LEVEL) lastLoudAtMs = nowMs; + return lastLoudAtMs !== null && nowMs - lastLoudAtMs < SPEAKING_HOLD_MS; + }; +} + +/** Maps a linear 0..1 audio level to meter height; speech sits low on the linear scale. */ +export function voiceMeterLevel(level: number): number { + return Math.min(1, Math.sqrt(Math.max(0, level)) * 1.6); +} + +/** Keeps the newest words of a live caption; native text only truncates multi-line text at the end. */ +export function voiceCaptionTail(text: string, maxChars = 160): string { + if (text.length <= maxChars) return text; + return `…${text.slice(-maxChars).trimStart()}`; +} + +export interface VoiceModeNotice { + readonly title: string; + readonly message?: string; +} + +/** What to tell the user about a finished conversation. Null when they ended it themselves. */ +export function resolveVoiceModeNotice(state: VoiceModeState): VoiceModeNotice | null { + if (state.phase !== "idle") return null; + if (state.error) return { title: "Voice conversation failed", message: state.error }; + if (state.endedReason) { + return state.endedReason === "ended" + ? { title: "Voice conversation ended" } + : { title: "Voice conversation ended", message: describeVoiceEnd(state.endedReason) }; + } + return null; +} diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 5b8d483dc102..23088cc8459b 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -169,6 +169,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.mcpAppsToolInfo]: AuthOrchestrationReadScope, [WS_METHODS.mcpAppsReadResource]: AuthOrchestrationReadScope, [WS_METHODS.mcpAppsUpdateModelContext]: AuthOrchestrationOperateScope, + [WS_METHODS.providerVoiceSession]: AuthOrchestrationOperateScope, [WS_METHODS.subscribeVcsStatus]: AuthOrchestrationReadScope, [WS_METHODS.subscribeWorktreeSetup]: AuthOrchestrationReadScope, [WS_METHODS.worktreeSetupCancel]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index d8ef1461ef73..5780acc8fc93 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -113,6 +113,7 @@ import { codexAppServerArgs, resolveCodexLaunchArgs } from "../../provider/codex import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; +import { makeCodexVoice, readRealtimeDelegationFromItem } from "../../provider/CodexVoice.ts"; import { MCP_APP_EXTENSION_ID, MCP_APP_MIME_TYPE, @@ -1876,6 +1877,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // terminal, otherwise the root terminal can strand child projections. const deferredRootTerminals = yield* Ref.make(new Map()); const offeredContinuationItemsByTurn = yield* Ref.make(new Map>()); + const sessionScope = yield* Effect.scope; + const voice = yield* makeCodexVoice(client, sessionScope); + /** The provider thread holding the voice conversation, while one runs. */ + const voiceProviderThread = yield* Ref.make( + undefined, + ); + /** Native turns Codex started for a voice handoff -> the spoken request. */ + const voiceResubmits = yield* Ref.make(new Map()); const finalAnswerItemIdsByTurn = yield* Ref.make(new Map>()); const completedFinalAnswerTextsByTurn = yield* Ref.make(new Map>()); // Native completion and the interrupt timeout share one finalization @@ -4608,6 +4617,27 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("item/started", (payload) => Effect.gen(function* () { + if (payload.item.type === "userMessage") { + const handoff = readRealtimeDelegationFromItem(payload.item); + const voiceThread = yield* Ref.get(voiceProviderThread); + if ( + handoff !== undefined && + voiceThread?.nativeThreadRef?.nativeId === payload.threadId && + !(yield* Ref.get(activeTurns)).has(payload.turnId) && + !(yield* Ref.get(pendingRootTurns)).has(payload.threadId) + ) { + // Codex started this turn itself for a voice handoff. V2 runs + // are orchestrator-owned, so stop it and resubmit the request + // as an ordinary run once it settles (see turn/completed). + yield* Ref.update(voiceResubmits, (current) => + new Map(current).set(payload.turnId, handoff), + ); + yield* client + .request("turn/interrupt", { threadId: payload.threadId, turnId: payload.turnId }) + .pipe(Effect.ignore, Effect.forkIn(sessionScope)); + return; + } + } const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { return; @@ -5974,6 +6004,27 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { + const voiceResubmit = (yield* Ref.get(voiceResubmits)).get(payload.turn.id); + if (voiceResubmit !== undefined) { + yield* Ref.update(voiceResubmits, (current) => { + const updated = new Map(current); + updated.delete(payload.turn.id); + return updated; + }); + const voiceThread = yield* Ref.get(voiceProviderThread); + if (voiceThread !== undefined && continuationRequests !== undefined) { + yield* voice.expectSpokenTurn(voiceResubmit); + yield* continuationRequests.offer({ + threadId: voiceThread.appThreadId ?? input.threadId, + providerThreadId: voiceThread.id, + driver: CODEX_PROVIDER, + detail: voiceResubmit, + delivery: "message_text", + origin: "voice", + }); + } + return; + } const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); if (context === undefined) { return; @@ -7135,6 +7186,33 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ), ), + startVoiceSession: (voiceInput) => + Stream.unwrap( + Effect.gen(function* () { + const nativeThreadId = yield* getNativeThreadId(voiceInput.providerThread); + yield* ensureInitialized; + yield* Ref.set(voiceProviderThread, voiceInput.providerThread); + return voice.start({ + providerThreadId: nativeThreadId, + offerSdp: voiceInput.offerSdp, + ...(adapterOptions.settings.voice + ? { voice: adapterOptions.settings.voice } + : {}), + }); + }), + ).pipe( + Stream.ensuring(Ref.set(voiceProviderThread, undefined)), + Stream.mapError((cause) => + cause._tag === "ProviderAdapterProtocolError" + ? cause + : new ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: + cause._tag === "CodexVoiceSessionError" ? cause.detail : cause.message, + payload: cause, + }), + ), + ), uploadFeedback: (feedbackInput) => Effect.gen(function* () { const threadId = yield* getNativeThreadId(feedbackInput.providerThread); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 689c66a1d402..b6723cd3cd7b 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -244,6 +244,38 @@ describe("ProviderContinuationService", () => { }); }); + it.effect("resubmits a spoken voice request as the user's own message", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + yield* requests.offer({ + threadId, + providerThreadId, + driver, + detail: "run the tests", + delivery: "message_text", + origin: "voice", + }); + const command = (yield* Queue.take(dispatched)) as { + readonly createdBy: string; + readonly creationSource: string; + readonly text: string; + readonly notification?: OrchestrationV2Notification; + }; + assert.equal(command.createdBy, "user"); + assert.equal(command.creationSource, "server"); + assert.equal(command.text, "run the tests"); + assert.isUndefined(command.notification); + }).pipe( + Effect.provide( + testLayer({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + it.effect("dispatches a current delegated completion as one server-owned queued message", () => { return Effect.gen(function* () { const dispatched = yield* Queue.unbounded(); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index c0b0c55b7221..55c9036aafb2 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -209,26 +209,40 @@ export const layer = Layer.effectDiscard( ordinal: projection.messages.length + 1, })); const commandId = CommandId.make(`provider-continuation:${messageId}`); - const dispatch = threads.dispatch({ - type: "message.dispatch", - commandId, - threadId: request.threadId, - messageId, - text: request.detail ?? CONTINUATION_MESSAGE_TEXT, - notification: request.notification ?? { - source: { kind: "background_task" }, - outcome: "updated", - summary: "Background activity updated", - }, - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: "agent", - // "provider" marks an adapter-buffered wake, which ClaudeAdapterV2 - // detects to attach the buffered CLI output and drop this text. A - // message_text wake has no buffered output, so it must not carry that - // marker or the turn settles immediately having prompted nothing. - creationSource: request.delivery === "message_text" ? "server" : "provider", - }); + const dispatch = threads.dispatch( + request.origin === "voice" + ? { + type: "message.dispatch", + commandId, + threadId: request.threadId, + messageId, + text: request.detail ?? "", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "server", + } + : { + type: "message.dispatch", + commandId, + threadId: request.threadId, + messageId, + text: request.detail ?? CONTINUATION_MESSAGE_TEXT, + notification: request.notification ?? { + source: { kind: "background_task" }, + outcome: "updated", + summary: "Background activity updated", + }, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + // "provider" marks an adapter-buffered wake, which ClaudeAdapterV2 + // detects to attach the buffered CLI output and drop this text. A + // message_text wake has no buffered output, so it must not carry that + // marker or the turn settles immediately having prompted nothing. + creationSource: request.delivery === "message_text" ? "server" : "provider", + }, + ); // The adapter's buffered wake has nowhere else to go once the // instance is off: drop it rather than warn, since nothing here // retries it either way. This catch sits outside dispatchIfCurrent so diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 9ed238bf5d2c..2cfeed17fc8d 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -77,6 +77,7 @@ const CODEX_PRESENTATION = { displayName: "Codex", showInteractionModeToggle: true, reportsContextWindow: true, + supportsVoice: true, } as const; export interface CodexAppServerProviderSnapshot { diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts new file mode 100644 index 000000000000..528db80bd354 --- /dev/null +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -0,0 +1,273 @@ +import * as NodeAssert from "node:assert/strict"; + +import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; +import { describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import type * as CodexClient from "effect-codex-app-server/client"; + +import { + CodexVoiceSessionError, + makeCodexVoice, + readRealtimeDelegationInput, + readSpeakableAnswer, +} from "./CodexVoice.ts"; + +const PROVIDER_THREAD = "provider-thread-1"; + +function delegation(input: string) { + return `\n ${input}\n`; +} + +const makeFakeClient = Effect.gen(function* () { + const handlers = new Map Effect.Effect>>(); + const requests: Array<{ readonly method: string; readonly params: unknown }> = []; + const startRequested = yield* Deferred.make(); + const speechRequested = yield* Deferred.make(); + const stopRequested = yield* Deferred.make(); + const client = { + handleServerNotification: ( + method: string, + handler: (payload: unknown) => Effect.Effect, + ) => + Effect.sync(() => { + handlers.set(method, [...(handlers.get(method) ?? []), handler]); + }), + raw: { + request: (method: string, params: unknown) => + Effect.gen(function* () { + requests.push({ method, params }); + if (method === "thread/realtime/start") + yield* Deferred.succeed(startRequested, undefined); + if (method === "thread/realtime/appendSpeech") { + yield* Deferred.succeed(speechRequested, undefined); + } + if (method === "thread/realtime/stop") yield* Deferred.succeed(stopRequested, undefined); + return {}; + }), + }, + } as unknown as CodexClient.CodexAppServerClient["Service"]; + const emit = (method: string, payload: unknown) => + Effect.forEach(handlers.get(method) ?? [], (handler) => Effect.ignore(handler(payload)), { + discard: true, + }); + return { client, requests, emit, startRequested, speechRequested, stopRequested }; +}); + +const startVoice = Effect.gen(function* () { + const fake = yield* makeFakeClient; + const scope = yield* Scope.make(); + const voice = yield* makeCodexVoice(fake.client, scope); + const fiber = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "v=offer\r\n", voice: "cove" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + return { ...fake, fiber }; +}); + +describe("readRealtimeDelegationInput", () => { + it("unwraps and unescapes the spoken request", () => { + NodeAssert.equal( + readRealtimeDelegationInput(delegation("run the tests & fix <Foo>")), + "run the tests & fix ", + ); + }); + + it("ignores ordinary user messages", () => { + NodeAssert.equal(readRealtimeDelegationInput("please fix the build"), undefined); + }); +}); + +describe("readSpeakableAnswer", () => { + it("speaks final answers without the channel marker", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "[FINAL] All green.", phase: null }), + "All green.", + ); + }); + + it("keeps commentary private", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "Looking…", phase: "commentary" }), + undefined, + ); + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "[COMMENTARY] Looking…", phase: null }), + undefined, + ); + }); + + it("leaves oversized answers on screen only", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "x".repeat(5000), phase: "final_answer" }), + undefined, + ); + }); +}); + +describe("makeCodexVoice", () => { + it.effect("starts a WebRTC realtime session and relays signaling until Codex closes it", () => + Effect.gen(function* () { + const voice = yield* startVoice; + NodeAssert.deepEqual(voice.requests[0], { + method: "thread/realtime/start", + params: { + threadId: PROVIDER_THREAD, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + voice: "cove", + }, + }); + + yield* voice.emit("thread/realtime/started", { threadId: PROVIDER_THREAD, version: "v3" }); + yield* voice.emit("thread/realtime/sdp", { threadId: PROVIDER_THREAD, sdp: "v=answer\r\n" }); + yield* voice.emit("thread/realtime/transcript/delta", { + threadId: "another-thread", + role: "user", + delta: "ignored", + }); + yield* voice.emit("thread/realtime/transcript/delta", { + threadId: PROVIDER_THREAD, + role: "user", + delta: "hello", + }); + yield* voice.emit("thread/realtime/transcript/done", { + threadId: PROVIDER_THREAD, + role: "assistant", + text: "Hi there", + }); + yield* voice.emit("thread/realtime/closed", { + threadId: PROVIDER_THREAD, + reason: "transport_closed", + }); + + const events = yield* Fiber.join(voice.fiber); + NodeAssert.deepEqual(events, [ + { type: "started" }, + { type: "answer", sdp: "v=answer\r\n" }, + { type: "transcript.delta", role: "user", delta: "hello" }, + { type: "transcript.done", role: "assistant", text: "Hi there" }, + { type: "closed", reason: "transport_closed" }, + ] satisfies ReadonlyArray); + // Codex already closed the session, so there is nothing to stop. + NodeAssert.ok(!voice.requests.some((request) => request.method === "thread/realtime/stop")); + }), + ); + + it.effect("stops the realtime session when the client unsubscribes", () => + Effect.gen(function* () { + const voice = yield* startVoice; + yield* Fiber.interrupt(voice.fiber); + yield* Deferred.await(voice.stopRequested); + NodeAssert.deepEqual(voice.requests.at(-1), { + method: "thread/realtime/stop", + params: { threadId: PROVIDER_THREAD }, + }); + }), + ); + + it.effect("fails the stream with Codex's realtime error", () => + Effect.gen(function* () { + const voice = yield* startVoice; + yield* voice.emit("thread/realtime/error", { + threadId: PROVIDER_THREAD, + message: "realtime unavailable", + }); + const exit = yield* Fiber.await(voice.fiber); + NodeAssert.ok(Exit.isFailure(exit)); + const error = exit.cause.reasons[0]; + NodeAssert.ok(error?._tag === "Fail" && error.error instanceof CodexVoiceSessionError); + NodeAssert.equal(error.error.detail, "realtime unavailable"); + }), + ); + + it.effect("speaks the answer of a handoff the host resubmitted as its own turn", () => + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const scope = yield* Scope.make(); + const voice = yield* makeCodexVoice(fake.client, scope); + const fiber = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "v=offer\r\n" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + + yield* voice.expectSpokenTurn("run the tests"); + yield* fake.emit("item/started", { + threadId: PROVIDER_THREAD, + turnId: "resubmitted", + item: { + type: "userMessage", + id: "user-resubmitted", + content: [{ type: "text", text: "\nrun the tests" }], + }, + }); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "resubmitted", + item: { type: "agentMessage", id: "answer", text: "All green.", phase: "final_answer" }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "resubmitted", status: "completed" }, + }); + + yield* Deferred.await(fake.speechRequested); + NodeAssert.deepEqual(fake.requests.at(-1), { + method: "thread/realtime/appendSpeech", + params: { threadId: PROVIDER_THREAD, text: "All green." }, + }); + yield* Fiber.interrupt(fiber); + }), + ); + + it.effect("speaks the final answer of a voice-delegated turn, not typed turns", () => + Effect.gen(function* () { + const voice = yield* startVoice; + const userMessage = (turnId: string, text: string) => + voice.emit("item/started", { + threadId: PROVIDER_THREAD, + turnId, + item: { type: "userMessage", id: `user-${turnId}`, content: [{ type: "text", text }] }, + }); + const agentMessage = (turnId: string, text: string, phase: string) => + voice.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId, + item: { type: "agentMessage", id: `agent-${turnId}-${phase}`, text, phase }, + }); + const complete = (turnId: string) => + voice.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: turnId, status: "completed" }, + }); + + yield* userMessage("typed", "fix the build"); + yield* agentMessage("typed", "Fixed it.", "final_answer"); + yield* complete("typed"); + + yield* userMessage("voice", delegation("run the tests")); + yield* agentMessage("voice", "Running them now.", "commentary"); + yield* agentMessage("voice", "All 42 tests pass.", "final_answer"); + yield* complete("voice"); + + yield* Deferred.await(voice.speechRequested); + NodeAssert.deepEqual( + voice.requests.filter((request) => request.method === "thread/realtime/appendSpeech"), + [ + { + method: "thread/realtime/appendSpeech", + params: { threadId: PROVIDER_THREAD, text: "All 42 tests pass." }, + }, + ], + ); + yield* Fiber.interrupt(voice.fiber); + }), + ); +}); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts new file mode 100644 index 000000000000..f43ae817aa3c --- /dev/null +++ b/apps/server/src/provider/CodexVoice.ts @@ -0,0 +1,292 @@ +/** + * Codex voice conversations: a thread-scoped realtime session whose audio runs + * over WebRTC between the client and OpenAI. The app-server only relays the + * SDP handshake, captions, and lifecycle; this module owns that signaling. + * + * Handoffs are client-managed, matching the Codex TUI: the realtime model + * delegates work by starting a Codex turn whose user message is wrapped in + * ``, and when that turn finishes we hand its final + * answer back with `thread/realtime/appendSpeech` so the voice reads it out. + */ +import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import type * as CodexClient from "effect-codex-app-server/client"; +import type * as CodexErrors from "effect-codex-app-server/errors"; + +const DELEGATION_OPEN = ""; +const DELEGATION_CLOSE = ""; +// Codex rejects speech over 1,000 tokens and may add its own prefix. +const MAX_SPEAKABLE_CHARS = 990 * 4; + +function unescapeXmlText(text: string): string { + return text.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); +} + +/** The spoken request inside a Codex realtime delegation message, if `text` is one. */ +export function readRealtimeDelegationInput(text: string): string | undefined { + const trimmed = text.trim(); + if (!trimmed.startsWith(DELEGATION_OPEN) || !trimmed.endsWith(DELEGATION_CLOSE)) { + return undefined; + } + const match = /([\s\S]*?)<\/input>/.exec(trimmed); + const input = match?.[1] ? unescapeXmlText(match[1]).trim() : ""; + return input.length > 0 ? input : undefined; +} + +function readUserMessageText(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; content?: unknown }; + if (record.type !== "userMessage" || !Array.isArray(record.content)) return undefined; + return (record.content as ReadonlyArray<{ type?: unknown; text?: unknown }>) + .flatMap((entry) => + entry.type === "text" && typeof entry.text === "string" ? [entry.text] : [], + ) + .join("\n"); +} + +/** Reads the delegation input from a Codex `userMessage` item. */ +export function readRealtimeDelegationFromItem(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; content?: unknown }; + if (record.type !== "userMessage" || !Array.isArray(record.content)) return undefined; + const [only, ...rest] = record.content as ReadonlyArray<{ type?: unknown; text?: unknown }>; + if (rest.length > 0 || only?.type !== "text" || typeof only.text !== "string") return undefined; + return readRealtimeDelegationInput(only.text); +} + +/** + * Text worth speaking from a completed `agentMessage`. Commentary stays private + * to the voice handoff, and oversized answers stay on screen only. + */ +export function readSpeakableAnswer(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; text?: unknown; phase?: unknown; questions?: unknown }; + if (record.type !== "agentMessage" || typeof record.text !== "string") return undefined; + if (record.phase === "commentary" || (Array.isArray(record.questions) && record.questions.length)) + return undefined; + const text = record.text.trim(); + if ( + record.phase !== "final_answer" && + (text.startsWith("[ANALYSIS]") || text.startsWith("[COMMENTARY]")) + ) { + return undefined; + } + const spoken = text.replace(/^\[FINAL\]\s*/, ""); + if (spoken.length === 0 || spoken.length > MAX_SPEAKABLE_CHARS) return undefined; + return spoken; +} + +function transcriptRole(role: string): "user" | "assistant" { + return role === "user" ? "user" : "assistant"; +} + +export class CodexVoiceSessionError extends Data.TaggedError("CodexVoiceSessionError")<{ + readonly detail: string; +}> { + override get message(): string { + return this.detail; + } +} + +type VoiceQueue = Queue.Queue; + +interface ActiveVoice { + readonly providerThreadId: string; + readonly queue: VoiceQueue; + /** Delegated turn id -> latest speakable answer in that turn. */ + readonly delegatedTurns: Map; + /** Handoffs resubmitted as ordinary runs, awaiting their native turn. */ + readonly expectedHandoffs: Array; + closed: boolean; +} + +export interface CodexVoiceStartInput { + readonly providerThreadId: string; + readonly offerSdp: string; + readonly voice?: string | undefined; +} + +export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( + client: CodexClient.CodexAppServerClient["Service"], + /** Outlives notification handlers, which must not block on requests. */ + runtimeScope: Scope.Scope, +) { + const activeRef = yield* Ref.make(undefined); + + const withActive = ( + threadId: string, + f: (active: ActiveVoice) => Effect.Effect, + ): Effect.Effect => + Effect.flatMap(Ref.get(activeRef), (active) => + active && !active.closed && active.providerThreadId === threadId ? f(active) : Effect.void, + ); + + const offer = (active: ActiveVoice, event: ProviderVoiceSessionEvent) => + Queue.offer(active.queue, event).pipe(Effect.asVoid); + + const finish = (active: ActiveVoice, event: ProviderVoiceSessionEvent) => + Effect.suspend(() => { + active.closed = true; + return offer(active, event).pipe(Effect.andThen(Queue.end(active.queue)), Effect.asVoid); + }); + + yield* client.handleServerNotification("thread/realtime/started", (payload) => + withActive(payload.threadId, (active) => offer(active, { type: "started" })), + ); + yield* client.handleServerNotification("thread/realtime/sdp", (payload) => + withActive(payload.threadId, (active) => offer(active, { type: "answer", sdp: payload.sdp })), + ); + yield* client.handleServerNotification("thread/realtime/transcript/delta", (payload) => + withActive(payload.threadId, (active) => + offer(active, { + type: "transcript.delta", + role: transcriptRole(payload.role), + delta: payload.delta, + }), + ), + ); + yield* client.handleServerNotification("thread/realtime/transcript/done", (payload) => + withActive(payload.threadId, (active) => + offer(active, { + type: "transcript.done", + role: transcriptRole(payload.role), + text: payload.text, + }), + ), + ); + yield* client.handleServerNotification("thread/realtime/error", (payload) => + withActive(payload.threadId, (active) => + Effect.suspend(() => { + active.closed = true; + return Queue.fail(active.queue, new CodexVoiceSessionError({ detail: payload.message })); + }).pipe(Effect.asVoid), + ), + ); + yield* client.handleServerNotification("thread/realtime/closed", (payload) => + withActive(payload.threadId, (active) => + finish(active, { type: "closed", ...(payload.reason ? { reason: payload.reason } : {}) }), + ), + ); + + yield* client.handleServerNotification("item/started", (payload) => + withActive(payload.threadId, (active) => + Effect.sync(() => { + if (readRealtimeDelegationFromItem(payload.item) !== undefined) { + active.delegatedTurns.set(payload.turnId, undefined); + return; + } + const text = readUserMessageText(payload.item); + const expectedIndex = + text === undefined + ? -1 + : active.expectedHandoffs.findIndex((expected) => text.includes(expected)); + if (expectedIndex !== -1) { + active.expectedHandoffs.splice(expectedIndex, 1); + active.delegatedTurns.set(payload.turnId, undefined); + } + }), + ), + ); + yield* client.handleServerNotification("item/completed", (payload) => + withActive(payload.threadId, (active) => + Effect.sync(() => { + if (!active.delegatedTurns.has(payload.turnId)) return; + const answer = readSpeakableAnswer(payload.item); + if (answer !== undefined) active.delegatedTurns.set(payload.turnId, answer); + }), + ), + ); + yield* client.handleServerNotification("turn/completed", (payload) => + withActive(payload.threadId, (active) => + Effect.suspend(() => { + const answer = active.delegatedTurns.get(payload.turn.id); + active.delegatedTurns.delete(payload.turn.id); + if (answer === undefined || payload.turn.status !== "completed") return Effect.void; + return client.raw + .request("thread/realtime/appendSpeech", { + threadId: active.providerThreadId, + text: answer, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Codex voice could not speak the delegated answer", { cause }), + ), + Effect.forkIn(runtimeScope), + Effect.asVoid, + ); + }), + ), + ); + + const stop = (active: ActiveVoice) => + Effect.gen(function* () { + yield* Ref.update(activeRef, (current) => (current === active ? undefined : current)); + if (active.closed) return; + active.closed = true; + yield* client.raw + .request("thread/realtime/stop", { threadId: active.providerThreadId }) + .pipe(Effect.ignore); + }); + + /** + * Starts a voice conversation. The stream ends when Codex closes the + * session; interrupting it (client unsubscribe or disconnect) stops it. + */ + const start = ( + input: CodexVoiceStartInput, + ): Stream.Stream< + ProviderVoiceSessionEvent, + CodexVoiceSessionError | CodexErrors.CodexAppServerError + > => + Stream.unwrap( + Effect.gen(function* () { + const queue: VoiceQueue = yield* Queue.unbounded< + ProviderVoiceSessionEvent, + CodexVoiceSessionError | Cause.Done + >(); + const active: ActiveVoice = { + providerThreadId: input.providerThreadId, + queue, + delegatedTurns: new Map(), + expectedHandoffs: [], + closed: false, + }; + const previous = yield* Ref.getAndSet(activeRef, active); + if (previous && !previous.closed) { + // One conversation per thread: a newer client takes over. + yield* finish(previous, { type: "closed", reason: "replaced" }); + yield* client.raw + .request("thread/realtime/stop", { threadId: previous.providerThreadId }) + .pipe(Effect.ignore); + } + yield* Effect.addFinalizer(() => stop(active)); + yield* client.raw.request("thread/realtime/start", { + threadId: input.providerThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: input.offerSdp }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + ...(input.voice ? { voice: input.voice } : {}), + }); + return Stream.fromQueue(queue); + }), + ); + + /** + * Marks the next turn whose user message carries `text` as a voice handoff, + * for hosts that resubmit Codex's delegation as their own run. + */ + const expectSpokenTurn = (text: string) => + Effect.map(Ref.get(activeRef), (active) => { + if (active && !active.closed) active.expectedHandoffs.push(text); + }); + + return { start, expectSpokenTurn }; +}); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index cc400c170f4a..16c6aef73e66 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -115,6 +115,7 @@ const makeCodexConfig = (overrides: Partial): CodexSettings => ({ shadowHomePath: "", launchArgs: "", customModels: [], + voice: "", ...overrides, }); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 375c2ac78aa1..0093f16f0b77 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -74,6 +74,7 @@ import { ProjectWriteFileError, ProjectMutationError, ProviderUploadFeedbackError, + ProviderVoiceSessionError, ProviderSetupError, RelayClientInstallFailedError, type RelayClientInstallProgressEvent, @@ -2307,6 +2308,53 @@ const layerWsRpc = ( }), ), ), + [WS_METHODS.providerVoiceSession]: (input) => + observeRpcStream( + WS_METHODS.providerVoiceSession, + Stream.unwrap( + Effect.gen(function* () { + const voiceError = (detail: string) => + new ProviderVoiceSessionError({ threadId: input.threadId, detail }); + const projection = yield* threadManagement.getThreadRecords(input.threadId, [ + "providerThreads", + ]); + const providerThread = + projection.providerThreads.find( + (candidate) => candidate.id === projection.thread.activeProviderThreadId, + ) ?? projection.providerThreads.at(-1); + const providerSessionId = providerThread?.providerSessionId ?? null; + if (providerThread === undefined || providerSessionId === null) { + return yield* voiceError("Send a message before starting voice."); + } + const runtime = Option.getOrNull(yield* providerSessionsV2.get(providerSessionId)); + if (runtime === null) { + return yield* voiceError( + "The provider session is no longer running. Send a message first.", + ); + } + if (runtime.startVoiceSession === undefined) { + return yield* voiceError( + `Provider '${runtime.driver}' does not support voice conversations.`, + ); + } + return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp }); + }), + ).pipe( + Stream.mapError((cause) => + cause._tag === "ProviderVoiceSessionError" + ? cause + : new ProviderVoiceSessionError({ + threadId: input.threadId, + detail: + "detail" in cause && typeof cause.detail === "string" + ? cause.detail + : cause.message, + cause, + }), + ), + ), + { "rpc.aggregate": "provider" }, + ), [WS_METHODS.serverUpdateProvider]: (input) => providerMaintenanceRunner.updateProvider(input), [WS_METHODS.providerConsumeResetCredit]: (input) => diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index b760810a3876..06d091c41f63 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -187,6 +187,8 @@ import { collapseExpandedComposerCursor, parseStandaloneComposerSlashCommand, } from "../composer-logic"; +import { resolveThreadVoiceAvailability } from "../voice/voiceAvailability"; +import { toggleThreadVoice } from "../voice/voiceMode"; import { derivePendingApprovals, derivePendingUserInputs, @@ -9229,18 +9231,31 @@ export default function ChatView(props: ChatViewProps) { } return; } - // Providers without the legacy toggle receive their native commands unchanged. - const standaloneSlashCommand = - sendInteractionModeEnabled && + const parsedSlashCommand = composerImages.length === 0 && composerFiles.length === 0 && sendableComposerTerminalContexts.length === 0 && composerPreviewAnnotations.length === 0 && composerReviewComments.length === 0 && - composerThreadContexts.length === 0 + composerThreadContexts.length === 0 && + multipleModelSelections === null ? parseStandaloneComposerSlashCommand(trimmed) : null; - if (standaloneSlashCommand && multipleModelSelections === null) { + if (parsedSlashCommand === "voice") { + // Providers without voice receive "/voice" as a normal message. + const voiceAvailability = resolveThreadVoiceAvailability(activeThread, providerStatuses); + if (voiceAvailability.kind !== "unsupported") { + toggleThreadVoice(routeThreadRef, voiceAvailability); + promptRef.current = ""; + clearComposerDraftContent(composerDraftTarget); + composerRef.current?.resetCursorState(); + return; + } + } + // Providers without the legacy toggle receive their native commands unchanged. + const standaloneSlashCommand = + sendInteractionModeEnabled && parsedSlashCommand !== "voice" ? parsedSlashCommand : null; + if (standaloneSlashCommand) { handleInteractionModeChange(standaloneSlashCommand); promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 7280f7457baa..668954b7afa8 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -58,6 +58,7 @@ import { useLocation, useNavigate, useParams } from "@tanstack/react-router"; import * as Option from "effect/Option"; import { ArrowLeftIcon, + AudioLinesIcon, ChartNoAxesColumnIcon, CheckIcon, ChevronRightIcon, @@ -69,6 +70,8 @@ import { MessageSquareDashedIcon, LinkIcon, MessageSquareIcon, + MicIcon, + MicOffIcon, MonitorIcon, MoonIcon, PaletteIcon, @@ -163,6 +166,8 @@ import { import { selectThreadTerminalUiState, useTerminalUiStateStore } from "../terminalUiStateStore"; import { buildThreadRouteParams, resolveThreadRouteTarget } from "../threadRoutes"; import { useAvailableSettingsSearchItems } from "./settings/useAvailableSettingsSearchItems"; +import { resolveThreadVoiceAvailability } from "../voice/voiceAvailability"; +import { toggleThreadVoice, useVoiceModeState, voiceMode } from "../voice/voiceMode"; import { applyWslEnvironmentConfiguration, parseWslUncPath, @@ -665,6 +670,7 @@ function OpenCommandPaletteDialog(props: { const activeThreadServerConfig = useServerConfigs().get( activeThread?.environmentId ?? ("" as EnvironmentId), ); + const voiceState = useVoiceModeState(); const activeThreadReferenceCopyTarget = referenceThreadRef === null || (pathname === "/pull-requests" && !openPanelPullRequestUrl) ? null @@ -2007,6 +2013,66 @@ function OpenCommandPaletteDialog(props: { }); } + const voiceEngaged = voiceState.phase !== "idle"; + const voiceTarget = activeThread + ? scopeThreadRef(activeThread.environmentId, activeThread.id) + : null; + const voiceOnActiveThread = voiceTarget !== null && voiceMode.isActiveFor(voiceTarget); + if (activeThread !== null && voiceTarget !== null && !voiceOnActiveThread) { + const voiceAvailability = resolveThreadVoiceAvailability( + activeThread, + activeThreadServerConfig?.providers ?? [], + ); + if (voiceAvailability.kind !== "unsupported") { + actionItems.push({ + kind: "action", + value: "action:voice-start", + searchTerms: ["voice", "talk", "speak", "conversation", "microphone", "realtime"], + title: "Start voice conversation", + description: + voiceAvailability.kind === "unavailable" ? voiceAvailability.reason : undefined, + disabled: voiceAvailability.kind === "unavailable", + icon: , + shortcutCommand: "voice.toggle", + run: async () => { + toggleThreadVoice(voiceTarget, voiceAvailability); + }, + }); + } + } + // Stop and mute act on the one conversation, wherever it runs. + if (voiceEngaged) { + actionItems.push( + { + kind: "action", + value: "action:voice-stop", + searchTerms: ["voice", "stop", "end", "conversation", "hang up"], + title: "Stop voice conversation", + icon: , + // F8 toggles the current thread, so it only stops a conversation held here. + ...(voiceOnActiveThread ? { shortcutCommand: "voice.toggle" as const } : {}), + run: async () => { + voiceMode.stop(); + }, + }, + { + kind: "action", + value: "action:voice-mute", + searchTerms: ["voice", "mute", "unmute", "microphone"], + title: voiceState.muted ? "Unmute voice" : "Mute voice", + icon: voiceState.muted ? ( + + ) : ( + + ), + shortcutCommand: "voice.toggleMute", + run: async () => { + voiceMode.toggleMuted(); + }, + }, + ); + } + actionItems.push({ kind: "action", value: "action:open-file-picker", diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 9162c4b67e0d..8e868f5daa3a 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -341,6 +341,9 @@ import { } from "./composerScrollGesture"; import { prepareVideoFirstFrame } from "../../lib/videoFirstFrame"; import { observeResize } from "~/lib/observeResize"; +import { resolveVoiceAvailability } from "../../voice/voiceAvailability"; +import { toggleThreadVoice, useVoicePhaseFor, voiceMode } from "../../voice/voiceMode"; +import { VoiceModeStrip } from "./VoiceModeStrip"; function ComposerVideoThumbnail({ file }: { file: File }) { const setVideo = useCallback( @@ -1081,6 +1084,7 @@ import { Select, SelectItem, SelectPopup, SelectValue } from "../ui/select"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { toastManager } from "../ui/toast"; import { + AudioLinesIcon, FileIcon, BotIcon, CircleAlertIcon, @@ -2240,6 +2244,34 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) [selectedProviderEntry], ); const compactCommandAvailable = providerSupportsManualCompaction(selectedProviderEntry); + // Voice follows the provider that owns the thread; a draft without a thread + // shell follows the composer's selection and waits for its first message. + const voiceThread = activeThread ?? props.activeThreadShell ?? null; + const voiceProviderInstanceId = voiceThread?.providerInstanceId ?? selectedInstanceId; + const voiceThreadHasMessages = voiceThread?.latestUserMessageAt != null; + const voiceThreadHasSession = voiceThread?.activeProviderThreadId != null; + const voiceAvailability = useMemo( + () => + resolveVoiceAvailability({ + provider: providerStatuses.find( + (provider) => provider.instanceId === voiceProviderInstanceId, + ), + hasMessages: voiceThreadHasMessages, + hasSession: voiceThreadHasSession, + }), + [providerStatuses, voiceProviderInstanceId, voiceThreadHasMessages, voiceThreadHasSession], + ); + const voicePhase = useVoicePhaseFor(routeThreadRef); + const voiceEngaged = voicePhase !== "idle"; + // A conversation stays stoppable here even if its provider stopped offering voice. + const voiceSupported = voiceAvailability.kind !== "unsupported" || voiceEngaged; + const toggleVoice = useCallback( + () => toggleThreadVoice(routeThreadRef, voiceAvailability), + [routeThreadRef, voiceAvailability], + ); + const voiceButtonDisabledReason = + voiceAvailability.kind === "unavailable" && !voiceEngaged ? voiceAvailability.reason : null; + const voiceButtonLabel = voiceEngaged ? "Stop voice conversation" : "Start voice conversation"; // Memoized so the composer menu memo below can cache between renders. const selectedProviderSkills = useMemo( () => @@ -2737,6 +2769,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }, ] as const) : []), + ...(voiceSupported + ? ([ + { + id: "slash:voice", + type: "slash-command", + command: "voice", + label: "/voice", + description: voiceEngaged + ? "Stop the voice conversation" + : "Start a voice conversation", + }, + ] as const) + : []), ] satisfies ReadonlyArray>; const slashMenuSkills = getProviderSkillsForSlashMenu( selectedProviderSkills, @@ -2856,6 +2901,8 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) selectedProviderSlashCommands, selectedProviderStatus, settings.showSkillsInSlashMenu, + voiceEngaged, + voiceSupported, workspaceEntries.entries, ]); @@ -4052,6 +4099,16 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) } return; } + if (item.command === "voice") { + const applied = applyPromptReplacement(trigger.rangeStart, trigger.rangeEnd, "", { + expectedText: snapshot.value.slice(trigger.rangeStart, trigger.rangeEnd), + }); + if (applied) { + setComposerHighlightedItemId(null); + toggleVoice(); + } + return; + } if (!planModeUiEnabled) return; void handleInteractionModeChange(item.command === "plan" ? "plan" : "default"); const applied = applyPromptReplacement(trigger.rangeStart, trigger.rangeEnd, "", { @@ -4174,6 +4231,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) planModeUiEnabled, onUsageLimitsCommand, resolveActiveComposerTrigger, + toggleVoice, ], ); @@ -5773,6 +5831,32 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) terminalOpen, ]); + useEffect(() => { + const handler = (event: globalThis.KeyboardEvent) => { + const command = resolveShortcutCommand(event, keybindings, { + context: { + terminalFocus: getTerminalFocusOwner() !== null, + terminalOpen, + modelPickerOpen: isComposerModelPickerOpen, + }, + }); + if (command === "voice.toggleMute") { + // Muting is global: the microphone belongs to the one conversation. + if (voiceMode.getState().phase === "idle") return; + event.preventDefault(); + event.stopPropagation(); + if (!event.repeat) voiceMode.toggleMuted(); + return; + } + if (command !== "voice.toggle" || !voiceSupported || isCommandPaletteOpen()) return; + event.preventDefault(); + event.stopPropagation(); + if (!event.repeat) toggleVoice(); + }; + window.addEventListener("keydown", handler, true); + return () => window.removeEventListener("keydown", handler, true); + }, [isComposerModelPickerOpen, keybindings, terminalOpen, toggleVoice, voiceSupported]); + // ------------------------------------------------------------------ // Callbacks: attachments // ------------------------------------------------------------------ @@ -6779,6 +6863,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ) : null} + {voiceEngaged ? ( + + + + ) : null} {showComposerTopDrawer && (!isTasksDrawerOpen || hasBlockingComposerTopDrawer) ? ( ) : null} + {voiceSupported && !isComposerResting ? ( + + event.preventDefault()} + onClick={voiceButtonDisabledReason === null ? toggleVoice : undefined} + aria-disabled={voiceButtonDisabledReason !== null || undefined} + aria-pressed={voiceEngaged} + aria-label={voiceButtonLabel} + /> + } + > + + + {voiceButtonDisabledReason ?? voiceButtonLabel} + + ) : null} ; + label: string; + /** Fixed bar height when levels are not animated (reduced motion). */ + staticLevel: number; +}) { + return ( + + {METER_BARS.map((bar) => ( + + ))} + + ); +} + +/** + * Live status for the thread's voice conversation: state, mic and speaker + * meters, mute and stop, and the latest captions. Meters are written straight + * to the DOM from the controller's level samples so the composer never + * re-renders per sample. + */ +export const VoiceModeStrip = memo(function VoiceModeStrip({ + target, +}: { + target: VoiceSessionTarget; +}) { + const state = useVoiceModeState(); + const isOnThread = isVoiceTargetFor(state, target) && state.phase !== "idle"; + const isActive = isOnThread && state.phase === "active"; + const prefersReducedMotion = useMediaQuery("(prefers-reduced-motion: reduce)"); + const microphoneMeterRef = useRef(null); + const speakerMeterRef = useRef(null); + const [speaking, setSpeaking] = useState(false); + + useEffect(() => { + if (!isActive) return; + // Teardown publishes a silent sample, which clears `speaking` for the next conversation. + return voiceMode.subscribeLevels((levels) => { + setSpeaking(levels.speaker > SPEAKING_LEVEL); + if (prefersReducedMotion) return; + applyMeterLevel(microphoneMeterRef.current, levels.microphone); + applyMeterLevel(speakerMeterRef.current, levels.speaker); + }); + }, [isActive, prefersReducedMotion]); + + if (!isOnThread) return null; + + const status = !isActive + ? "Connecting…" + : speaking + ? "Speaking" + : state.muted + ? "Muted" + : "Listening"; + const captions = state.captions.slice(-VISIBLE_CAPTIONS); + // Reduced motion keeps the meters still: full while a channel is live, low while muted. + const staticMicrophoneLevel = prefersReducedMotion && isActive && !state.muted ? 0.4 : 0; + const staticSpeakerLevel = prefersReducedMotion && speaking ? 0.4 : 0; + + return ( + + + {isActive ? : } + + + {status} + + {isActive ? ( + + + {state.muted ? ( + + ) : ( + + )} + + + + + + + + ) : null} + + + + + + + {captions.length > 0 ? ( + + {captions.map((caption, index) => ( +

+ {caption.text} +

+ ))} + + ) : null} + + ); +}); diff --git a/apps/web/src/composer-logic.test.ts b/apps/web/src/composer-logic.test.ts index df3e54b4ddfc..f3c3eb75269e 100644 --- a/apps/web/src/composer-logic.test.ts +++ b/apps/web/src/composer-logic.test.ts @@ -813,8 +813,13 @@ describe("parseStandaloneComposerSlashCommand", () => { expect(parseStandaloneComposerSlashCommand("/default")).toBe("default"); }); + it("parses standalone /voice command", () => { + expect(parseStandaloneComposerSlashCommand("/Voice ")).toBe("voice"); + }); + it("ignores slash commands with extra message text", () => { expect(parseStandaloneComposerSlashCommand("/plan explain this")).toBeNull(); + expect(parseStandaloneComposerSlashCommand("/voice please")).toBeNull(); }); }); diff --git a/apps/web/src/composer-logic.ts b/apps/web/src/composer-logic.ts index a44b999f2ba6..c27a6735c71a 100644 --- a/apps/web/src/composer-logic.ts +++ b/apps/web/src/composer-logic.ts @@ -12,7 +12,7 @@ import { import { resolveShortcutCommand, type ShortcutEventLike } from "./keybindings"; export type ComposerTriggerKind = "path" | "pull-request" | "slash-command" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "voice"; export type ComposerSubmissionIntent = "foreground" | "background" | "alternate"; export interface ComposerTrigger { @@ -342,12 +342,13 @@ export function composerStateAtPromptEnd( export function parseStandaloneComposerSlashCommand( text: string, ): Exclude | null { - const match = /^\/(plan|default)\s*$/i.exec(text.trim()); + const match = /^\/(plan|default|voice)\s*$/i.exec(text.trim()); if (!match) { return null; } const command = match[1]?.toLowerCase(); if (command === "plan") return "plan"; + if (command === "voice") return "voice"; return "default"; } diff --git a/apps/web/src/keybindings.test.ts b/apps/web/src/keybindings.test.ts index d1e842205efb..cd7f43e4b45f 100644 --- a/apps/web/src/keybindings.test.ts +++ b/apps/web/src/keybindings.test.ts @@ -643,6 +643,23 @@ describe("thread navigation helpers", () => { ), ); }); + + it("toggles voice with a bare F8 unless the terminal has focus", () => { + const input = event({ key: "F8", code: "F8" }); + assert.strictEqual( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: false }, + }), + "voice.toggle", + ); + assert.isNull( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: true }, + }), + ); + }); }); describe("model picker navigation helpers", () => { diff --git a/apps/web/src/voice/voiceAvailability.test.ts b/apps/web/src/voice/voiceAvailability.test.ts new file mode 100644 index 000000000000..0d5b0ff85e1f --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.test.ts @@ -0,0 +1,40 @@ +import { ProviderInstanceId, type ServerProvider } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { resolveThreadVoiceAvailability } from "./voiceAvailability"; + +const codex = ProviderInstanceId.make("codex"); +const claude = ProviderInstanceId.make("claudeAgent"); +const providers = [ + { instanceId: codex, supportsVoice: true }, + { instanceId: claude }, +] as unknown as ReadonlyArray; + +function thread(input: { readonly provider: ProviderInstanceId; readonly messages?: boolean }) { + return { + providerInstanceId: input.provider, + latestUserMessageAt: input.messages ? "2026-09-22T00:00:00.000Z" : null, + activeProviderThreadId: input.messages ? "provider-thread-1" : null, + }; +} + +describe("resolveThreadVoiceAvailability", () => { + it("hides voice for providers that do not support it", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: claude, messages: true }), providers), + ).toEqual({ kind: "unsupported" }); + }); + + it("explains that a new thread needs a first message", () => { + expect(resolveThreadVoiceAvailability(thread({ provider: codex }), providers)).toEqual({ + kind: "unavailable", + reason: "Send a message before starting voice", + }); + }); + + it("is ready once a Codex thread has run", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: codex, messages: true }), providers), + ).toEqual({ kind: "ready" }); + }); +}); diff --git a/apps/web/src/voice/voiceAvailability.ts b/apps/web/src/voice/voiceAvailability.ts new file mode 100644 index 000000000000..c376510845f4 --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.ts @@ -0,0 +1,46 @@ +import type { ProviderInstanceId, ServerProvider } from "@t3tools/contracts"; + +/** + * Whether a thread can hold a voice conversation. `unsupported` hides every + * voice entry point; `unavailable` shows them disabled with `reason`. + */ +export type VoiceAvailability = + | { readonly kind: "unsupported" } + | { readonly kind: "unavailable"; readonly reason: string } + | { readonly kind: "ready" }; + +export const VOICE_UNSUPPORTED: VoiceAvailability = { kind: "unsupported" }; + +export function resolveVoiceAvailability(input: { + /** The provider instance that runs (or will run) the thread's turns. */ + readonly provider: Pick | null | undefined; + readonly hasMessages: boolean; + readonly hasSession: boolean; +}): VoiceAvailability { + if (input.provider?.supportsVoice !== true) return VOICE_UNSUPPORTED; + // The server hands voice requests to the thread's bound provider session, + // which only exists once a turn has been sent. + if (!input.hasMessages || !input.hasSession) { + return { kind: "unavailable", reason: "Send a message before starting voice" }; + } + return { kind: "ready" }; +} + +/** The thread fields voice availability reads (orchestrator V2 thread shells). */ +export interface VoiceThread { + readonly providerInstanceId: ProviderInstanceId; + readonly latestUserMessageAt: string | null; + readonly activeProviderThreadId: string | null; +} + +/** Voice follows the provider instance that owns the thread. */ +export function resolveThreadVoiceAvailability( + thread: VoiceThread, + providers: ReadonlyArray, +): VoiceAvailability { + return resolveVoiceAvailability({ + provider: providers.find((provider) => provider.instanceId === thread.providerInstanceId), + hasMessages: thread.latestUserMessageAt !== null, + hasSession: thread.activeProviderThreadId !== null, + }); +} diff --git a/apps/web/src/voice/voiceMode.ts b/apps/web/src/voice/voiceMode.ts new file mode 100644 index 000000000000..9a5defee23b0 --- /dev/null +++ b/apps/web/src/voice/voiceMode.ts @@ -0,0 +1,144 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + VoiceModeController, + describeVoiceEnd, + makeVoiceSessionOpener, + type VoiceMediaStream, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionTarget, +} from "@t3tools/client-runtime/voice-mode"; +import * as Effect from "effect/Effect"; +import { AtomRegistry } from "effect/unstable/reactivity"; +import { useSyncExternalStore } from "react"; + +import { stackedThreadToast, toastManager } from "../components/ui/toast"; +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import type { VoiceAvailability } from "./voiceAvailability"; + +function getMicrophone(): Promise { + const mediaDevices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices; + // Browsers only expose the microphone API on secure origins, so a remote + // environment opened over plain HTTP has no `mediaDevices` at all. + if (!mediaDevices?.getUserMedia) { + return Promise.reject( + new Error("Voice needs microphone access, which requires HTTPS when connecting remotely."), + ); + } + return mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); +} + +function playRemoteAudio(stream: VoiceMediaStream): () => void { + const audio = new Audio(); + audio.autoplay = true; + audio.srcObject = stream as MediaStream; + void audio.play().catch(() => undefined); + return () => { + audio.pause(); + audio.srcObject = null; + }; +} + +// The voice stream runs on the connection runtime, which owns the environment +// registry. Mounting keeps that runtime alive for the conversation. +const openSession = makeVoiceSessionOpener((effect) => + Effect.runFork( + Effect.scoped( + AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.andThen(AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime)), + Effect.flatMap((context) => Effect.provideContext(effect, context)), + ), + ).pipe(Effect.orDie), + ), +); + +const platform: VoiceModePlatform = { + // The DOM's overloaded addEventListener does not fit the structural `never` + // listener in VoicePeerConnection, though the runtime shape matches. + createPeerConnection: () => new RTCPeerConnection() as unknown as VoicePeerConnection, + getMicrophone, + playRemoteAudio, + openSession, +}; + +/** The app's single voice conversation. The microphone is exclusive, so there is one per client. */ +export const voiceMode = new VoiceModeController(platform); + +export function isVoiceTargetFor( + state: VoiceModeState, + target: VoiceSessionTarget | null, +): boolean { + return ( + target !== null && + state.target !== null && + state.target.environmentId === target.environmentId && + state.target.threadId === target.threadId + ); +} + +export function useVoiceModeState(): VoiceModeState { + return useSyncExternalStore(voiceMode.subscribe, voiceMode.getState, voiceMode.getState); +} + +/** The conversation phase on `target`; "idle" while voice is off or on another thread. */ +export function useVoicePhaseFor(target: VoiceSessionTarget | null): VoiceModePhase { + const getPhase = () => { + const state = voiceMode.getState(); + return isVoiceTargetFor(state, target) ? state.phase : "idle"; + }; + return useSyncExternalStore(voiceMode.subscribe, getPhase, getPhase); +} + +/** + * Starts or stops voice on `target`. An unavailable thread explains why + * instead of failing on the server. + */ +export function toggleThreadVoice(target: VoiceSessionTarget, availability: VoiceAvailability) { + if (voiceMode.isActiveFor(target)) { + voiceMode.stop(); + return; + } + if (availability.kind === "unsupported") return; + if (availability.kind === "unavailable") { + toastManager.add({ type: "info", title: availability.reason }); + return; + } + void voiceMode.start(target); +} + +// Errors and provider-side ends surface once as a toast on the conversation's +// thread. A user stop leaves neither set, so it stays silent. +let lastPhase: VoiceModePhase = "idle"; +voiceMode.subscribe((state) => { + const previousPhase = lastPhase; + lastPhase = state.phase; + if (state.phase !== "idle" || state.target === null) return; + if (state.error === null && state.endedReason === null) return; + const threadRef = scopeThreadRef( + EnvironmentId.make(state.target.environmentId), + ThreadId.make(state.target.threadId), + ); + if (state.error !== null) { + toastManager.add( + stackedThreadToast({ + type: "error", + title: previousPhase === "active" ? "Voice conversation ended" : "Couldn't start voice", + description: state.error, + data: { threadRef }, + }), + ); + } else if (state.endedReason !== null) { + toastManager.add({ + type: "info", + title: describeVoiceEnd(state.endedReason), + data: { threadRef }, + }); + } + voiceMode.dismiss(); +}); diff --git a/docs/internals/voice-conversations.md b/docs/internals/voice-conversations.md new file mode 100644 index 000000000000..1b9dc06e08ea --- /dev/null +++ b/docs/internals/voice-conversations.md @@ -0,0 +1,34 @@ +# Voice conversations + +A voice conversation is a Codex realtime session bound to one thread. The user +talks to a voice model, and when it wants work done it starts an ordinary Codex +turn on that thread. This is separate from [voice input](voice-input.md), which +only transcribes into the composer. + +Audio never touches the T3 socket. The client owns the microphone, speaker, and +WebRTC peer, and audio flows directly between the client and OpenAI. The +`provider.voice.session` stream carries only the SDP handshake, captions, and +lifecycle. That keeps remote and tunneled clients as cheap as local ones, and it +is why the [shared controller](../../packages/client-runtime/src/voice-mode/controller.ts) +takes a platform object: browsers and react-native-webrtc supply the same peer API. + +The stream's lifetime is the conversation's lifetime. When the client +unsubscribes or disconnects, the server stops the realtime session, so a dropped +client cannot leave a billed session running. + +Handoffs are client-managed, as in the Codex TUI. The +[Codex voice module](../../apps/server/src/provider/Layers/CodexVoice.ts) hands +each voice turn's final answer back with `thread/realtime/appendSpeech`, which +keeps commentary and reasoning unspoken. + +Codex starts or steers the delegated turn itself, with a user message wrapped in +``, and it cannot be told not to. Steering an active run +needs nothing special. An idle thread is different: V2 runs are +orchestrator-owned and there is no provider-initiated run yet, so +[CodexAdapterV2](../../apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts) +interrupts that turn and, once it settles, offers a `voice` continuation. The +request then runs as an ordinary user message, and Codex's history keeps a stub +of the interrupted turn. Captions are ephemeral and are not persisted. + +A thread needs an existing provider session binding before voice can start, +because Codex attaches realtime to an existing provider thread. diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index bdc87902a63b..b044b74cc8c6 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -92,6 +92,29 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques the current session, or permanently. See [Permission modes](./permission-modes.md) for command and file approvals. +## Voice conversations + +A Codex thread can hold a spoken conversation. You talk with +a voice model that hands coding requests to Codex and reads its answers back to +you. Send at least one message in the thread first. + +Start or stop it with the waveform button in the composer. On web and desktop you +can also send `/voice` or press `F8`. While it runs, the strip above the composer shows whether it is +listening or speaking, with buttons to mute your microphone and stop. The command +palette offers the same actions, and **Voice: Toggle Mute** can be bound in +**Settings > Keybindings**. + +Choose the voice in **Settings > Providers > Codex > Voice**. The change applies +to your next conversation. + +On mobile, the conversation stops when you leave the thread or the app. The mobile +app needs an update that includes voice support. + +Voice needs a microphone and permission to use it. When you open T3 Code from +another device, the page must be served over HTTPS for the browser to allow the +microphone. Live captions are not saved: the thread keeps only the requests +handed to Codex and Codex's replies. + ## Codex says I hit a usage limit When Codex stops on a usage limit, the thread names the window that ran out and diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0366c65b1ed2..c7e54c7ec9d8 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -111,6 +111,10 @@ "types": "./src/voice-input/index.ts", "default": "./src/voice-input/index.ts" }, + "./voice-mode": { + "types": "./src/voice-mode/index.ts", + "default": "./src/voice-mode/index.ts" + }, "./t3ToolSummary": { "types": "./src/t3ToolSummary.ts", "default": "./src/t3ToolSummary.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index c11a49897b88..f890e18b110c 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -76,6 +76,7 @@ export type EnvironmentStreamCommandRpcTag = | typeof WS_METHODS.chatGptHandoffSubscribe | typeof WS_METHODS.cloudInstallRelayClient | typeof WS_METHODS.serverUpdateServerWithProgress + | typeof WS_METHODS.providerVoiceSession | typeof WS_METHODS.gitRunStackedAction; export type EnvironmentStreamRpcTag = diff --git a/packages/client-runtime/src/voice-mode/controller.test.ts b/packages/client-runtime/src/voice-mode/controller.test.ts new file mode 100644 index 000000000000..1be4869bc3f2 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + appendVoiceCaption, + finishVoiceCaption, + VoiceModeController, + type VoiceMediaStream, + type VoiceModePlatform, + type VoicePeerConnection, + type VoiceSessionHandlers, +} from "./controller.ts"; + +const target = { environmentId: "env-1", threadId: "thread-1" }; + +function makeFakePlatform(options: { readonly denyMicrophone?: boolean } = {}) { + const track = { enabled: true, stopped: false, stop: () => void (track.stopped = true) }; + const microphone: VoiceMediaStream = { getTracks: () => [track], getAudioTracks: () => [track] }; + const listeners = new Map void>(); + const peer = { + connectionState: "new", + closed: false, + remoteSdp: null as string | null, + dataChannels: [] as string[], + addTrack: () => undefined, + createDataChannel: (label: string) => void peer.dataChannels.push(label), + createOffer: async () => ({ sdp: "v=offer\r\n" }), + setLocalDescription: async () => undefined, + setRemoteDescription: async (description: { sdp: string }) => { + peer.remoteSdp = description.sdp; + }, + getStats: async () => ({ forEach: () => undefined }), + addEventListener: (type: string, listener: (event: never) => void) => + void listeners.set(type, listener), + close: () => void (peer.closed = true), + connect: () => { + peer.connectionState = "connected"; + listeners.get("connectionstatechange")?.({} as never); + }, + }; + const sessions: Array<{ + readonly offerSdp: string; + readonly handlers: VoiceSessionHandlers; + cancelled: boolean; + }> = []; + const platform: VoiceModePlatform = { + createPeerConnection: () => peer as unknown as VoicePeerConnection, + getMicrophone: async () => { + if (options.denyMicrophone) { + throw Object.assign(new Error("denied"), { name: "NotAllowedError" }); + } + return microphone; + }, + playRemoteAudio: () => () => undefined, + openSession: (input, handlers) => { + const session = { offerSdp: input.offerSdp, handlers, cancelled: false }; + sessions.push(session); + return () => void (session.cancelled = true); + }, + }; + return { platform, peer, track, sessions }; +} + +async function startConversation() { + const fake = makeFakePlatform(); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + const session = fake.sessions[0]!; + return { ...fake, controller, session }; +} + +describe("VoiceModeController", () => { + it("negotiates WebRTC through the voice session and activates once both sides are up", async () => { + const { controller, peer, session } = await startConversation(); + expect(peer.dataChannels).toEqual(["oai-events"]); + expect(session.offerSdp).toBe("v=offer\r\n"); + expect(controller.getState().phase).toBe("connecting"); + + session.handlers.onEvent({ type: "answer", sdp: "v=answer\r\n" }); + await Promise.resolve(); + expect(peer.remoteSdp).toBe("v=answer\r\n"); + + session.handlers.onEvent({ type: "started" }); + expect(controller.getState().phase).toBe("connecting"); + peer.connect(); + expect(controller.getState().phase).toBe("active"); + controller.stop(); + }); + + it("keeps ephemeral captions and reports why Codex ended the conversation", async () => { + const { controller, peer, session, track } = await startConversation(); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: "run the" }); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: " tests" }); + expect(controller.getState().captions).toEqual([ + { role: "user", text: "run the tests", final: false }, + ]); + + session.handlers.onEvent({ type: "closed", reason: "transport_closed" }); + const state = controller.getState(); + expect(state.phase).toBe("idle"); + expect(state.captions).toEqual([]); + expect(state.endedReason).toBe("transport_closed"); + expect(state.target).toEqual(target); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + }); + + it("ends the server session and releases the microphone when the user stops", async () => { + const { controller, peer, session, track } = await startConversation(); + controller.toggle(target); + expect(session.cancelled).toBe(true); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + expect(controller.getState()).toMatchObject({ phase: "idle", endedReason: null, error: null }); + }); + + it("mutes by disabling the microphone track", async () => { + const { controller, track } = await startConversation(); + controller.toggleMuted(); + expect(track.enabled).toBe(false); + expect(controller.getState().muted).toBe(true); + controller.toggleMuted(); + expect(track.enabled).toBe(true); + controller.stop(); + }); + + it("explains a denied microphone without opening a session", async () => { + const fake = makeFakePlatform({ denyMicrophone: true }); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + expect(fake.sessions).toHaveLength(0); + expect(controller.getState()).toMatchObject({ + phase: "idle", + error: "Microphone access was denied.", + }); + }); +}); + +describe("voice captions", () => { + it("interleaves speakers and settles each with its final text", () => { + let captions = appendVoiceCaption([], "user", "hello"); + captions = appendVoiceCaption(captions, "assistant", "Hi"); + captions = appendVoiceCaption(captions, "user", " there"); + captions = finishVoiceCaption(captions, "user", "Hello there."); + expect(captions).toEqual([ + { role: "user", text: "Hello there.", final: true }, + { role: "assistant", text: "Hi", final: false }, + ]); + }); + + it("keeps only the most recent captions", () => { + let captions = finishVoiceCaption([], "user", "one"); + for (const text of ["two", "three", "four", "five"]) { + captions = finishVoiceCaption(captions, "assistant", text); + } + expect(captions.map((caption) => caption.text)).toEqual(["two", "three", "four", "five"]); + }); +}); diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts new file mode 100644 index 000000000000..42ab725dbcdf --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -0,0 +1,433 @@ +// @effect-diagnostics globalTimers:off - WebRTC lifecycle runs on platform callbacks, outside an Effect runtime. +import type { ProviderVoiceSessionEvent, ProviderVoiceTranscriptRole } from "@t3tools/contracts"; + +/** + * Voice conversations (Codex realtime voice). The client owns the microphone, + * speaker, and WebRTC peer; the environment only relays the SDP handshake and + * captions over `provider.voice.session`. Audio never crosses the T3 socket, + * so voice works the same locally, over a relay, or through a tunnel. + * + * The WebRTC surface is structural so browsers and react-native-webrtc both fit. + */ + +export interface VoiceMediaTrack { + enabled: boolean; + stop(): void; +} + +export interface VoiceMediaStream { + getTracks(): ReadonlyArray; + getAudioTracks(): ReadonlyArray; + /** react-native-webrtc frees the native stream here; browsers have no equivalent. */ + release?(): void; +} + +interface VoiceSessionDescription { + readonly sdp?: string | null | undefined; +} + +interface VoiceStatsReport { + forEach(callback: (stat: { readonly type?: string; readonly kind?: string }) => void): void; +} + +export interface VoicePeerConnection { + readonly connectionState: string; + addTrack(track: never, stream: never): unknown; + createDataChannel(label: string, options?: { readonly ordered?: boolean }): unknown; + createOffer(): Promise; + setLocalDescription(description: never): Promise; + setRemoteDescription(description: never): Promise; + getStats(): Promise; + addEventListener(type: "track" | "connectionstatechange", listener: (event: never) => void): void; + close(): void; +} + +export interface VoiceSessionTarget { + readonly environmentId: string; + readonly threadId: string; +} + +export interface VoiceSessionHandlers { + readonly onEvent: (event: ProviderVoiceSessionEvent) => void; + readonly onError: (message: string) => void; + readonly onEnd: () => void; +} + +export interface VoiceModePlatform { + readonly createPeerConnection: () => VoicePeerConnection; + /** Resolves an echo-cancelled microphone stream, prompting for permission if needed. */ + readonly getMicrophone: () => Promise; + /** Plays the assistant's audio. Returns a disposer. */ + readonly playRemoteAudio: (stream: VoiceMediaStream) => () => void; + /** Opens `provider.voice.session`. Returns a cancel function that ends the conversation. */ + readonly openSession: ( + input: VoiceSessionTarget & { readonly offerSdp: string }, + handlers: VoiceSessionHandlers, + ) => () => void; +} + +export type VoiceModePhase = "idle" | "connecting" | "active"; + +export interface VoiceCaption { + readonly role: ProviderVoiceTranscriptRole; + readonly text: string; + readonly final: boolean; +} + +export interface VoiceModeState { + readonly phase: VoiceModePhase; + /** The conversation's thread; kept after it ends so its error or end notice can show there. */ + readonly target: VoiceSessionTarget | null; + readonly muted: boolean; + /** Recent captions, oldest first. Ephemeral: they vanish when the conversation ends. */ + readonly captions: ReadonlyArray; + readonly error: string | null; + /** Why the last conversation ended when the user did not end it. */ + readonly endedReason: string | null; +} + +/** Mic and speaker loudness in 0..1, sampled while a conversation is active. */ +export interface VoiceLevels { + readonly microphone: number; + readonly speaker: number; +} + +export const VOICE_CONNECT_TIMEOUT_MS = 20_000; +const LEVEL_SAMPLE_INTERVAL_MS = 100; +const MAX_CAPTIONS = 4; +const MAX_CAPTION_CHARS = 1024; + +export const INITIAL_VOICE_MODE_STATE: VoiceModeState = { + phase: "idle", + target: null, + muted: false, + captions: [], + error: null, + endedReason: null, +}; + +/** Appends a streaming caption delta, continuing the speaker's open caption. */ +export function appendVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + delta: string, +): ReadonlyArray { + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return [...captions, { role, text: delta.trimStart(), final: false }].slice(-MAX_CAPTIONS); + } + const open = captions[openIndex]!; + const text = (open.text + delta).slice(-MAX_CAPTION_CHARS); + return captions.map((caption, index) => (index === openIndex ? { ...caption, text } : caption)); +} + +/** Settles the speaker's open caption with the final transcript text. */ +export function finishVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + text: string, +): ReadonlyArray { + const finalText = text.trim().slice(-MAX_CAPTION_CHARS); + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return finalText + ? [...captions, { role, text: finalText, final: true }].slice(-MAX_CAPTIONS) + : captions; + } + if (!finalText) return captions.filter((_, index) => index !== openIndex); + return captions.map((caption, index) => + index === openIndex ? { role, text: finalText, final: true } : caption, + ); +} + +/** User-facing text for why a conversation ended without the user stopping it. */ +export function describeVoiceEnd(reason: string): string { + switch (reason) { + case "replaced": + return "Voice conversation moved to another window or device."; + case "ended": + return "Voice conversation ended."; + default: + return `Voice conversation ended: ${reason.replaceAll("_", " ")}.`; + } +} + +function describeError(cause: unknown): string { + if (cause && typeof cause === "object" && "name" in cause) { + const name = String((cause as { name: unknown }).name); + if (name === "NotAllowedError" || name === "SecurityError") { + return "Microphone access was denied."; + } + if (name === "NotFoundError") return "No microphone was found."; + } + if (cause instanceof Error && cause.message) return cause.message; + return "Voice conversation failed."; +} + +interface ActiveSession { + readonly target: VoiceSessionTarget; + readonly id: number; + microphone: VoiceMediaStream | null; + peer: VoicePeerConnection | null; + cancelSession: (() => void) | null; + stopRemoteAudio: (() => void) | null; + backendStarted: boolean; + peerConnected: boolean; + connectTimer: ReturnType | null; + levelTimer: ReturnType | null; + sampling: boolean; +} + +/** + * Owns at most one voice conversation. Starting on another thread ends the + * current one first, since the microphone is exclusive. + */ +export class VoiceModeController { + private state: VoiceModeState = INITIAL_VOICE_MODE_STATE; + private session: ActiveSession | null = null; + private nextSessionId = 1; + private readonly listeners = new Set<(state: VoiceModeState) => void>(); + private readonly levelListeners = new Set<(levels: VoiceLevels) => void>(); + + private readonly platform: VoiceModePlatform; + + constructor(platform: VoiceModePlatform) { + this.platform = platform; + } + + getState = (): VoiceModeState => this.state; + + subscribe = (listener: (state: VoiceModeState) => void): (() => void) => { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + }; + + subscribeLevels = (listener: (levels: VoiceLevels) => void): (() => void) => { + this.levelListeners.add(listener); + return () => this.levelListeners.delete(listener); + }; + + isActiveFor(target: VoiceSessionTarget): boolean { + const current = this.state.target; + return ( + this.state.phase !== "idle" && + current !== null && + current.environmentId === target.environmentId && + current.threadId === target.threadId + ); + } + + toggle(target: VoiceSessionTarget): void { + if (this.isActiveFor(target)) { + this.stop(); + } else { + void this.start(target); + } + } + + async start(target: VoiceSessionTarget): Promise { + this.stop(); + const session: ActiveSession = { + target, + id: this.nextSessionId++, + microphone: null, + peer: null, + cancelSession: null, + stopRemoteAudio: null, + backendStarted: false, + peerConnected: false, + connectTimer: null, + levelTimer: null, + sampling: false, + }; + this.session = session; + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + phase: "connecting", + target, + muted: this.state.muted, + }); + + try { + const microphone = await this.platform.getMicrophone(); + if (this.session !== session) { + for (const track of microphone.getTracks()) track.stop(); + microphone.release?.(); + return; + } + session.microphone = microphone; + // Timed from here so a first-run permission prompt doesn't count against it. + session.connectTimer = setTimeout( + () => this.fail(session, "Voice connection timed out."), + VOICE_CONNECT_TIMEOUT_MS, + ); + for (const track of microphone.getAudioTracks()) track.enabled = !this.state.muted; + + const peer = this.platform.createPeerConnection(); + session.peer = peer; + peer.addEventListener( + "track", + (event: { readonly streams?: ReadonlyArray }) => { + const [remote] = event.streams ?? []; + if (!remote || this.session !== session) return; + session.stopRemoteAudio?.(); + session.stopRemoteAudio = this.platform.playRemoteAudio(remote); + }, + ); + peer.addEventListener("connectionstatechange", () => { + if (this.session !== session) return; + if (peer.connectionState === "connected") { + session.peerConnected = true; + this.maybeActivate(session); + } else if (peer.connectionState === "failed") { + this.fail(session, "Voice connection was lost."); + } + }); + for (const track of microphone.getAudioTracks()) { + peer.addTrack(track as never, microphone as never); + } + // Codex's realtime transport requires the ordered events channel in the offer. + peer.createDataChannel("oai-events", { ordered: true }); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer as never); + if (this.session !== session) return; + if (!offer.sdp) throw new Error("Could not create a voice connection offer."); + + session.cancelSession = this.platform.openSession( + { ...target, offerSdp: offer.sdp }, + { + onEvent: (event) => this.handleEvent(session, event), + onError: (message) => this.fail(session, message), + onEnd: () => this.end(session, null), + }, + ); + } catch (cause) { + this.fail(session, describeError(cause)); + } + } + + stop(): void { + const session = this.session; + if (!session) return; + this.teardown(session); + this.setState({ ...INITIAL_VOICE_MODE_STATE, muted: this.state.muted }); + } + + setMuted(muted: boolean): void { + for (const track of this.session?.microphone?.getAudioTracks() ?? []) track.enabled = !muted; + this.setState({ ...this.state, muted }); + } + + toggleMuted(): void { + this.setMuted(!this.state.muted); + } + + /** Clears a finished conversation's error or end notice. */ + dismiss(): void { + if (this.state.phase !== "idle") return; + this.setState({ ...this.state, error: null, endedReason: null }); + } + + private handleEvent(session: ActiveSession, event: ProviderVoiceSessionEvent): void { + if (this.session !== session) return; + switch (event.type) { + case "answer": + session.peer + ?.setRemoteDescription({ type: "answer", sdp: event.sdp } as never) + .catch((cause: unknown) => this.fail(session, describeError(cause))); + return; + case "started": + session.backendStarted = true; + this.maybeActivate(session); + return; + case "transcript.delta": + this.setState({ + ...this.state, + captions: appendVoiceCaption(this.state.captions, event.role, event.delta), + }); + return; + case "transcript.done": + this.setState({ + ...this.state, + captions: finishVoiceCaption(this.state.captions, event.role, event.text), + }); + return; + case "closed": + this.end(session, event.reason ?? null); + return; + } + } + + private maybeActivate(session: ActiveSession): void { + if (!session.backendStarted || !session.peerConnected || this.state.phase === "active") return; + if (session.connectTimer) clearTimeout(session.connectTimer); + session.connectTimer = null; + session.levelTimer = setInterval(() => this.sampleLevels(session), LEVEL_SAMPLE_INTERVAL_MS); + this.setState({ ...this.state, phase: "active" }); + } + + private sampleLevels(session: ActiveSession): void { + // Native getStats serializes the whole report, so never stack calls. + if (this.levelListeners.size === 0 || !session.peer || session.sampling) return; + session.sampling = true; + session.peer + .getStats() + .finally(() => { + session.sampling = false; + }) + .then((report) => { + if (this.session !== session) return; + let microphone = 0; + let speaker = 0; + report.forEach((stat) => { + const level = (stat as { readonly audioLevel?: unknown }).audioLevel; + if (typeof level !== "number" || stat.kind !== "audio") return; + if (stat.type === "media-source") microphone = Math.max(microphone, level); + else if (stat.type === "inbound-rtp") speaker = Math.max(speaker, level); + }); + const levels = { microphone: this.state.muted ? 0 : microphone, speaker }; + for (const listener of this.levelListeners) listener(levels); + }) + .catch(() => undefined); + } + + private end(session: ActiveSession, reason: string | null): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + endedReason: reason ?? "ended", + }); + } + + private fail(session: ActiveSession, message: string): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + error: message, + }); + } + + private teardown(session: ActiveSession): void { + if (this.session === session) this.session = null; + if (session.connectTimer) clearTimeout(session.connectTimer); + if (session.levelTimer) clearInterval(session.levelTimer); + session.cancelSession?.(); + session.stopRemoteAudio?.(); + session.peer?.close(); + for (const track of session.microphone?.getTracks() ?? []) track.stop(); + session.microphone?.release?.(); + for (const listener of this.levelListeners) listener({ microphone: 0, speaker: 0 }); + } + + private setState(next: VoiceModeState): void { + this.state = next; + // Read the latest state per listener: an earlier listener may have changed it. + for (const listener of [...this.listeners]) listener(this.state); + } +} diff --git a/packages/client-runtime/src/voice-mode/index.ts b/packages/client-runtime/src/voice-mode/index.ts new file mode 100644 index 000000000000..b71b0a282a5b --- /dev/null +++ b/packages/client-runtime/src/voice-mode/index.ts @@ -0,0 +1,19 @@ +export { + INITIAL_VOICE_MODE_STATE, + VOICE_CONNECT_TIMEOUT_MS, + VoiceModeController, + appendVoiceCaption, + describeVoiceEnd, + finishVoiceCaption, + type VoiceCaption, + type VoiceLevels, + type VoiceMediaStream, + type VoiceMediaTrack, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionHandlers, + type VoiceSessionTarget, +} from "./controller.ts"; +export { makeVoiceSessionOpener } from "./session.ts"; diff --git a/packages/client-runtime/src/voice-mode/session.ts b/packages/client-runtime/src/voice-mode/session.ts new file mode 100644 index 000000000000..4c316369d518 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/session.ts @@ -0,0 +1,54 @@ +import { EnvironmentId, ThreadId, WS_METHODS } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import type * as Fiber from "effect/Fiber"; +import * as Stream from "effect/Stream"; + +import { EnvironmentRegistry } from "../connection/registry.ts"; +import { runStream } from "../rpc/client.ts"; +import type { VoiceModePlatform } from "./controller.ts"; + +function failureMessage(cause: Cause.Cause): string { + const error = Cause.squash(cause); + if (error && typeof error === "object") { + if ("detail" in error && typeof error.detail === "string") return error.detail; + if ("message" in error && typeof error.message === "string" && error.message) { + return error.message; + } + } + return "Voice conversation failed."; +} + +/** + * Builds `VoiceModePlatform.openSession` on an app runtime that provides the + * environment registry. Interrupting the returned fiber ends the conversation + * server-side. + */ +export function makeVoiceSessionOpener( + runFork: (effect: Effect.Effect) => Fiber.Fiber, +): VoiceModePlatform["openSession"] { + return ({ environmentId, threadId, offerSdp }, handlers) => { + const fiber = runFork( + EnvironmentRegistry.pipe( + Effect.flatMap((registry) => + registry + .runStream( + EnvironmentId.make(environmentId), + runStream(WS_METHODS.providerVoiceSession, { + threadId: ThreadId.make(threadId), + offerSdp, + }), + ) + .pipe(Stream.runForEach((event) => Effect.sync(() => handlers.onEvent(event)))), + ), + Effect.matchCause({ + onFailure: (cause) => { + if (!Cause.hasInterruptsOnly(cause)) handlers.onError(failureMessage(cause)); + }, + onSuccess: () => handlers.onEnd(), + }), + ), + ); + return () => fiber.interruptUnsafe(); + }; +} diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts index 04c3acab803d..6b57540fec72 100644 --- a/packages/contracts/src/keybindings.ts +++ b/packages/contracts/src/keybindings.ts @@ -97,6 +97,8 @@ export const STATIC_KEYBINDING_COMMANDS = [ "composer.workspace", "composer.previousWorktree", "composer.branch", + "voice.toggle", + "voice.toggleMute", "chat.new", "chat.newLocal", "chat.newWithoutProject", diff --git a/packages/contracts/src/provider.ts b/packages/contracts/src/provider.ts index f48ca86fbf08..7bf66ef4e807 100644 --- a/packages/contracts/src/provider.ts +++ b/packages/contracts/src/provider.ts @@ -142,6 +142,58 @@ export class ProviderUploadFeedbackError extends Schema.TaggedError()( + "ProviderVoiceSessionError", + { + threadId: ThreadId, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.detail; + } +} + const ProviderEventKind = Schema.Literals(["session", "notification", "request", "error"]); export const ProviderEvent = Schema.Struct({ diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 61304c52f860..4f7f6f3b57dd 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -159,6 +159,9 @@ import { ProviderUploadFeedbackError, ProviderUploadFeedbackInput, ProviderUploadFeedbackResult, + ProviderVoiceSessionError, + ProviderVoiceSessionEvent, + ProviderVoiceSessionInput, } from "./provider.ts"; import { ProviderInstanceId, ProviderInstanceMutation } from "./providerInstance.ts"; import { PullRequestReviewGuideResult } from "./pullRequestReviewGuide.ts"; @@ -404,6 +407,7 @@ export const WS_METHODS = { // Provider methods providerUploadFeedback: "provider.uploadFeedback", + providerVoiceSession: "provider.voice.session", providerAuthStart: "provider.auth.start", providerConsumeResetCredit: "provider.consumeResetCredit", providerAuthComplete: "provider.auth.complete", @@ -1345,6 +1349,13 @@ const WsProviderUploadFeedbackRpc = Rpc.make(WS_METHODS.providerUploadFeedback, error: Schema.Union([ProviderUploadFeedbackError, EnvironmentAuthorizationError]), }); +const WsProviderVoiceSessionRpc = Rpc.make(WS_METHODS.providerVoiceSession, { + payload: ProviderVoiceSessionInput, + success: ProviderVoiceSessionEvent, + error: Schema.Union([ProviderVoiceSessionError, EnvironmentAuthorizationError]), + stream: true, +}); + const WsSubscribeVcsStatusRpc = Rpc.make(WS_METHODS.subscribeVcsStatus, { payload: VcsStatusSubscriptionInput, success: VcsStatusStreamEvent, @@ -2002,6 +2013,7 @@ export const WsRpcGroup = RpcGroup.make( WsMcpAppsReadResourceRpc, WsMcpAppsUpdateModelContextRpc, WsProviderUploadFeedbackRpc, + WsProviderVoiceSessionRpc, WsSubscribeVcsStatusRpc, WsSubscribeWorktreeSetupRpc, WsWorktreeSetupCancelRpc, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 985e6bbc126e..9ae5a404e33a 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -265,6 +265,8 @@ export const ServerProvider = Schema.Struct({ requiresNewThreadForModelChange: Schema.optional(Schema.Boolean), supportsConversationRollback: Schema.optional(Schema.Boolean), supportsTextGeneration: Schema.optional(Schema.Boolean), + // The driver can hold realtime voice conversations (`provider.voice.session`). + supportsVoice: Schema.optional(Schema.Boolean), setup: Schema.optional( Schema.Struct({ canAuthenticate: Schema.Boolean, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index c0de88f4ac6e..4e68afc92974 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -609,6 +609,21 @@ export function makeProviderSettingsSchema; + export const CodexSettings = makeProviderSettingsSchema( { setupMode: Schema.optionalKey(Schema.Literals(["managed", "existing"])).pipe( @@ -659,9 +674,21 @@ export const CodexSettings = makeProviderSettingsSchema( Schema.withDecodingDefault(Effect.succeed([])), Schema.annotateKey({ providerSettingsForm: { hidden: true } }), ), + voice: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Voice", + description: "Voice used by voice conversations. Applies to the next conversation.", + providerSettingsForm: { + control: "select", + options: CODEX_VOICE_OPTIONS, + clearWhenEmpty: "omit", + }, + }), + ), }, { - order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs"], + order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs", "voice"], }, ); export type CodexSettings = typeof CodexSettings.Type; diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 809dbfac0110..5dea781c1fa5 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -1,4 +1,7 @@ -import type { OrchestrationV2HistoricalMessage } from "@t3tools/contracts"; +import type { + OrchestrationV2HistoricalMessage, + ProviderVoiceSessionEvent, +} from "@t3tools/contracts"; import { ChatAttachment, CheckpointId, @@ -635,6 +638,14 @@ export interface ProviderAdapterV2SessionRuntime { * provider are not interactive and their tool calls stay plain tool rows. */ readonly mcpApps?: ProviderAdapterV2McpApps; + /** + * Providers with realtime voice (Codex) hold a voice conversation on the + * thread. Interrupting the stream ends the conversation. + */ + readonly startVoiceSession?: (input: { + readonly providerThread: OrchestrationV2ProviderThread; + readonly offerSdp: string; + }) => Stream.Stream; readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index 5d08f22c13fb..575aee173997 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -47,6 +47,11 @@ export interface ProviderContinuationRequest { * wakes a run was dispatched for. Otherwise the service allocates one. */ readonly messageId?: MessageId; + /** + * `voice` resubmits a request the user spoke in a voice conversation, so the + * run reads as theirs: user-authored, with no background notification. + */ + readonly origin?: "voice"; readonly dispatchIfCurrent?: ( effect: Effect.Effect, ) => Effect.Effect, E, R>; diff --git a/packages/provider-core/src/server/snapshotProbe.ts b/packages/provider-core/src/server/snapshotProbe.ts index 071d9fd5ae07..45c715ac028d 100644 --- a/packages/provider-core/src/server/snapshotProbe.ts +++ b/packages/provider-core/src/server/snapshotProbe.ts @@ -92,6 +92,7 @@ export interface ServerProviderPresentation { readonly supportedRuntimeModes?: ReadonlyArray; readonly requiresNewThreadForModelChange?: boolean; readonly supportsConversationRollback?: boolean; + readonly supportsVoice?: boolean; } export type ServerProviderDraft = Omit; @@ -258,6 +259,7 @@ export function buildServerProvider(input: { ...(typeof input.presentation.requiresNewThreadForModelChange === "boolean" ? { requiresNewThreadForModelChange: input.presentation.requiresNewThreadForModelChange } : {}), + ...(input.presentation.supportsVoice ? { supportsVoice: true } : {}), enabled: input.enabled, installed: input.probe.installed, version: input.probe.version, diff --git a/packages/shared/src/composerTrigger.ts b/packages/shared/src/composerTrigger.ts index 6153978edb92..4819452a587d 100644 --- a/packages/shared/src/composerTrigger.ts +++ b/packages/shared/src/composerTrigger.ts @@ -4,7 +4,7 @@ export type ComposerTriggerKind = | "slash-command" | "slash-model" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "voice"; export interface ComposerTrigger { kind: ComposerTriggerKind; diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts index 53a2aa5d3efe..7851ce03b56d 100644 --- a/packages/shared/src/keybindings.ts +++ b/packages/shared/src/keybindings.ts @@ -181,6 +181,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray = [ { key: "mod+shift+g", command: "composer.branch", when: "!terminalFocus" }, { key: "mod+shift+l", command: "composer.previousWorktree", when: "!terminalFocus" }, { key: "mod+shift+k", command: "pullRequest.copyNumber", when: "!terminalFocus" }, + { key: "f8", command: "voice.toggle", when: "!terminalFocus" }, { key: "mod+shift+arrowup", command: "modelPicker.previousProvider", when: "modelPickerOpen" }, { key: "mod+shift+arrowdown", command: "modelPicker.nextProvider", when: "modelPickerOpen" }, { key: "mod+o", command: "editor.openFavorite" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdb699e66b36..49a5736afa29 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -556,6 +556,9 @@ importers: react-native-svg: specifier: 15.15.5 version: 15.15.5(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0) + react-native-webrtc: + specifier: 124.0.8 + version: 124.0.8(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)) react-native-webview: specifier: 14.0.1 version: 14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0) @@ -581,6 +584,9 @@ importers: specifier: 1.12.1 version: 1.12.1(@expo/metro-config@58.0.6(patch_hash=90fa980e8b6480631fb2fbb04559834e3c612db40c550cd828f11be297d8d760)(bufferutil@4.1.0)(expo@58.0.2)(typescript@7.0.2)(utf-8-validate@6.0.6))(metro-cache@0.87.1)(metro-transform-worker@0.87.1(bufferutil@4.1.0)(utf-8-validate@6.0.6))(metro@0.87.1(bufferutil@4.1.0)(utf-8-validate@6.0.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0)(tailwindcss@4.3.3) devDependencies: + '@config-plugins/react-native-webrtc': + specifier: 15.0.2 + version: 15.0.2(expo@58.0.2) '@effect/vitest': specifier: 4.0.2 version: 4.0.2(patch_hash=595b7d26fb635caba9c3d39fcb189bb78a000192f8d3b60ead7174b310d1ade9)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.2(patch_hash=ec61ee307eaec3e17f757e0e6a16296ea4b7a8c3188df0c6ec5e66301fd78688))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) @@ -2825,6 +2831,20 @@ packages: '@cloudflare/workers-types@5.20261004.1': resolution: {integrity: sha512-K9D4/Y6bSIDNYFzjqzFMNoWK99pLgfgFxqSa76kfW++3VCkTSzJbzsQRffir366G3kmNyjoigC6ekp7DxIkxkw==} + '@cloudflare/workers-types@5.20260907.1': + resolution: {integrity: sha512-HYI7eFb/MOcNzvbUW7ZwHVZL5YFXb4r6yHN3vx8gJCqde7yhn3bhuPKiQPw90UtGWWp7uOI+Que3kf8Svjn5cw==} + + '@config-plugins/react-native-webrtc@15.0.2': + resolution: {integrity: sha512-sH4T7Z4P2RowV91k9CEwEr0unn+396NBexxCNZwRXuJXwguDU1qZWpE6fcwN0730B8uiS83F7+CLTyhpC7qRHQ==} + peerDependencies: + expo: '>=56' + + '@connectrpc/connect-node@1.7.0': + resolution: {integrity: sha512-6vaPIkG/NyhxlYgytLoR9KYbPhczEboFB2OYWkA9qvUz1K7efXfeGrlRxoLtpa+r8VxyIOw73w5ktNe743nD+A==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@bufbuild/protobuf': ^1.10.0 + '@connectrpc/connect': 1.7.0 '@coderabbitai/config@1.1.0': resolution: {integrity: sha512-NvQKaoOD7fWb7lMKhid57ABVlIBSczw/S+ePk5YvoPuWdrDkyY+PWdqVtq3TRq7VY8f3SNRlMFjmAtXVab66pw==} @@ -8030,6 +8050,15 @@ packages: supports-color: optional: true + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -10141,6 +10170,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -10915,6 +10947,11 @@ packages: peerDependencies: react-native: '*' + react-native-webrtc@124.0.8: + resolution: {integrity: sha512-uuQxvmk+mvnk5U0tr+1N42sKZqgm41fJrBA+fmCvML9J9P4roSh2So82t5RHAlu/vE9vxu5AKgivAiH61clCBg==} + peerDependencies: + react-native: '>=0.60.0' + react-native-webview@14.0.1: resolution: {integrity: sha512-lBUuvFk459Mo+5vUNqwzGvn+I8rDJE6zvOulG2hWWudeEOJfWTsQwOfuYnRu+27djUIXrESFfC+EQE3PGu+FWA==} peerDependencies: @@ -13636,6 +13673,17 @@ snapshots: '@cloudflare/workers-types@4.20260604.1': {} '@cloudflare/workers-types@5.20261004.1': {} + '@cloudflare/workers-types@5.20260907.1': {} + + '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.2)': + dependencies: + expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) + + '@connectrpc/connect-node@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': + dependencies: + '@bufbuild/protobuf': 1.10.0 + '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.0) + undici: 5.29.0 '@coderabbitai/config@1.1.0': dependencies: @@ -18674,6 +18722,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.3.4: + dependencies: + ms: 2.1.2 + debug@4.4.3: dependencies: ms: 2.1.3 @@ -21161,6 +21213,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2): @@ -22130,6 +22184,14 @@ snapshots: dependencies: react-native: 0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6) + react-native-webrtc@124.0.8(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)): + dependencies: + base64-js: 1.5.1 + debug: 4.3.4 + react-native: 0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6) + transitivePeerDependencies: + - supports-color + react-native-webview@14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0): dependencies: escape-string-regexp: 4.0.0 diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 2d86a9c89d5b..ba4e43f5d082 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -1972,6 +1972,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { assert.include(entitlements, "webcredentials:clerk.example.com"); assert.include(entitlements, "webcredentials:example.clerk.accounts.dev"); assert.include(entitlements, "com.apple.security.cs.allow-jit"); + assert.include(entitlements, "com.apple.security.device.audio-input"); assert.notInclude(entitlements, "keychain-access-groups"); assert.notInclude(entitlements, "com.apple.developer.web-browser.public-key-credential"); }); diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 618aec4d6bba..bd16f2e821aa 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1413,6 +1413,8 @@ ${associatedDomains} com.apple.security.cs.disable-library-validation + com.apple.security.device.audio-input + `; @@ -2835,6 +2837,8 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( LSItemContentTypes: ["public.html", "public.xhtml"], }, ], + NSMicrophoneUsageDescription: + "T3 Code uses the microphone when you start a voice conversation with an agent.", }, protocols: [ { diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index f247cf5c17c8..b8a033368cbf 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -121,6 +121,21 @@ } ] }, + { + "bundles": ["android", "ios", "mobile"], + "license": "BSD-3-Clause", + "name": "WebRTC", + "sourceUrl": "https://webrtc.googlesource.com/src", + "generatedNotices": [ + { + "licenseId": "BSD-3-Clause", + "copyrights": ["Copyright (c) 2011, The WebRTC project authors. All rights reserved."], + "preamble": [ + "WebRTC\n\nVoice conversations use the libwebrtc build that react-native-webrtc links into the iOS and Android apps." + ] + } + ] + }, { "bundles": ["android", "assets", "mobile"], "license": "Apache-2.0", From 7473fab546169328d61aef001ee6c17984ec2203 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 22 Sep 2026 20:33:39 -0400 Subject: [PATCH 02/15] fix: harden voice handoff resubmits and mobile voice controls - Resubmit a Codex-started voice turn only once it reports interrupted, so a turn that finished before the interrupt landed never runs twice. - Carry each pending resubmit's own provider thread, and keep a replaced voice stream from clearing its successor's thread. - Mobile: gate voice on the provider thread existing, keep the voice strip reachable while a question card hides the composer, and keep a newer conversation's wake lock. - Drop unused exports flagged by knip and update the Codex settings fields test. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/features/threads/ThreadComposer.tsx | 5 +- .../features/threads/ThreadDetailScreen.tsx | 12 +- .../src/features/voice-mode/voiceMode.ts | 5 +- .../Adapters/CodexAdapterV2.test.ts | 114 ++++++++++++++++++ .../Adapters/CodexAdapterV2.ts | 38 ++++-- .../settings/ProviderSettingsForm.test.ts | 1 + apps/web/src/voice/voiceAvailability.ts | 2 +- .../src/voice-mode/controller.ts | 2 +- .../client-runtime/src/voice-mode/index.ts | 3 - packages/contracts/src/settings.ts | 2 +- 10 files changed, 163 insertions(+), 21 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index fe7c1ed15358..e101fccc0aac 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -538,11 +538,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer }); const offersVoiceConversation = voiceModeSupported && selectedProviderStatus?.supportsVoice === true; - // The server attaches voice to the thread's provider session, which the first message creates. + // The server attaches voice to the thread's provider thread, which the first + // run creates. An optimistic pending thread already has a message timestamp. const canToggleVoiceConversation = voiceConversationLive || (props.connectionState === "connected" && - props.selectedThread.latestUserMessageAt !== null && + props.selectedThread.activeProviderThreadId !== null && !voiceInput.isBusy); const toggleVoiceConversation = useCallback(() => voiceMode.toggle(voiceTarget), [voiceTarget]); const voiceModeButton = offersVoiceConversation ? ( diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index e4f489115f3a..f7feb30305d3 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -149,6 +149,8 @@ import { useThreadQueuedCount } from "./ThreadQueueControl"; import type { ThreadContentPresentation } from "./threadContentPresentation"; import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow"; import { useGlobalVoiceInput } from "../voice-input/VoiceInputProvider"; +import { ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode"; +import { useVoiceModePhase } from "../voice-mode/useVoiceMode"; export interface ThreadDetailScreenProps { readonly worktreeSetup?: WorktreeSetupCardProps | null; @@ -600,6 +602,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread // A pending user-input request or a failed creation owns the composer slot. const composerSlotHidden = activeUserInputRequestId !== null || props.creationState?.kind === "failed"; + const voiceTarget = useMemo( + () => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }), + [props.environmentId, props.selectedThread.id], + ); + const voiceLiveHere = useVoiceModePhase(voiceTarget) !== "idle"; // The open /usage-limits panel for this thread, model and turn. Only the open // moment is stored: the rows read live provider data, so a redeemed reset // credit or refreshed probe shows through. Anything that spends quota closes @@ -1386,7 +1393,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread ) : null} - {/* Hidden (not unmounted) while a user-input request owns the + {/* The composer below hides for a question card; keep a live + voice conversation's mute and stop controls reachable. */} + {activeUserInputRequestId !== null && voiceLiveHere ? : null} + {/* Hidden (not unmounted) while a user-input request owns the composer slot, so composer drafts and editor state survive. A rejected creation has no thread to send to; the failure card owns the slot instead. */} diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index f2eea23b8392..f66b9f388539 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -105,7 +105,10 @@ voiceMode.subscribe(({ phase }) => { } const activation = keepAwake; keepAwake = null; - activation?.then(() => deactivateKeepAwake(KEEP_AWAKE_TAG)).catch(() => undefined); + // Every conversation shares the tag, so a newer one must keep its lock. + activation + ?.then(() => (keepAwake === null ? deactivateKeepAwake(KEEP_AWAKE_TAG) : undefined)) + .catch(() => undefined); } }); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 9a1c8f6c900f..d8e7feb30410 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2032,6 +2032,120 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }); + it.effect("resubmits a voice handoff Codex started only after its turn is interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "voice-thread"; + const delegation = (input: string) => + `\n ${input}\n`; + const voiceTurn = ( + turnId: string, + request: string, + id: number, + status: "completed" | "interrupted", + ) => [ + { + type: "emit_inbound" as const, + label: `turn/started/${turnId}`, + frame: { + method: "turn/started", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound" as const, + label: `item/started/${turnId}`, + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId, + startedAtMs: 1782622440500, + item: { + type: "userMessage", + id: `user-${turnId}`, + content: [{ type: "text", text: delegation(request), text_elements: [] }], + }, + }, + }, + }, + { + type: "expect_outbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, method: "turn/interrupt", params: { threadId: nativeThreadId, turnId } }, + }, + { + type: "emit_inbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, result: {} }, + }, + { + type: "emit_inbound" as const, + label: `turn/completed/${turnId}`, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status }), + }, + }, + }, + ]; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "voice-handoff-resubmit", + entries: [ + // Initialization and thread/start only; Codex, not T3, starts these turns. + ...codexReplayPreamble({ nativeThreadId, nativeTurnId: "unused", prompt: "" }).slice( + 0, + -3, + ), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + }, + }, + }, + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: { id: 3, result: {} }, + }, + // The interrupt lost the race: Codex finished, so it must not run twice. + ...voiceTurn("voice-raced", "already done", 4, "completed"), + ...voiceTurn("voice-stopped", "run the tests", 5, "interrupted"), + ], + }), + ); + yield* harness.runtime.startVoiceSession!({ + providerThread: harness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe(Stream.runDrain, Effect.forkScoped); + + yield* awaitUntil(() => harness.continuationRequests.length === 1, "voice resubmit"); + assert.lengthOf(harness.continuationRequests, 1); + const request = harness.continuationRequests[0]; + assert.equal(request?.origin, "voice"); + assert.equal(request?.delivery, "message_text"); + assert.equal(request?.detail, "run the tests"); + assert.equal(request?.providerThreadId, harness.providerThread.id); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect.each(["supported", "unsupported", "invalid"] as const)( "delivers native history with %s app-server protocol", (response) => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 5780acc8fc93..1919cb048c01 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1883,8 +1883,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const voiceProviderThread = yield* Ref.make( undefined, ); - /** Native turns Codex started for a voice handoff -> the spoken request. */ - const voiceResubmits = yield* Ref.make(new Map()); + /** Native turns Codex started for a voice handoff -> the request and its thread. */ + const voiceResubmits = yield* Ref.make( + new Map< + string, + { readonly text: string; readonly providerThread: OrchestrationV2ProviderThread } + >(), + ); const finalAnswerItemIdsByTurn = yield* Ref.make(new Map>()); const completedFinalAnswerTextsByTurn = yield* Ref.make(new Map>()); // Native completion and the interrupt timeout share one finalization @@ -4622,7 +4627,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const voiceThread = yield* Ref.get(voiceProviderThread); if ( handoff !== undefined && - voiceThread?.nativeThreadRef?.nativeId === payload.threadId && + voiceThread !== undefined && + voiceThread.nativeThreadRef?.nativeId === payload.threadId && !(yield* Ref.get(activeTurns)).has(payload.turnId) && !(yield* Ref.get(pendingRootTurns)).has(payload.threadId) ) { @@ -4630,7 +4636,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // are orchestrator-owned, so stop it and resubmit the request // as an ordinary run once it settles (see turn/completed). yield* Ref.update(voiceResubmits, (current) => - new Map(current).set(payload.turnId, handoff), + new Map(current).set(payload.turnId, { + text: handoff, + providerThread: voiceThread, + }), ); yield* client .request("turn/interrupt", { threadId: payload.threadId, turnId: payload.turnId }) @@ -6011,14 +6020,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updated.delete(payload.turn.id); return updated; }); - const voiceThread = yield* Ref.get(voiceProviderThread); - if (voiceThread !== undefined && continuationRequests !== undefined) { - yield* voice.expectSpokenTurn(voiceResubmit); + // Resubmit only a turn Codex confirms it stopped. If the + // interrupt lost the race, Codex already did the work (and the + // voice speaks its answer), so running it again would repeat it. + if (payload.turn.status === "interrupted" && continuationRequests !== undefined) { + yield* voice.expectSpokenTurn(voiceResubmit.text); yield* continuationRequests.offer({ - threadId: voiceThread.appThreadId ?? input.threadId, - providerThreadId: voiceThread.id, + threadId: voiceResubmit.providerThread.appThreadId ?? input.threadId, + providerThreadId: voiceResubmit.providerThread.id, driver: CODEX_PROVIDER, - detail: voiceResubmit, + detail: voiceResubmit.text, delivery: "message_text", origin: "voice", }); @@ -7201,7 +7212,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }); }), ).pipe( - Stream.ensuring(Ref.set(voiceProviderThread, undefined)), + // A replaced conversation must not clear its successor's thread. + Stream.ensuring( + Ref.update(voiceProviderThread, (current) => + current === voiceInput.providerThread ? undefined : current, + ), + ), Stream.mapError((cause) => cause._tag === "ProviderAdapterProtocolError" ? cause diff --git a/apps/web/src/components/settings/ProviderSettingsForm.test.ts b/apps/web/src/components/settings/ProviderSettingsForm.test.ts index ff1de91addc9..b7100208bb88 100644 --- a/apps/web/src/components/settings/ProviderSettingsForm.test.ts +++ b/apps/web/src/components/settings/ProviderSettingsForm.test.ts @@ -17,6 +17,7 @@ describe("ProviderSettingsForm helpers", () => { "homePath", "shadowHomePath", "launchArgs", + "voice", ]); }); diff --git a/apps/web/src/voice/voiceAvailability.ts b/apps/web/src/voice/voiceAvailability.ts index c376510845f4..1e919f3b8d20 100644 --- a/apps/web/src/voice/voiceAvailability.ts +++ b/apps/web/src/voice/voiceAvailability.ts @@ -9,7 +9,7 @@ export type VoiceAvailability = | { readonly kind: "unavailable"; readonly reason: string } | { readonly kind: "ready" }; -export const VOICE_UNSUPPORTED: VoiceAvailability = { kind: "unsupported" }; +const VOICE_UNSUPPORTED: VoiceAvailability = { kind: "unsupported" }; export function resolveVoiceAvailability(input: { /** The provider instance that runs (or will run) the thread's turns. */ diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts index 42ab725dbcdf..8605cc340ff2 100644 --- a/packages/client-runtime/src/voice-mode/controller.ts +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -92,7 +92,7 @@ export interface VoiceLevels { readonly speaker: number; } -export const VOICE_CONNECT_TIMEOUT_MS = 20_000; +const VOICE_CONNECT_TIMEOUT_MS = 20_000; const LEVEL_SAMPLE_INTERVAL_MS = 100; const MAX_CAPTIONS = 4; const MAX_CAPTION_CHARS = 1024; diff --git a/packages/client-runtime/src/voice-mode/index.ts b/packages/client-runtime/src/voice-mode/index.ts index b71b0a282a5b..6d05d8bf22dc 100644 --- a/packages/client-runtime/src/voice-mode/index.ts +++ b/packages/client-runtime/src/voice-mode/index.ts @@ -1,10 +1,7 @@ export { INITIAL_VOICE_MODE_STATE, - VOICE_CONNECT_TIMEOUT_MS, VoiceModeController, - appendVoiceCaption, describeVoiceEnd, - finishVoiceCaption, type VoiceCaption, type VoiceLevels, type VoiceMediaStream, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 4e68afc92974..5d69bdb36359 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -611,7 +611,7 @@ export function makeProviderSettingsSchema Date: Tue, 22 Sep 2026 20:40:57 -0400 Subject: [PATCH 03/15] refactor: use Toggle for the voice button and a schema voice error Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/provider/CodexVoice.test.ts | 3 ++- apps/server/src/provider/CodexVoice.ts | 10 ++++++---- apps/web/src/components/chat/ChatComposer.tsx | 17 ++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts index 528db80bd354..fdc1a2daa2ce 100644 --- a/apps/server/src/provider/CodexVoice.test.ts +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -6,6 +6,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as CodexClient from "effect-codex-app-server/client"; @@ -183,7 +184,7 @@ describe("makeCodexVoice", () => { const exit = yield* Fiber.await(voice.fiber); NodeAssert.ok(Exit.isFailure(exit)); const error = exit.cause.reasons[0]; - NodeAssert.ok(error?._tag === "Fail" && error.error instanceof CodexVoiceSessionError); + NodeAssert.ok(error?._tag === "Fail" && Schema.is(CodexVoiceSessionError)(error.error)); NodeAssert.equal(error.error.detail, "realtime unavailable"); }), ); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts index f43ae817aa3c..31bec1152d25 100644 --- a/apps/server/src/provider/CodexVoice.ts +++ b/apps/server/src/provider/CodexVoice.ts @@ -10,10 +10,10 @@ */ import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; -import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as CodexClient from "effect-codex-app-server/client"; @@ -86,9 +86,11 @@ function transcriptRole(role: string): "user" | "assistant" { return role === "user" ? "user" : "assistant"; } -export class CodexVoiceSessionError extends Data.TaggedError("CodexVoiceSessionError")<{ - readonly detail: string; -}> { +/** Codex ended the realtime session with an error; `detail` is Codex's explanation for the user. */ +export class CodexVoiceSessionError extends Schema.TaggedError()( + "CodexVoiceSessionError", + { detail: Schema.String }, +) { override get message(): string { return this.detail; } diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 8e868f5daa3a..c8f68968b204 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1080,6 +1080,7 @@ function ComposerCommandMenuLayer(props: { anchor: HTMLElement | null; children: ); } import { Button } from "../ui/button"; +import { Toggle } from "../ui/toggle"; import { Select, SelectItem, SelectPopup, SelectValue } from "../ui/select"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { toastManager } from "../ui/toast"; @@ -7767,19 +7768,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) event.preventDefault()} - onClick={voiceButtonDisabledReason === null ? toggleVoice : undefined} + onPressedChange={() => { + if (voiceButtonDisabledReason === null) toggleVoice(); + }} aria-disabled={voiceButtonDisabledReason !== null || undefined} - aria-pressed={voiceEngaged} aria-label={voiceButtonLabel} /> } From a3dda807f5e54e577ccfd2e5652d82b871418661 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 22 Sep 2026 21:04:57 -0400 Subject: [PATCH 04/15] fix(web): dim the disabled voice glyph instead of restyling Toggle Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/components/chat/ChatComposer.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index c8f68968b204..2073184ccb5d 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -7772,7 +7772,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) variant="ghost" size="sm" // aria-disabled rather than disabled keeps the tooltip explaining why. - className="aria-disabled:cursor-not-allowed aria-disabled:opacity-64" + className="aria-disabled:cursor-not-allowed" pressed={voiceEngaged} onPointerDown={(event) => event.preventDefault()} onPressedChange={() => { @@ -7783,7 +7783,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) /> } > - + {/* Dim the glyph, not the Toggle, which owns its own disabled styling. */} + {voiceButtonDisabledReason ?? voiceButtonLabel} From bc3999c5610c26d1a4efa3cee6912c44f485a813 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 22 Sep 2026 21:11:33 -0400 Subject: [PATCH 05/15] fix: keep failed voice handoffs and let /voice always stop a live conversation - Resubmit a Codex-started voice turn that failed, not only one that was interrupted, so the spoken request is never silently dropped. - /voice stops an active conversation even if the provider no longer reports voice support. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Adapters/CodexAdapterV2.test.ts | 214 +++++++++--------- .../Adapters/CodexAdapterV2.ts | 12 +- apps/web/src/components/ChatView.tsx | 5 +- 3 files changed, 122 insertions(+), 109 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index d8e7feb30410..96d6126dc1e5 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2032,118 +2032,126 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }); - it.effect("resubmits a voice handoff Codex started only after its turn is interrupted", () => - Effect.scoped( - Effect.gen(function* () { - const nativeThreadId = "voice-thread"; - const delegation = (input: string) => - `\n ${input}\n`; - const voiceTurn = ( - turnId: string, - request: string, - id: number, - status: "completed" | "interrupted", - ) => [ - { - type: "emit_inbound" as const, - label: `turn/started/${turnId}`, - frame: { - method: "turn/started", - params: { - threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: turnId, status: "inProgress" }), + it.effect( + "resubmits a voice handoff Codex started only once its turn is interrupted or failed", + () => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "voice-thread"; + const delegation = (input: string) => + `\n ${input}\n`; + const voiceTurn = ( + turnId: string, + request: string, + id: number, + status: "completed" | "interrupted" | "failed", + ) => [ + { + type: "emit_inbound" as const, + label: `turn/started/${turnId}`, + frame: { + method: "turn/started", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status: "inProgress" }), + }, }, }, - }, - { - type: "emit_inbound" as const, - label: `item/started/${turnId}`, - frame: { - method: "item/started", - params: { - threadId: nativeThreadId, - turnId, - startedAtMs: 1782622440500, - item: { - type: "userMessage", - id: `user-${turnId}`, - content: [{ type: "text", text: delegation(request), text_elements: [] }], + { + type: "emit_inbound" as const, + label: `item/started/${turnId}`, + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId, + startedAtMs: 1782622440500, + item: { + type: "userMessage", + id: `user-${turnId}`, + content: [{ type: "text", text: delegation(request), text_elements: [] }], + }, }, }, }, - }, - { - type: "expect_outbound" as const, - label: `turn/interrupt/${turnId}`, - frame: { id, method: "turn/interrupt", params: { threadId: nativeThreadId, turnId } }, - }, - { - type: "emit_inbound" as const, - label: `turn/interrupt/${turnId}`, - frame: { id, result: {} }, - }, - { - type: "emit_inbound" as const, - label: `turn/completed/${turnId}`, - frame: { - method: "turn/completed", - params: { - threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: turnId, status }), + { + type: "expect_outbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, method: "turn/interrupt", params: { threadId: nativeThreadId, turnId } }, + }, + { + type: "emit_inbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, result: {} }, + }, + { + type: "emit_inbound" as const, + label: `turn/completed/${turnId}`, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status }), + }, }, }, - }, - ]; - const harness = yield* makeCodexReplayHarness( - makeCodexReplayTranscript({ - scenario: "voice-handoff-resubmit", - entries: [ - // Initialization and thread/start only; Codex, not T3, starts these turns. - ...codexReplayPreamble({ nativeThreadId, nativeTurnId: "unused", prompt: "" }).slice( - 0, - -3, - ), - { - type: "expect_outbound", - label: "thread/realtime/start", - frame: { - id: 3, - method: "thread/realtime/start", - params: { - threadId: nativeThreadId, - outputModality: "audio", - transport: { type: "webrtc", sdp: "v=offer\r\n" }, - version: "v3", - clientManagedHandoffs: true, - includeStartupContext: false, + ]; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "voice-handoff-resubmit", + entries: [ + // Initialization and thread/start only; Codex, not T3, starts these turns. + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused", + prompt: "", + }).slice(0, -3), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + }, }, }, - }, - { - type: "emit_inbound", - label: "thread/realtime/start", - frame: { id: 3, result: {} }, - }, - // The interrupt lost the race: Codex finished, so it must not run twice. - ...voiceTurn("voice-raced", "already done", 4, "completed"), - ...voiceTurn("voice-stopped", "run the tests", 5, "interrupted"), - ], - }), - ); - yield* harness.runtime.startVoiceSession!({ - providerThread: harness.providerThread, - offerSdp: "v=offer\r\n", - }).pipe(Stream.runDrain, Effect.forkScoped); - - yield* awaitUntil(() => harness.continuationRequests.length === 1, "voice resubmit"); - assert.lengthOf(harness.continuationRequests, 1); - const request = harness.continuationRequests[0]; - assert.equal(request?.origin, "voice"); - assert.equal(request?.delivery, "message_text"); - assert.equal(request?.detail, "run the tests"); - assert.equal(request?.providerThreadId, harness.providerThread.id); - }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), - ), + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: { id: 3, result: {} }, + }, + // The interrupt lost the race: Codex finished, so it must not run twice. + ...voiceTurn("voice-raced", "already done", 4, "completed"), + ...voiceTurn("voice-stopped", "run the tests", 5, "interrupted"), + // A failed turn never did the work either, so its request is kept. + ...voiceTurn("voice-failed", "open a pull request", 6, "failed"), + ], + }), + ); + yield* harness.runtime.startVoiceSession!({ + providerThread: harness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe(Stream.runDrain, Effect.forkScoped); + + yield* awaitUntil(() => harness.continuationRequests.length === 2, "voice resubmits"); + assert.deepEqual( + harness.continuationRequests.map((entry) => entry.detail), + ["run the tests", "open a pull request"], + ); + const request = harness.continuationRequests[0]; + assert.equal(request?.origin, "voice"); + assert.equal(request?.delivery, "message_text"); + assert.equal(request?.detail, "run the tests"); + assert.equal(request?.providerThreadId, harness.providerThread.id); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), ); it.effect.each(["supported", "unsupported", "invalid"] as const)( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 1919cb048c01..5ecb468b942a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -6020,10 +6020,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updated.delete(payload.turn.id); return updated; }); - // Resubmit only a turn Codex confirms it stopped. If the - // interrupt lost the race, Codex already did the work (and the - // voice speaks its answer), so running it again would repeat it. - if (payload.turn.status === "interrupted" && continuationRequests !== undefined) { + // Resubmit a turn Codex stopped or failed, so the request isn't + // lost. If the interrupt lost the race and the turn completed, + // Codex already did the work (and the voice speaks its answer), + // so running it again would repeat it. + if ( + (payload.turn.status === "interrupted" || payload.turn.status === "failed") && + continuationRequests !== undefined + ) { yield* voice.expectSpokenTurn(voiceResubmit.text); yield* continuationRequests.offer({ threadId: voiceResubmit.providerThread.appThreadId ?? input.threadId, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 06d091c41f63..76e299a09413 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -188,7 +188,7 @@ import { parseStandaloneComposerSlashCommand, } from "../composer-logic"; import { resolveThreadVoiceAvailability } from "../voice/voiceAvailability"; -import { toggleThreadVoice } from "../voice/voiceMode"; +import { toggleThreadVoice, voiceMode } from "../voice/voiceMode"; import { derivePendingApprovals, derivePendingUserInputs, @@ -9244,7 +9244,8 @@ export default function ChatView(props: ChatViewProps) { if (parsedSlashCommand === "voice") { // Providers without voice receive "/voice" as a normal message. const voiceAvailability = resolveThreadVoiceAvailability(activeThread, providerStatuses); - if (voiceAvailability.kind !== "unsupported") { + // An active conversation always stops, even if the provider stopped offering voice. + if (voiceAvailability.kind !== "unsupported" || voiceMode.isActiveFor(routeThreadRef)) { toggleThreadVoice(routeThreadRef, voiceAvailability); promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); From 5e14eb0851116d775a9f36c41c048b46976345a2 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Thu, 24 Sep 2026 20:04:29 -0400 Subject: [PATCH 06/15] fix(server): don't let a conditional default shadow a user's unconditional shortcut Startup default sync only treated identical `when` strings as conflicts, so a default like `f8 voice.toggle when !terminalFocus` was appended after a user's bare `f8` binding and won whenever the terminal lacked focus. A default is now skipped when its shortcut matches a user rule and either side is unconditional. Also corrects the voice lifecycle comment: screens pushed over the thread keep a conversation running. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/features/voice-mode/useVoiceMode.ts | 3 ++- apps/server/src/keybindings.test.ts | 18 ++++++++++++++ apps/server/src/keybindings.ts | 24 +++++++++---------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts index f0adc61fd6f3..ee6507cda85b 100644 --- a/apps/mobile/src/features/voice-mode/useVoiceMode.ts +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -26,7 +26,8 @@ export function useVoiceModePhase(target: VoiceSessionTarget): VoiceModePhase { /** * Ties a conversation to the screen showing its thread: leaving the thread - * ends it, so the microphone is never live without its controls on screen. + * ends it. Screens pushed over the thread (files, review, settings) keep it + * running, like a call; its controls return with the thread. * Also reports failures and remote endings for this thread, once. */ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { diff --git a/apps/server/src/keybindings.test.ts b/apps/server/src/keybindings.test.ts index 0ea44fb3ca07..82b79f42b887 100644 --- a/apps/server/src/keybindings.test.ts +++ b/apps/server/src/keybindings.test.ts @@ -397,6 +397,24 @@ it.layer(NodeServices.layer)("keybindings", (it) => { ); }); + it.effect("skips a conditional default that would shadow an unconditional user shortcut", () => + Effect.gen(function* () { + const { keybindingsConfigPath } = yield* ServerConfig.ServerConfig; + yield* writeKeybindingsConfig(keybindingsConfigPath, [ + { key: "f8", command: "script.custom-action.run" }, + ]); + + yield* Effect.gen(function* () { + const keybindings = yield* Keybindings.Keybindings; + yield* keybindings.syncDefaultKeybindingsOnStartup; + }); + + const persisted = yield* readKeybindingsConfig(keybindingsConfigPath); + assert.isFalse(persisted.some((entry) => entry.command === "voice.toggle")); + assert.isTrue(persisted.some((entry) => entry.command === "navigation.back")); + }).pipe(Effect.provide(makeKeybindingsLayer())), + ); + it.effect("upserts custom keybindings to configured path", () => Effect.gen(function* () { const { keybindingsConfigPath } = yield* ServerConfig.ServerConfig; diff --git a/apps/server/src/keybindings.ts b/apps/server/src/keybindings.ts index 496a5852979a..132c199c0b69 100644 --- a/apps/server/src/keybindings.ts +++ b/apps/server/src/keybindings.ts @@ -129,19 +129,19 @@ const LATE_DEFAULT_KEYBINDINGS: ReadonlyArray<{ }, ]; -function keybindingShortcutContext(rule: KeybindingRule): string | null { +function encodedRuleShortcut(rule: KeybindingRule): string | null { const parsed = parseKeybindingShortcut(rule.key); - if (!parsed) return null; - const encoded = encodeShortcut(parsed); - if (!encoded) return null; - return `${encoded}\u0000${rule.when ?? ""}`; + return parsed ? encodeShortcut(parsed) : null; } -function hasSameShortcutContext(left: KeybindingRule, right: KeybindingRule): boolean { - const leftContext = keybindingShortcutContext(left); - const rightContext = keybindingShortcutContext(right); - if (!leftContext || !rightContext) return false; - return leftContext === rightContext; +/** + * Whether appending `rule` after `existing` would shadow it: same shortcut, and + * either the same `when` or one of them applies in every context. + */ +function shadowsShortcut(existing: KeybindingRule, rule: KeybindingRule): boolean { + const existingShortcut = encodedRuleShortcut(existing); + if (!existingShortcut || existingShortcut !== encodedRuleShortcut(rule)) return false; + return existing.when === undefined || rule.when === undefined || existing.when === rule.when; } function keybindingRuleFromUpsertInput(input: ServerUpsertKeybindingInput): KeybindingRule { @@ -544,9 +544,7 @@ const make = Effect.gen(function* () { if (existingCommands.has(defaultRule.command)) { continue; } - const conflictingEntry = customConfig.find((entry) => - hasSameShortcutContext(entry, defaultRule), - ); + const conflictingEntry = customConfig.find((entry) => shadowsShortcut(entry, defaultRule)); if (conflictingEntry) { shortcutConflictWarnings.push({ defaultCommand: defaultRule.command, From 5e485b1f0efdfc71e25d064320d5921cfd642b4e Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Thu, 1 Oct 2026 15:05:54 -0400 Subject: [PATCH 07/15] fix: follow the base's namespace imports in voice code The base now imports IdAllocator, ProviderContinuationRequests, and EnvironmentRegistry as namespaces. Update the three voice call sites that still used the old named imports. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/mobile/src/features/voice-mode/voiceMode.ts | 4 +++- .../src/orchestration-v2/Adapters/CodexAdapterV2.test.ts | 2 +- .../src/orchestration-v2/ProviderContinuationService.test.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index f66b9f388539..279ed3e5cb2c 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -28,7 +28,9 @@ function loadWebRTC(): typeof WebRTC { } /** Runs the voice stream on the connection runtime, holding it for the conversation's lifetime. */ -function runOnConnectionRuntime(effect: Effect.Effect) { +function runOnConnectionRuntime( + effect: Effect.Effect, +) { return Effect.runFork( Effect.gen(function* () { yield* AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 96d6126dc1e5..f010b1c187c6 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2150,7 +2150,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(request?.delivery, "message_text"); assert.equal(request?.detail, "run the tests"); assert.equal(request?.providerThreadId, harness.providerThread.id); - }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + }).pipe(Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))), ), ); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index b6723cd3cd7b..671733615d03 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -248,7 +248,7 @@ describe("ProviderContinuationService", () => { return Effect.gen(function* () { const dispatched = yield* Queue.unbounded(); yield* Effect.gen(function* () { - const requests = yield* ProviderContinuationRequests; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; yield* requests.offer({ threadId, providerThreadId, From 82f509753a305e451cc3df4d8ae7ac168bb04998 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sun, 4 Oct 2026 14:43:42 -0400 Subject: [PATCH 08/15] fix(server): preserve late defaults with voice shortcut migration --- apps/server/src/keybindings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/keybindings.ts b/apps/server/src/keybindings.ts index 132c199c0b69..8d46c958b748 100644 --- a/apps/server/src/keybindings.ts +++ b/apps/server/src/keybindings.ts @@ -561,7 +561,7 @@ const make = Effect.gen(function* () { for (const { alongside, rule } of pendingLateDefaults) { if ( customConfig.some((entry) => isSameKeybindingRule(entry, alongside)) && - !customConfig.some((entry) => hasSameShortcutContext(entry, rule)) + !customConfig.some((entry) => shadowsShortcut(entry, rule)) ) { missingDefaults.push(rule); } From 2004c4044d17beda16b0063b568a1a01bd27aca7 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sun, 4 Oct 2026 14:44:39 -0400 Subject: [PATCH 09/15] fix(mobile): combine unused Android permission exclusions --- apps/mobile/app.config.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 4ee5bd63acbd..082938cc5566 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -314,14 +314,15 @@ const config: ExpoConfig = { foregroundImage: variant.assets.androidAdaptiveForeground, monochromeImage: variant.assets.androidMonochromeIcon, }, - // react-native-webrtc's plugin requests overlay access for screen sharing, which voice does not use. - blockedPermissions: ["android.permission.SYSTEM_ALERT_WINDOW"], // Opts into OnBackInvokedCallback-based back dispatch (Android 13+). // JS back handling survives it via react-native's Android 16 shim plus // withAndroidPredictiveBackCompat on Android 13-15. predictiveBackGestureEnabled: true, - // expo-sensors declares this for its pedometer, which the app does not use. - blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"], + // The app uses neither expo-sensors' pedometer nor WebRTC screen sharing. + blockedPermissions: [ + "android.permission.ACTIVITY_RECOGNITION", + "android.permission.SYSTEM_ALERT_WINDOW", + ], }, web: { favicon: variant.assets.appIcon, From a2d2452507759a5257df6950e3879aace6b68ad0 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 5 Oct 2026 21:24:58 -0400 Subject: [PATCH 10/15] fix(voice): adapt reactivity imports and lockfile to current upstream --- apps/mobile/src/features/voice-mode/voiceMode.ts | 2 +- apps/web/src/voice/voiceMode.ts | 2 +- pnpm-lock.yaml | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts index 279ed3e5cb2c..72c2434d6c11 100644 --- a/apps/mobile/src/features/voice-mode/voiceMode.ts +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -6,7 +6,7 @@ import { type VoicePeerConnection, } from "@t3tools/client-runtime/voice-mode"; import * as Effect from "effect/Effect"; -import { AtomRegistry } from "effect/unstable/reactivity"; +import { AtomRegistry } from "effect/reactivity"; import { setAudioModeAsync, setIsAudioActiveAsync } from "expo-audio"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { AppState, NativeModules } from "react-native"; diff --git a/apps/web/src/voice/voiceMode.ts b/apps/web/src/voice/voiceMode.ts index 9a5defee23b0..e0b02ee04fe4 100644 --- a/apps/web/src/voice/voiceMode.ts +++ b/apps/web/src/voice/voiceMode.ts @@ -12,7 +12,7 @@ import { type VoiceSessionTarget, } from "@t3tools/client-runtime/voice-mode"; import * as Effect from "effect/Effect"; -import { AtomRegistry } from "effect/unstable/reactivity"; +import { AtomRegistry } from "effect/reactivity"; import { useSyncExternalStore } from "react"; import { stackedThreadToast, toastManager } from "../components/ui/toast"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 49a5736afa29..04ff1e39e632 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2831,8 +2831,6 @@ packages: '@cloudflare/workers-types@5.20261004.1': resolution: {integrity: sha512-K9D4/Y6bSIDNYFzjqzFMNoWK99pLgfgFxqSa76kfW++3VCkTSzJbzsQRffir366G3kmNyjoigC6ekp7DxIkxkw==} - '@cloudflare/workers-types@5.20260907.1': - resolution: {integrity: sha512-HYI7eFb/MOcNzvbUW7ZwHVZL5YFXb4r6yHN3vx8gJCqde7yhn3bhuPKiQPw90UtGWWp7uOI+Que3kf8Svjn5cw==} '@config-plugins/react-native-webrtc@15.0.2': resolution: {integrity: sha512-sH4T7Z4P2RowV91k9CEwEr0unn+396NBexxCNZwRXuJXwguDU1qZWpE6fcwN0730B8uiS83F7+CLTyhpC7qRHQ==} @@ -13673,7 +13671,6 @@ snapshots: '@cloudflare/workers-types@4.20260604.1': {} '@cloudflare/workers-types@5.20261004.1': {} - '@cloudflare/workers-types@5.20260907.1': {} '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.2)': dependencies: From 8641915ab48ccf3fca425ad9954ce93dc6366f83 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 6 Oct 2026 13:58:02 -0400 Subject: [PATCH 11/15] fix(voice): reconcile current upstream dependency lockfile --- pnpm-lock.yaml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04ff1e39e632..5b82a5db8328 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2832,21 +2832,14 @@ packages: '@cloudflare/workers-types@5.20261004.1': resolution: {integrity: sha512-K9D4/Y6bSIDNYFzjqzFMNoWK99pLgfgFxqSa76kfW++3VCkTSzJbzsQRffir366G3kmNyjoigC6ekp7DxIkxkw==} + '@coderabbitai/config@1.1.0': + resolution: {integrity: sha512-NvQKaoOD7fWb7lMKhid57ABVlIBSczw/S+ePk5YvoPuWdrDkyY+PWdqVtq3TRq7VY8f3SNRlMFjmAtXVab66pw==} + '@config-plugins/react-native-webrtc@15.0.2': resolution: {integrity: sha512-sH4T7Z4P2RowV91k9CEwEr0unn+396NBexxCNZwRXuJXwguDU1qZWpE6fcwN0730B8uiS83F7+CLTyhpC7qRHQ==} peerDependencies: expo: '>=56' - '@connectrpc/connect-node@1.7.0': - resolution: {integrity: sha512-6vaPIkG/NyhxlYgytLoR9KYbPhczEboFB2OYWkA9qvUz1K7efXfeGrlRxoLtpa+r8VxyIOw73w5ktNe743nD+A==} - engines: {node: '>=16.0.0'} - peerDependencies: - '@bufbuild/protobuf': ^1.10.0 - '@connectrpc/connect': 1.7.0 - - '@coderabbitai/config@1.1.0': - resolution: {integrity: sha512-NvQKaoOD7fWb7lMKhid57ABVlIBSczw/S+ePk5YvoPuWdrDkyY+PWdqVtq3TRq7VY8f3SNRlMFjmAtXVab66pw==} - '@connectrpc/connect-web@1.7.0': resolution: {integrity: sha512-qyP0YOnUPRWwCc/VfsoydMJvkb7EyUPr2q9sHgBuJzbADjiqck1gKH5V5ZPzPhTLBvmz5UvG+wiZ5sMRQHU1MQ==} peerDependencies: @@ -13672,20 +13665,14 @@ snapshots: '@cloudflare/workers-types@5.20261004.1': {} - '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.2)': - dependencies: - expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) - - '@connectrpc/connect-node@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': - dependencies: - '@bufbuild/protobuf': 1.10.0 - '@connectrpc/connect': 1.7.0(@bufbuild/protobuf@1.10.0) - undici: 5.29.0 - '@coderabbitai/config@1.1.0': dependencies: zod: 4.5.4 + '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.2)': + dependencies: + expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) + '@connectrpc/connect-web@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': dependencies: '@bufbuild/protobuf': 1.10.0 From d3b652449777c73e67f48bffb87e2a42c7cd09ff Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 6 Oct 2026 13:59:13 -0400 Subject: [PATCH 12/15] fix(voice): use renamed upstream test layers --- apps/server/src/keybindings.test.ts | 2 +- .../src/orchestration-v2/ProviderContinuationService.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/keybindings.test.ts b/apps/server/src/keybindings.test.ts index 82b79f42b887..4d071b482dea 100644 --- a/apps/server/src/keybindings.test.ts +++ b/apps/server/src/keybindings.test.ts @@ -412,7 +412,7 @@ it.layer(NodeServices.layer)("keybindings", (it) => { const persisted = yield* readKeybindingsConfig(keybindingsConfigPath); assert.isFalse(persisted.some((entry) => entry.command === "voice.toggle")); assert.isTrue(persisted.some((entry) => entry.command === "navigation.back")); - }).pipe(Effect.provide(makeKeybindingsLayer())), + }).pipe(Effect.provide(layerKeybindings())), ); it.effect("upserts custom keybindings to configured path", () => diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 671733615d03..09283c077f05 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -269,7 +269,7 @@ describe("ProviderContinuationService", () => { assert.isUndefined(command.notification); }).pipe( Effect.provide( - testLayer({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), + layerTest({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), ), Effect.scoped, ); From bcfaa70c9148dbd5f6a09aa7440debd8642ab645 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Thu, 8 Oct 2026 16:08:28 -0400 Subject: [PATCH 13/15] fix(server): use shared RPC instrumentation for voice streams --- .../src/observability/RpcInstrumentation.ts | 1 + apps/server/src/ws.ts | 84 +++++++++---------- 2 files changed, 41 insertions(+), 44 deletions(-) diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 167ce8d11fff..1d95505478b9 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -15,6 +15,7 @@ type WsRpcMethod = RpcGroup.Rpcs["_tag"]; * prefix. Adding an RPC to `WsRpcGroup` without a label is a type error. */ const RPC_AGGREGATES = { + [WS_METHODS.providerVoiceSession]: "provider", [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 0093f16f0b77..cc4e453acba0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2309,51 +2309,47 @@ const layerWsRpc = ( ), ), [WS_METHODS.providerVoiceSession]: (input) => - observeRpcStream( - WS_METHODS.providerVoiceSession, - Stream.unwrap( - Effect.gen(function* () { - const voiceError = (detail: string) => - new ProviderVoiceSessionError({ threadId: input.threadId, detail }); - const projection = yield* threadManagement.getThreadRecords(input.threadId, [ - "providerThreads", - ]); - const providerThread = - projection.providerThreads.find( - (candidate) => candidate.id === projection.thread.activeProviderThreadId, - ) ?? projection.providerThreads.at(-1); - const providerSessionId = providerThread?.providerSessionId ?? null; - if (providerThread === undefined || providerSessionId === null) { - return yield* voiceError("Send a message before starting voice."); - } - const runtime = Option.getOrNull(yield* providerSessionsV2.get(providerSessionId)); - if (runtime === null) { - return yield* voiceError( - "The provider session is no longer running. Send a message first.", - ); - } - if (runtime.startVoiceSession === undefined) { - return yield* voiceError( - `Provider '${runtime.driver}' does not support voice conversations.`, - ); - } - return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp }); - }), - ).pipe( - Stream.mapError((cause) => - cause._tag === "ProviderVoiceSessionError" - ? cause - : new ProviderVoiceSessionError({ - threadId: input.threadId, - detail: - "detail" in cause && typeof cause.detail === "string" - ? cause.detail - : cause.message, - cause, - }), - ), + Stream.unwrap( + Effect.gen(function* () { + const voiceError = (detail: string) => + new ProviderVoiceSessionError({ threadId: input.threadId, detail }); + const projection = yield* threadManagement.getThreadRecords(input.threadId, [ + "providerThreads", + ]); + const providerThread = + projection.providerThreads.find( + (candidate) => candidate.id === projection.thread.activeProviderThreadId, + ) ?? projection.providerThreads.at(-1); + const providerSessionId = providerThread?.providerSessionId ?? null; + if (providerThread === undefined || providerSessionId === null) { + return yield* voiceError("Send a message before starting voice."); + } + const runtime = Option.getOrNull(yield* providerSessionsV2.get(providerSessionId)); + if (runtime === null) { + return yield* voiceError( + "The provider session is no longer running. Send a message first.", + ); + } + if (runtime.startVoiceSession === undefined) { + return yield* voiceError( + `Provider '${runtime.driver}' does not support voice conversations.`, + ); + } + return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp }); + }), + ).pipe( + Stream.mapError((cause) => + cause._tag === "ProviderVoiceSessionError" + ? cause + : new ProviderVoiceSessionError({ + threadId: input.threadId, + detail: + "detail" in cause && typeof cause.detail === "string" + ? cause.detail + : cause.message, + cause, + }), ), - { "rpc.aggregate": "provider" }, ), [WS_METHODS.serverUpdateProvider]: (input) => providerMaintenanceRunner.updateProvider(input), From 9675bc9d3aa2990e16de3b175787d8efd5a1faf5 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Fri, 9 Oct 2026 17:22:50 -0400 Subject: [PATCH 14/15] test(voice): preserve custom shortcuts and provide MCP session service --- apps/server/src/keybindings.test.ts | 2 ++ .../server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/keybindings.test.ts b/apps/server/src/keybindings.test.ts index 4d071b482dea..ce0358032766 100644 --- a/apps/server/src/keybindings.test.ts +++ b/apps/server/src/keybindings.test.ts @@ -280,8 +280,10 @@ it.layer(NodeServices.layer)("keybindings", (it) => { ); for (const defaultRule of Keybindings.DEFAULT_KEYBINDINGS) { + if (defaultRule.command === "threadPanel.toggle") continue; assert.isTrue(byCommand.has(defaultRule.command), `expected ${defaultRule.command}`); } + assert.isFalse(byCommand.has("threadPanel.toggle")); assert.isTrue(byCommand.has("script.run-tests.run")); const configState = yield* keybindings.loadConfigState; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index f010b1c187c6..681323619e4b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2150,7 +2150,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(request?.delivery, "message_text"); assert.equal(request?.detail, "run the tests"); assert.equal(request?.providerThreadId, harness.providerThread.id); - }).pipe(Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))), + }).pipe(Effect.provide(Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer))), ), ); From 9ff30232d999996b6070414cb01cf72c27b0b796 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Fri, 9 Oct 2026 17:27:00 -0400 Subject: [PATCH 15/15] fix(voice): use the provider protocol error from provider-core --- apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 5ecb468b942a..13fd2b5ddb7c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -7225,7 +7225,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Stream.mapError((cause) => cause._tag === "ProviderAdapterProtocolError" ? cause - : new ProviderAdapterProtocolError({ + : new ProviderAdapter.ProviderAdapterProtocolError({ driver: CODEX_PROVIDER, detail: cause._tag === "CodexVoiceSessionError" ? cause.detail : cause.message,