diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index c9a49b1d735b..c45f0fd1d818 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -94,6 +94,10 @@ function makeFakeBrowserWindow() { reloadIgnoringCache: vi.fn(), replaceMisspelling: vi.fn(), send: vi.fn(), + session: { + setPermissionCheckHandler: vi.fn(), + setPermissionRequestHandler: vi.fn(), + }, setBackgroundThrottling: vi.fn(), setWindowOpenHandler: vi.fn(), toggleDevTools: vi.fn(), diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 75393d54a1b2..c6ef8a074db0 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -32,6 +32,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts"; +import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts"; import { makeQuitShortcutHandler } from "./QuitHold.ts"; import { copyContextMenuImage } from "./ContextMenuImage.ts"; @@ -440,6 +441,10 @@ export const make = Effect.gen(function* () { if (environment.platform === "darwin") { window.setAutoHideCursor(false); } + installMainWindowPermissionHandlers(window.webContents.session, { + appUrl: applicationUrl, + platform: environment.platform, + }); let boundsPersistFiber: Fiber.Fiber | undefined; let pendingBoundsPersistFiber: Fiber.Fiber | undefined; let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds; diff --git a/apps/desktop/src/window/MainWindowPermissions.test.ts b/apps/desktop/src/window/MainWindowPermissions.test.ts new file mode 100644 index 000000000000..971bcb7a3320 --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { decideMainWindowMediaRequest, isAppUrl } from "./MainWindowPermissions.ts"; + +const APP_URL = "t3code://app/"; + +function decide(overrides: Partial[0]> = {}) { + return decideMainWindowMediaRequest({ + requestingUrl: "t3code://app/threads/abc", + appUrl: APP_URL, + mediaTypes: ["audio"], + platform: "darwin", + microphoneStatus: () => "granted", + ...overrides, + }); +} + +describe("isAppUrl", () => { + it("matches the app shell and nothing that merely shares its prefix", () => { + expect(isAppUrl("t3code://app", APP_URL)).toBe(true); + expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true); + expect(isAppUrl("t3code://application/", APP_URL)).toBe(false); + expect(isAppUrl("https://example.com/", APP_URL)).toBe(false); + expect(isAppUrl(undefined, APP_URL)).toBe(false); + }); +}); + +describe("decideMainWindowMediaRequest", () => { + it("denies media to frames outside the app shell", () => { + expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny"); + }); + + it("asks macOS for the microphone until the user has granted it", () => { + expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone"); + expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone"); + expect(decide()).toBe("grant"); + }); + + it("grants the microphone directly off macOS", () => { + expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant"); + }); + + it("keeps granting video-only capture such as preview tab recording", () => { + expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant"); + }); +}); diff --git a/apps/desktop/src/window/MainWindowPermissions.ts b/apps/desktop/src/window/MainWindowPermissions.ts new file mode 100644 index 000000000000..5b9ff65e1f65 --- /dev/null +++ b/apps/desktop/src/window/MainWindowPermissions.ts @@ -0,0 +1,63 @@ +import * as Electron from "electron"; + +export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone"; + +/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */ +export function isAppUrl(url: string | undefined, appUrl: string): boolean { + if (!url) return false; + // Custom schemes have an opaque WHATWG origin, so compare the serialized prefix. + const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl; + return url === appOrigin || url.startsWith(`${appOrigin}/`); +} + +/** + * Media requests come from the app shell (voice conversations use the + * microphone; preview tab recording routes `getDisplayMedia` through the same + * permission). Frames from any other origin never get media. On macOS the + * microphone also needs the system's consent, asked for on first use. + */ +export function decideMainWindowMediaRequest(input: { + readonly requestingUrl: string; + readonly appUrl: string; + readonly mediaTypes: ReadonlyArray<"audio" | "video">; + readonly platform: NodeJS.Platform; + readonly microphoneStatus: () => string; +}): MediaPermissionDecision { + if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny"; + if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant"; + return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone"; +} + +/** + * Installs the main window session's permission handlers. Only `media` is + * decided here; every other permission keeps Electron's default of allowing + * it, which is what this session did before it had handlers. + */ +export function installMainWindowPermissionHandlers( + session: Electron.Session, + options: { readonly appUrl: string; readonly platform: NodeJS.Platform }, +): void { + session.setPermissionRequestHandler((_webContents, permission, callback, details) => { + if (permission !== "media") { + callback(true); + return; + } + const decision = decideMainWindowMediaRequest({ + requestingUrl: details.requestingUrl, + appUrl: options.appUrl, + mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [], + platform: options.platform, + microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"), + }); + if (decision !== "ask-microphone") { + callback(decision === "grant"); + return; + } + Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => { + callback(false); + }); + }); + session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) => + permission === "media" ? isAppUrl(requestingOrigin, options.appUrl) : true, + ); +} diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 57717d2d8458..082938cc5566 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -220,6 +220,10 @@ const sharingPlugin: NonNullable[number] = [ }, ]; +const MICROPHONE_PERMISSION = + "Allow T3 Code to use your microphone for voice input and voice conversations."; +const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes."; + // These aliases match the fonts' PostScript names on iOS. Register the same // names on Android so React Native and the native composer use one set of // family names without waiting for runtime font loading. @@ -314,8 +318,11 @@ const config: ExpoConfig = { // JS back handling survives it via react-native's Android 16 shim plus // withAndroidPredictiveBackCompat on Android 13-15. predictiveBackGestureEnabled: true, - // expo-sensors declares this for its pedometer, which the app does not use. - blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"], + // The app uses neither expo-sensors' pedometer nor WebRTC screen sharing. + blockedPermissions: [ + "android.permission.ACTIVITY_RECOGNITION", + "android.permission.SYSTEM_ALERT_WINDOW", + ], }, web: { favicon: variant.assets.appIcon, @@ -382,22 +389,30 @@ const config: ExpoConfig = { [ "expo-audio", { - microphonePermission: "Allow T3 Code to use your microphone for voice input.", - recordAudioAndroid: false, + microphonePermission: MICROPHONE_PERMISSION, + recordAudioAndroid: true, enableBackgroundPlayback: false, enableBackgroundRecording: false, }, ], + // Voice conversations run over WebRTC. Pass both usage strings so its + // plugin cannot fall back to generic ones. + [ + "@config-plugins/react-native-webrtc", + { cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION }, + ], [ "expo-camera", { - cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.", + cameraPermission: CAMERA_PERMISSION, microphonePermission: false, barcodeScannerEnabled: true, recordAudioAndroid: false, }, ], - ["expo-image-picker", { photosPermission: false, microphonePermission: false }], + // `microphonePermission: false` would block RECORD_AUDIO on Android for every + // package, including voice conversations, so share the app's usage string. + ["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }], [ "expo-splash-screen", { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 16eccc711442..b23659ba52e9 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -129,6 +129,7 @@ "react-native-screens": "file:deps/react-native-screens-5.0.0-t3.27.tgz", "react-native-shiki-engine": "^0.3.12", "react-native-svg": "15.15.5", + "react-native-webrtc": "124.0.8", "react-native-webview": "14.0.1", "react-native-worklets": "0.13.0", "remark-gfm": "catalog:", @@ -139,6 +140,7 @@ "uniwind": "1.12.1" }, "devDependencies": { + "@config-plugins/react-native-webrtc": "15.0.2", "@effect/vitest": "catalog:", "@pierre/trees": "1.0.0-beta.6", "@types/mdast": "^4.0.4", diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 64b74b0e9fd5..9fb4e3c37d1a 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -82,6 +82,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar"; import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight"; import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing"; import IconMicrophone from "@tabler/icons-react-native/IconMicrophone"; +import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff"; import IconLink from "@tabler/icons-react-native/IconLink"; import IconListNumbers from "@tabler/icons-react-native/IconListNumbers"; import IconMenu2 from "@tabler/icons-react-native/IconMenu2"; @@ -127,6 +128,7 @@ import IconUserQuestion from "@tabler/icons-react-native/IconUserQuestion"; import IconUserPlus from "@tabler/icons-react-native/IconUserPlus"; import IconUsers from "@tabler/icons-react-native/IconUsers"; import IconWifi from "@tabler/icons-react-native/IconWifi"; +import IconWaveSine from "@tabler/icons-react-native/IconWaveSine"; import IconWifiOff from "@tabler/icons-react-native/IconWifiOff"; import IconWorld from "@tabler/icons-react-native/IconWorld"; import IconX from "@tabler/icons-react-native/IconX"; @@ -221,6 +223,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { macstudio: IconDeviceDesktop, magnifyingglass: IconSearch, mic: IconMicrophone, + "mic.slash": IconMicrophoneOff, paintbrush: IconPalette, pencil: IconPencil, "person.crop.circle": IconUserCircle, @@ -269,6 +272,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = { "tray.and.arrow.up": IconUpload, trash: IconTrash, wifi: IconWifi, + waveform: IconWaveSine, "wifi.slash": IconWifiOff, xmark: IconX, "xmark.circle.fill": IconCircleXFilled, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index f84ad7cbce01..e101fccc0aac 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -117,6 +117,9 @@ import { } from "../voice-input/ComposerDictationControl"; import { useVoiceInputController } from "../voice-input/useVoiceInputController"; import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; +import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode"; +import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode"; +import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode"; import { rememberModelOptions, withRememberedModelOptions, @@ -515,15 +518,41 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer onUsageLimits: usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined, }); + const voiceTarget = useMemo( + () => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }), + [props.environmentId, props.selectedThread.id], + ); + const voicePhase = useVoiceModePhase(voiceTarget); + useVoiceModeThreadLifecycle(voiceTarget); + // Dictation and a voice conversation both need the microphone. + const voiceConversationLive = voicePhase !== "idle"; const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, + disabled: voiceConversationLive, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); + const offersVoiceConversation = + voiceModeSupported && selectedProviderStatus?.supportsVoice === true; + // The server attaches voice to the thread's provider thread, which the first + // run creates. An optimistic pending thread already has a message timestamp. + const canToggleVoiceConversation = + voiceConversationLive || + (props.connectionState === "connected" && + props.selectedThread.activeProviderThreadId !== null && + !voiceInput.isBusy); + const toggleVoiceConversation = useCallback(() => voiceMode.toggle(voiceTarget), [voiceTarget]); + const voiceModeButton = offersVoiceConversation ? ( + + ) : null; const voicePresentation = resolveVoiceComposerPresentation( voiceInput.state, voiceInput.elapsedSeconds, @@ -833,6 +862,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer {selectedProviderStatus.compatibilityAdvisory.message} ) : null} + + {voiceConversationLive ? : null} + {modelUnavailable ? ( Model unavailable. Open model settings. @@ -1075,9 +1107,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer ) : null} {!isExpanded ? ( + {voiceModeButton} @@ -1168,10 +1202,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer )} + {isVoiceInputPresented ? null : voiceModeButton} ({ environmentId: props.environmentId, threadId: props.selectedThread.id }), + [props.environmentId, props.selectedThread.id], + ); + const voiceLiveHere = useVoiceModePhase(voiceTarget) !== "idle"; // The open /usage-limits panel for this thread, model and turn. Only the open // moment is stored: the rows read live provider data, so a redeemed reset // credit or refreshed probe shows through. Anything that spends quota closes @@ -1386,7 +1393,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread ) : null} - {/* Hidden (not unmounted) while a user-input request owns the + {/* The composer below hides for a question card; keep a live + voice conversation's mute and stop controls reachable. */} + {activeUserInputRequestId !== null && voiceLiveHere ? : null} + {/* Hidden (not unmounted) while a user-input request owns the composer slot, so composer drafts and editor state survive. A rejected creation has no thread to send to; the failure card owns the slot instead. */} diff --git a/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx new file mode 100644 index 000000000000..de8512926413 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/ComposerVoiceMode.tsx @@ -0,0 +1,170 @@ +import type { VoiceModePhase } from "@t3tools/client-runtime/voice-mode"; +import { memo, useEffect, useState } from "react"; +import { Pressable, View } from "react-native"; +import Animated, { + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, + type SharedValue, +} from "react-native-reanimated"; + +import { AppText as Text } from "../../components/AppText"; +import { SymbolView, type AppSymbolName } from "../../components/AppSymbol"; +import { cn } from "../../lib/cn"; +import { useVoiceModeState } from "./useVoiceMode"; +import { voiceMode } from "./voiceMode"; +import { + createSpeakingDetector, + resolveVoiceStripStatus, + VOICE_STRIP_STATUS_LABEL, + voiceCaptionTail, + voiceMeterLevel, +} from "./voiceModePresentation"; + +const LEVEL_TIMING = { duration: 100, reduceMotion: ReduceMotion.System } as const; +const METER_HEIGHT = 16; +const VISIBLE_CAPTIONS = 2; + +/** Starts or ends a voice conversation on the thread. Lit while one is live there. */ +export function ComposerVoiceModeButton(props: { + readonly phase: VoiceModePhase; + readonly disabled: boolean; + readonly onPress: () => void; +}) { + const live = props.phase !== "idle"; + return ( + + + + + + ); +} + +const LevelBar = memo(function LevelBar(props: { + readonly level: SharedValue; + readonly className: string; +}) { + const { level } = props; + const animatedStyle = useAnimatedStyle(() => ({ + transform: [{ scaleY: 0.15 + level.value * 0.85 }], + })); + return ( + + ); +}); + +function StripButton(props: { + readonly accessibilityLabel: string; + readonly icon: AppSymbolName; + readonly onPress: () => void; +}) { + return ( + + + + ); +} + +/** + * The live conversation's status, levels, controls, and newest captions. + * Render only while this thread's conversation is connecting or active. + */ +export function ComposerVoiceStrip() { + const state = useVoiceModeState(); + const microphoneLevel = useSharedValue(0); + const speakerLevel = useSharedValue(0); + const [speaking, setSpeaking] = useState(false); + + // Levels arrive at 10Hz; they drive shared values directly, and the + // speaking flag only re-renders when it flips. + useEffect(() => { + const isSpeaking = createSpeakingDetector(); + const unsubscribe = voiceMode.subscribeLevels((levels) => { + microphoneLevel.value = withTiming(voiceMeterLevel(levels.microphone), LEVEL_TIMING); + speakerLevel.value = withTiming(voiceMeterLevel(levels.speaker), LEVEL_TIMING); + setSpeaking(isSpeaking(levels.speaker, Date.now())); + }); + return () => { + unsubscribe(); + }; + }, [microphoneLevel, speakerLevel]); + + const status = resolveVoiceStripStatus({ phase: state.phase, muted: state.muted, speaking }); + const captions = state.captions.slice(-VISIBLE_CAPTIONS); + + return ( + + + + + + + + {VOICE_STRIP_STATUS_LABEL[status]} + + voiceMode.toggleMuted()} + /> + voiceMode.stop()} + /> + + {captions.map((caption, index) => ( + + {voiceCaptionTail(caption.text)} + + ))} + + ); +} diff --git a/apps/mobile/src/features/voice-mode/useVoiceMode.ts b/apps/mobile/src/features/voice-mode/useVoiceMode.ts new file mode 100644 index 000000000000..ee6507cda85b --- /dev/null +++ b/apps/mobile/src/features/voice-mode/useVoiceMode.ts @@ -0,0 +1,59 @@ +import type { + VoiceModePhase, + VoiceModeState, + VoiceSessionTarget, +} from "@t3tools/client-runtime/voice-mode"; +import { useCallback, useEffect, useSyncExternalStore } from "react"; +import { Alert } from "react-native"; + +import { voiceMode } from "./voiceMode"; +import { resolveVoiceModeNotice } from "./voiceModePresentation"; + +export function useVoiceModeState(): VoiceModeState { + return useSyncExternalStore(voiceMode.subscribe, voiceMode.getState); +} + +/** This thread's conversation phase. A primitive, so captions do not re-render the caller. */ +export function useVoiceModePhase(target: VoiceSessionTarget): VoiceModePhase { + const { environmentId, threadId } = target; + const getSnapshot = useCallback( + () => + voiceMode.isActiveFor({ environmentId, threadId }) ? voiceMode.getState().phase : "idle", + [environmentId, threadId], + ); + return useSyncExternalStore(voiceMode.subscribe, getSnapshot); +} + +/** + * Ties a conversation to the screen showing its thread: leaving the thread + * ends it. Screens pushed over the thread (files, review, settings) keep it + * running, like a call; its controls return with the thread. + * Also reports failures and remote endings for this thread, once. + */ +export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void { + const { environmentId, threadId } = target; + + useEffect( + () => () => { + if (voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop(); + }, + [environmentId, threadId], + ); + + useEffect(() => { + // Reads the live state: dismissing notifies listeners re-entrantly, so a + // listener's argument can already be stale. + const report = () => { + const state = voiceMode.getState(); + if (state.target?.environmentId !== environmentId || state.target.threadId !== threadId) { + return; + } + const notice = resolveVoiceModeNotice(state); + if (!notice) return; + voiceMode.dismiss(); + Alert.alert(notice.title, notice.message); + }; + report(); + return voiceMode.subscribe(report); + }, [environmentId, threadId]); +} diff --git a/apps/mobile/src/features/voice-mode/voiceMode.ts b/apps/mobile/src/features/voice-mode/voiceMode.ts new file mode 100644 index 000000000000..72c2434d6c11 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceMode.ts @@ -0,0 +1,122 @@ +import type { EnvironmentRegistry } from "@t3tools/client-runtime/connection"; +import { + VoiceModeController, + makeVoiceSessionOpener, + type VoiceModePhase, + type VoicePeerConnection, +} from "@t3tools/client-runtime/voice-mode"; +import * as Effect from "effect/Effect"; +import { AtomRegistry } from "effect/reactivity"; +import { setAudioModeAsync, setIsAudioActiveAsync } from "expo-audio"; +import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { AppState, NativeModules } from "react-native"; +import type * as WebRTC from "react-native-webrtc"; + +import { connectionAtomRuntime } from "../../connection/runtime"; +import { appAtomRegistry } from "../../state/atom-registry"; + +/** + * False on binaries built before react-native-webrtc was added. Importing the + * package there throws, so it only loads once a conversation starts. + */ +export const voiceModeSupported: boolean = NativeModules.WebRTCModule != null; + +let webrtc: typeof WebRTC | null = null; +function loadWebRTC(): typeof WebRTC { + webrtc ??= require("react-native-webrtc") as typeof WebRTC; + return webrtc; +} + +/** Runs the voice stream on the connection runtime, holding it for the conversation's lifetime. */ +function runOnConnectionRuntime( + effect: Effect.Effect, +) { + return Effect.runFork( + Effect.gen(function* () { + yield* AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime); + const context = yield* AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime); + yield* Effect.provideContext(effect, context); + }).pipe(Effect.scoped, Effect.orDie), + ); +} + +/** + * react-native-webrtc's published declarations drop its EventTarget base + * (`lib/typescript` omits the vendored event-target-shim types), so the class + * types without `addEventListener`. The runtime class extends the shim and + * supports it, which is all the controller uses. + */ +function createPeerConnection(): VoicePeerConnection { + return new (loadWebRTC().RTCPeerConnection)() as unknown as VoicePeerConnection; +} + +export const voiceMode = new VoiceModeController({ + createPeerConnection, + getMicrophone: () => loadWebRTC().mediaDevices.getUserMedia({ audio: true }), + // Native WebRTC plays remote audio tracks through its own audio device; there is nothing to attach. + playRemoteAudio: () => () => undefined, + openSession: makeVoiceSessionOpener(runOnConnectionRuntime), +}); + +/* + * WebRTC's iOS audio session uses voice-chat mode, which plays through the + * earpiece. Once the call is live, re-apply play-and-record with the speaker + * as the default route (headphones and Bluetooth still win). WebRTC only sets + * its category when its audio unit starts, which has happened by then. On + * Android the same call turns the speakerphone on. + */ +async function routeVoiceToSpeaker(): Promise { + await setAudioModeAsync({ + allowsRecording: true, + playsInSilentMode: true, + interruptionMode: "doNotMix", + shouldPlayInBackground: false, + shouldRouteThroughEarpiece: false, + }); +} + +/** Mirrors dictation's release so interrupted audio in other apps resumes. */ +async function releaseVoiceAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + await setIsAudioActiveAsync(false); + } +} + +const KEEP_AWAKE_TAG = "voice-mode"; +let previousPhase: VoiceModePhase = "idle"; +let routedAudio = false; +let keepAwake: Promise | null = null; + +voiceMode.subscribe(({ phase }) => { + if (phase === previousPhase) return; + previousPhase = phase; + if (phase === "active" && !routedAudio) { + routedAudio = true; + routeVoiceToSpeaker().catch(() => undefined); + } + if (phase !== "idle" && !keepAwake) { + // A hands-free conversation must not auto-lock: backgrounding ends it. + keepAwake = activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch(() => undefined); + } + if (phase === "idle") { + if (routedAudio) { + routedAudio = false; + releaseVoiceAudio().catch(() => undefined); + } + const activation = keepAwake; + keepAwake = null; + // Every conversation shares the tag, so a newer one must keep its lock. + activation + ?.then(() => (keepAwake === null ? deactivateKeepAwake(KEEP_AWAKE_TAG) : undefined)) + .catch(() => undefined); + } +}); + +// iOS cuts the microphone in the background (the app has no background audio +// mode), and Android restricts background capture without a foreground +// service. End the conversation rather than leave it silently deaf. +AppState.addEventListener("change", (state) => { + if (state === "background") voiceMode.stop(); +}); diff --git a/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts b/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts new file mode 100644 index 000000000000..de2db045ec22 --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceModePresentation.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vite-plus/test"; +import { INITIAL_VOICE_MODE_STATE } from "@t3tools/client-runtime/voice-mode"; + +import { + createSpeakingDetector, + resolveVoiceModeNotice, + resolveVoiceStripStatus, + voiceCaptionTail, +} from "./voiceModePresentation"; + +describe("resolveVoiceStripStatus", () => { + it("shows connecting until the conversation is live, then prefers the assistant's voice", () => { + expect(resolveVoiceStripStatus({ phase: "connecting", muted: true, speaking: true })).toBe( + "connecting", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: true, speaking: true })).toBe( + "speaking", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: true, speaking: false })).toBe( + "muted", + ); + expect(resolveVoiceStripStatus({ phase: "active", muted: false, speaking: false })).toBe( + "listening", + ); + }); +}); + +describe("createSpeakingDetector", () => { + it("holds through short pauses and releases after sustained silence", () => { + const isSpeaking = createSpeakingDetector(); + expect(isSpeaking(0, 0)).toBe(false); + expect(isSpeaking(0.3, 100)).toBe(true); + expect(isSpeaking(0, 300)).toBe(true); + expect(isSpeaking(0.001, 600)).toBe(true); + expect(isSpeaking(0, 700)).toBe(false); + }); +}); + +describe("voiceCaptionTail", () => { + it("keeps the newest words of long captions", () => { + expect(voiceCaptionTail("short")).toBe("short"); + expect(voiceCaptionTail("one two three four", 9)).toBe("…hree four"); + }); +}); + +describe("resolveVoiceModeNotice", () => { + const target = { environmentId: "env", threadId: "thread" }; + + it("stays quiet while live and after the user stops", () => { + expect(resolveVoiceModeNotice(INITIAL_VOICE_MODE_STATE)).toBeNull(); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, phase: "active", target }), + ).toBeNull(); + }); + + it("reports failures and endings the user did not choose", () => { + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, error: "Mic denied" }), + ).toEqual({ title: "Voice conversation failed", message: "Mic denied" }); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, endedReason: "ended" }), + ).toEqual({ title: "Voice conversation ended" }); + expect( + resolveVoiceModeNotice({ ...INITIAL_VOICE_MODE_STATE, target, endedReason: "replaced" }), + ).toEqual({ + title: "Voice conversation ended", + message: "Voice conversation moved to another window or device.", + }); + }); +}); diff --git a/apps/mobile/src/features/voice-mode/voiceModePresentation.ts b/apps/mobile/src/features/voice-mode/voiceModePresentation.ts new file mode 100644 index 000000000000..118d9257b1ce --- /dev/null +++ b/apps/mobile/src/features/voice-mode/voiceModePresentation.ts @@ -0,0 +1,68 @@ +import { + describeVoiceEnd, + type VoiceModePhase, + type VoiceModeState, +} from "@t3tools/client-runtime/voice-mode"; + +export type VoiceStripStatus = "connecting" | "listening" | "speaking" | "muted"; + +export const VOICE_STRIP_STATUS_LABEL: Record = { + connecting: "Connecting…", + listening: "Listening", + speaking: "Speaking", + muted: "Muted", +}; + +/** The assistant's voice outranks mute: while it talks, that is what the user hears. */ +export function resolveVoiceStripStatus(input: { + readonly phase: VoiceModePhase; + readonly muted: boolean; + readonly speaking: boolean; +}): VoiceStripStatus { + if (input.phase !== "active") return "connecting"; + if (input.speaking) return "speaking"; + return input.muted ? "muted" : "listening"; +} + +const SPEAKING_LEVEL = 0.02; +const SPEAKING_HOLD_MS = 600; + +/** + * Turns sampled speaker levels into a steady "speaking" flag. The flag holds + * through the short gaps between words so the status label does not flicker. + */ +export function createSpeakingDetector(): (level: number, nowMs: number) => boolean { + let lastLoudAtMs: number | null = null; + return (level, nowMs) => { + if (level >= SPEAKING_LEVEL) lastLoudAtMs = nowMs; + return lastLoudAtMs !== null && nowMs - lastLoudAtMs < SPEAKING_HOLD_MS; + }; +} + +/** Maps a linear 0..1 audio level to meter height; speech sits low on the linear scale. */ +export function voiceMeterLevel(level: number): number { + return Math.min(1, Math.sqrt(Math.max(0, level)) * 1.6); +} + +/** Keeps the newest words of a live caption; native text only truncates multi-line text at the end. */ +export function voiceCaptionTail(text: string, maxChars = 160): string { + if (text.length <= maxChars) return text; + return `…${text.slice(-maxChars).trimStart()}`; +} + +export interface VoiceModeNotice { + readonly title: string; + readonly message?: string; +} + +/** What to tell the user about a finished conversation. Null when they ended it themselves. */ +export function resolveVoiceModeNotice(state: VoiceModeState): VoiceModeNotice | null { + if (state.phase !== "idle") return null; + if (state.error) return { title: "Voice conversation failed", message: state.error }; + if (state.endedReason) { + return state.endedReason === "ended" + ? { title: "Voice conversation ended" } + : { title: "Voice conversation ended", message: describeVoiceEnd(state.endedReason) }; + } + return null; +} diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 5b8d483dc102..23088cc8459b 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -169,6 +169,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.mcpAppsToolInfo]: AuthOrchestrationReadScope, [WS_METHODS.mcpAppsReadResource]: AuthOrchestrationReadScope, [WS_METHODS.mcpAppsUpdateModelContext]: AuthOrchestrationOperateScope, + [WS_METHODS.providerVoiceSession]: AuthOrchestrationOperateScope, [WS_METHODS.subscribeVcsStatus]: AuthOrchestrationReadScope, [WS_METHODS.subscribeWorktreeSetup]: AuthOrchestrationReadScope, [WS_METHODS.worktreeSetupCancel]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/keybindings.test.ts b/apps/server/src/keybindings.test.ts index 0ea44fb3ca07..ce0358032766 100644 --- a/apps/server/src/keybindings.test.ts +++ b/apps/server/src/keybindings.test.ts @@ -280,8 +280,10 @@ it.layer(NodeServices.layer)("keybindings", (it) => { ); for (const defaultRule of Keybindings.DEFAULT_KEYBINDINGS) { + if (defaultRule.command === "threadPanel.toggle") continue; assert.isTrue(byCommand.has(defaultRule.command), `expected ${defaultRule.command}`); } + assert.isFalse(byCommand.has("threadPanel.toggle")); assert.isTrue(byCommand.has("script.run-tests.run")); const configState = yield* keybindings.loadConfigState; @@ -397,6 +399,24 @@ it.layer(NodeServices.layer)("keybindings", (it) => { ); }); + it.effect("skips a conditional default that would shadow an unconditional user shortcut", () => + Effect.gen(function* () { + const { keybindingsConfigPath } = yield* ServerConfig.ServerConfig; + yield* writeKeybindingsConfig(keybindingsConfigPath, [ + { key: "f8", command: "script.custom-action.run" }, + ]); + + yield* Effect.gen(function* () { + const keybindings = yield* Keybindings.Keybindings; + yield* keybindings.syncDefaultKeybindingsOnStartup; + }); + + const persisted = yield* readKeybindingsConfig(keybindingsConfigPath); + assert.isFalse(persisted.some((entry) => entry.command === "voice.toggle")); + assert.isTrue(persisted.some((entry) => entry.command === "navigation.back")); + }).pipe(Effect.provide(layerKeybindings())), + ); + it.effect("upserts custom keybindings to configured path", () => Effect.gen(function* () { const { keybindingsConfigPath } = yield* ServerConfig.ServerConfig; diff --git a/apps/server/src/keybindings.ts b/apps/server/src/keybindings.ts index 496a5852979a..8d46c958b748 100644 --- a/apps/server/src/keybindings.ts +++ b/apps/server/src/keybindings.ts @@ -129,19 +129,19 @@ const LATE_DEFAULT_KEYBINDINGS: ReadonlyArray<{ }, ]; -function keybindingShortcutContext(rule: KeybindingRule): string | null { +function encodedRuleShortcut(rule: KeybindingRule): string | null { const parsed = parseKeybindingShortcut(rule.key); - if (!parsed) return null; - const encoded = encodeShortcut(parsed); - if (!encoded) return null; - return `${encoded}\u0000${rule.when ?? ""}`; + return parsed ? encodeShortcut(parsed) : null; } -function hasSameShortcutContext(left: KeybindingRule, right: KeybindingRule): boolean { - const leftContext = keybindingShortcutContext(left); - const rightContext = keybindingShortcutContext(right); - if (!leftContext || !rightContext) return false; - return leftContext === rightContext; +/** + * Whether appending `rule` after `existing` would shadow it: same shortcut, and + * either the same `when` or one of them applies in every context. + */ +function shadowsShortcut(existing: KeybindingRule, rule: KeybindingRule): boolean { + const existingShortcut = encodedRuleShortcut(existing); + if (!existingShortcut || existingShortcut !== encodedRuleShortcut(rule)) return false; + return existing.when === undefined || rule.when === undefined || existing.when === rule.when; } function keybindingRuleFromUpsertInput(input: ServerUpsertKeybindingInput): KeybindingRule { @@ -544,9 +544,7 @@ const make = Effect.gen(function* () { if (existingCommands.has(defaultRule.command)) { continue; } - const conflictingEntry = customConfig.find((entry) => - hasSameShortcutContext(entry, defaultRule), - ); + const conflictingEntry = customConfig.find((entry) => shadowsShortcut(entry, defaultRule)); if (conflictingEntry) { shortcutConflictWarnings.push({ defaultCommand: defaultRule.command, @@ -563,7 +561,7 @@ const make = Effect.gen(function* () { for (const { alongside, rule } of pendingLateDefaults) { if ( customConfig.some((entry) => isSameKeybindingRule(entry, alongside)) && - !customConfig.some((entry) => hasSameShortcutContext(entry, rule)) + !customConfig.some((entry) => shadowsShortcut(entry, rule)) ) { missingDefaults.push(rule); } diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 167ce8d11fff..1d95505478b9 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -15,6 +15,7 @@ type WsRpcMethod = RpcGroup.Rpcs["_tag"]; * prefix. Adding an RPC to `WsRpcGroup` without a label is a type error. */ const RPC_AGGREGATES = { + [WS_METHODS.providerVoiceSession]: "provider", [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2", [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration", [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration", diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 9a1c8f6c900f..681323619e4b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2032,6 +2032,128 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }); + it.effect( + "resubmits a voice handoff Codex started only once its turn is interrupted or failed", + () => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "voice-thread"; + const delegation = (input: string) => + `\n ${input}\n`; + const voiceTurn = ( + turnId: string, + request: string, + id: number, + status: "completed" | "interrupted" | "failed", + ) => [ + { + type: "emit_inbound" as const, + label: `turn/started/${turnId}`, + frame: { + method: "turn/started", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound" as const, + label: `item/started/${turnId}`, + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId, + startedAtMs: 1782622440500, + item: { + type: "userMessage", + id: `user-${turnId}`, + content: [{ type: "text", text: delegation(request), text_elements: [] }], + }, + }, + }, + }, + { + type: "expect_outbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, method: "turn/interrupt", params: { threadId: nativeThreadId, turnId } }, + }, + { + type: "emit_inbound" as const, + label: `turn/interrupt/${turnId}`, + frame: { id, result: {} }, + }, + { + type: "emit_inbound" as const, + label: `turn/completed/${turnId}`, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: turnId, status }), + }, + }, + }, + ]; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "voice-handoff-resubmit", + entries: [ + // Initialization and thread/start only; Codex, not T3, starts these turns. + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused", + prompt: "", + }).slice(0, -3), + { + type: "expect_outbound", + label: "thread/realtime/start", + frame: { + id: 3, + method: "thread/realtime/start", + params: { + threadId: nativeThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + }, + }, + }, + { + type: "emit_inbound", + label: "thread/realtime/start", + frame: { id: 3, result: {} }, + }, + // The interrupt lost the race: Codex finished, so it must not run twice. + ...voiceTurn("voice-raced", "already done", 4, "completed"), + ...voiceTurn("voice-stopped", "run the tests", 5, "interrupted"), + // A failed turn never did the work either, so its request is kept. + ...voiceTurn("voice-failed", "open a pull request", 6, "failed"), + ], + }), + ); + yield* harness.runtime.startVoiceSession!({ + providerThread: harness.providerThread, + offerSdp: "v=offer\r\n", + }).pipe(Stream.runDrain, Effect.forkScoped); + + yield* awaitUntil(() => harness.continuationRequests.length === 2, "voice resubmits"); + assert.deepEqual( + harness.continuationRequests.map((entry) => entry.detail), + ["run the tests", "open a pull request"], + ); + const request = harness.continuationRequests[0]; + assert.equal(request?.origin, "voice"); + assert.equal(request?.delivery, "message_text"); + assert.equal(request?.detail, "run the tests"); + assert.equal(request?.providerThreadId, harness.providerThread.id); + }).pipe(Effect.provide(Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer))), + ), + ); + it.effect.each(["supported", "unsupported", "invalid"] as const)( "delivers native history with %s app-server protocol", (response) => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index d8ef1461ef73..13fd2b5ddb7c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -113,6 +113,7 @@ import { codexAppServerArgs, resolveCodexLaunchArgs } from "../../provider/codex import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; +import { makeCodexVoice, readRealtimeDelegationFromItem } from "../../provider/CodexVoice.ts"; import { MCP_APP_EXTENSION_ID, MCP_APP_MIME_TYPE, @@ -1876,6 +1877,19 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // terminal, otherwise the root terminal can strand child projections. const deferredRootTerminals = yield* Ref.make(new Map()); const offeredContinuationItemsByTurn = yield* Ref.make(new Map>()); + const sessionScope = yield* Effect.scope; + const voice = yield* makeCodexVoice(client, sessionScope); + /** The provider thread holding the voice conversation, while one runs. */ + const voiceProviderThread = yield* Ref.make( + undefined, + ); + /** Native turns Codex started for a voice handoff -> the request and its thread. */ + const voiceResubmits = yield* Ref.make( + new Map< + string, + { readonly text: string; readonly providerThread: OrchestrationV2ProviderThread } + >(), + ); const finalAnswerItemIdsByTurn = yield* Ref.make(new Map>()); const completedFinalAnswerTextsByTurn = yield* Ref.make(new Map>()); // Native completion and the interrupt timeout share one finalization @@ -4608,6 +4622,31 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("item/started", (payload) => Effect.gen(function* () { + if (payload.item.type === "userMessage") { + const handoff = readRealtimeDelegationFromItem(payload.item); + const voiceThread = yield* Ref.get(voiceProviderThread); + if ( + handoff !== undefined && + voiceThread !== undefined && + voiceThread.nativeThreadRef?.nativeId === payload.threadId && + !(yield* Ref.get(activeTurns)).has(payload.turnId) && + !(yield* Ref.get(pendingRootTurns)).has(payload.threadId) + ) { + // Codex started this turn itself for a voice handoff. V2 runs + // are orchestrator-owned, so stop it and resubmit the request + // as an ordinary run once it settles (see turn/completed). + yield* Ref.update(voiceResubmits, (current) => + new Map(current).set(payload.turnId, { + text: handoff, + providerThread: voiceThread, + }), + ); + yield* client + .request("turn/interrupt", { threadId: payload.threadId, turnId: payload.turnId }) + .pipe(Effect.ignore, Effect.forkIn(sessionScope)); + return; + } + } const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { return; @@ -5974,6 +6013,33 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { + const voiceResubmit = (yield* Ref.get(voiceResubmits)).get(payload.turn.id); + if (voiceResubmit !== undefined) { + yield* Ref.update(voiceResubmits, (current) => { + const updated = new Map(current); + updated.delete(payload.turn.id); + return updated; + }); + // Resubmit a turn Codex stopped or failed, so the request isn't + // lost. If the interrupt lost the race and the turn completed, + // Codex already did the work (and the voice speaks its answer), + // so running it again would repeat it. + if ( + (payload.turn.status === "interrupted" || payload.turn.status === "failed") && + continuationRequests !== undefined + ) { + yield* voice.expectSpokenTurn(voiceResubmit.text); + yield* continuationRequests.offer({ + threadId: voiceResubmit.providerThread.appThreadId ?? input.threadId, + providerThreadId: voiceResubmit.providerThread.id, + driver: CODEX_PROVIDER, + detail: voiceResubmit.text, + delivery: "message_text", + origin: "voice", + }); + } + return; + } const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); if (context === undefined) { return; @@ -7135,6 +7201,38 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ), ), + startVoiceSession: (voiceInput) => + Stream.unwrap( + Effect.gen(function* () { + const nativeThreadId = yield* getNativeThreadId(voiceInput.providerThread); + yield* ensureInitialized; + yield* Ref.set(voiceProviderThread, voiceInput.providerThread); + return voice.start({ + providerThreadId: nativeThreadId, + offerSdp: voiceInput.offerSdp, + ...(adapterOptions.settings.voice + ? { voice: adapterOptions.settings.voice } + : {}), + }); + }), + ).pipe( + // A replaced conversation must not clear its successor's thread. + Stream.ensuring( + Ref.update(voiceProviderThread, (current) => + current === voiceInput.providerThread ? undefined : current, + ), + ), + Stream.mapError((cause) => + cause._tag === "ProviderAdapterProtocolError" + ? cause + : new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: + cause._tag === "CodexVoiceSessionError" ? cause.detail : cause.message, + payload: cause, + }), + ), + ), uploadFeedback: (feedbackInput) => Effect.gen(function* () { const threadId = yield* getNativeThreadId(feedbackInput.providerThread); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 689c66a1d402..09283c077f05 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -244,6 +244,38 @@ describe("ProviderContinuationService", () => { }); }); + it.effect("resubmits a spoken voice request as the user's own message", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + yield* requests.offer({ + threadId, + providerThreadId, + driver, + detail: "run the tests", + delivery: "message_text", + origin: "voice", + }); + const command = (yield* Queue.take(dispatched)) as { + readonly createdBy: string; + readonly creationSource: string; + readonly text: string; + readonly notification?: OrchestrationV2Notification; + }; + assert.equal(command.createdBy, "user"); + assert.equal(command.creationSource, "server"); + assert.equal(command.text, "run the tests"); + assert.isUndefined(command.notification); + }).pipe( + Effect.provide( + layerTest({ dispatched, getThreadRecords: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + it.effect("dispatches a current delegated completion as one server-owned queued message", () => { return Effect.gen(function* () { const dispatched = yield* Queue.unbounded(); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index c0b0c55b7221..55c9036aafb2 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -209,26 +209,40 @@ export const layer = Layer.effectDiscard( ordinal: projection.messages.length + 1, })); const commandId = CommandId.make(`provider-continuation:${messageId}`); - const dispatch = threads.dispatch({ - type: "message.dispatch", - commandId, - threadId: request.threadId, - messageId, - text: request.detail ?? CONTINUATION_MESSAGE_TEXT, - notification: request.notification ?? { - source: { kind: "background_task" }, - outcome: "updated", - summary: "Background activity updated", - }, - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: "agent", - // "provider" marks an adapter-buffered wake, which ClaudeAdapterV2 - // detects to attach the buffered CLI output and drop this text. A - // message_text wake has no buffered output, so it must not carry that - // marker or the turn settles immediately having prompted nothing. - creationSource: request.delivery === "message_text" ? "server" : "provider", - }); + const dispatch = threads.dispatch( + request.origin === "voice" + ? { + type: "message.dispatch", + commandId, + threadId: request.threadId, + messageId, + text: request.detail ?? "", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "server", + } + : { + type: "message.dispatch", + commandId, + threadId: request.threadId, + messageId, + text: request.detail ?? CONTINUATION_MESSAGE_TEXT, + notification: request.notification ?? { + source: { kind: "background_task" }, + outcome: "updated", + summary: "Background activity updated", + }, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + // "provider" marks an adapter-buffered wake, which ClaudeAdapterV2 + // detects to attach the buffered CLI output and drop this text. A + // message_text wake has no buffered output, so it must not carry that + // marker or the turn settles immediately having prompted nothing. + creationSource: request.delivery === "message_text" ? "server" : "provider", + }, + ); // The adapter's buffered wake has nowhere else to go once the // instance is off: drop it rather than warn, since nothing here // retries it either way. This catch sits outside dispatchIfCurrent so diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 9ed238bf5d2c..2cfeed17fc8d 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -77,6 +77,7 @@ const CODEX_PRESENTATION = { displayName: "Codex", showInteractionModeToggle: true, reportsContextWindow: true, + supportsVoice: true, } as const; export interface CodexAppServerProviderSnapshot { diff --git a/apps/server/src/provider/CodexVoice.test.ts b/apps/server/src/provider/CodexVoice.test.ts new file mode 100644 index 000000000000..fdc1a2daa2ce --- /dev/null +++ b/apps/server/src/provider/CodexVoice.test.ts @@ -0,0 +1,274 @@ +import * as NodeAssert from "node:assert/strict"; + +import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; +import { describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import type * as CodexClient from "effect-codex-app-server/client"; + +import { + CodexVoiceSessionError, + makeCodexVoice, + readRealtimeDelegationInput, + readSpeakableAnswer, +} from "./CodexVoice.ts"; + +const PROVIDER_THREAD = "provider-thread-1"; + +function delegation(input: string) { + return `\n ${input}\n`; +} + +const makeFakeClient = Effect.gen(function* () { + const handlers = new Map Effect.Effect>>(); + const requests: Array<{ readonly method: string; readonly params: unknown }> = []; + const startRequested = yield* Deferred.make(); + const speechRequested = yield* Deferred.make(); + const stopRequested = yield* Deferred.make(); + const client = { + handleServerNotification: ( + method: string, + handler: (payload: unknown) => Effect.Effect, + ) => + Effect.sync(() => { + handlers.set(method, [...(handlers.get(method) ?? []), handler]); + }), + raw: { + request: (method: string, params: unknown) => + Effect.gen(function* () { + requests.push({ method, params }); + if (method === "thread/realtime/start") + yield* Deferred.succeed(startRequested, undefined); + if (method === "thread/realtime/appendSpeech") { + yield* Deferred.succeed(speechRequested, undefined); + } + if (method === "thread/realtime/stop") yield* Deferred.succeed(stopRequested, undefined); + return {}; + }), + }, + } as unknown as CodexClient.CodexAppServerClient["Service"]; + const emit = (method: string, payload: unknown) => + Effect.forEach(handlers.get(method) ?? [], (handler) => Effect.ignore(handler(payload)), { + discard: true, + }); + return { client, requests, emit, startRequested, speechRequested, stopRequested }; +}); + +const startVoice = Effect.gen(function* () { + const fake = yield* makeFakeClient; + const scope = yield* Scope.make(); + const voice = yield* makeCodexVoice(fake.client, scope); + const fiber = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "v=offer\r\n", voice: "cove" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + return { ...fake, fiber }; +}); + +describe("readRealtimeDelegationInput", () => { + it("unwraps and unescapes the spoken request", () => { + NodeAssert.equal( + readRealtimeDelegationInput(delegation("run the tests & fix <Foo>")), + "run the tests & fix ", + ); + }); + + it("ignores ordinary user messages", () => { + NodeAssert.equal(readRealtimeDelegationInput("please fix the build"), undefined); + }); +}); + +describe("readSpeakableAnswer", () => { + it("speaks final answers without the channel marker", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "[FINAL] All green.", phase: null }), + "All green.", + ); + }); + + it("keeps commentary private", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "Looking…", phase: "commentary" }), + undefined, + ); + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "[COMMENTARY] Looking…", phase: null }), + undefined, + ); + }); + + it("leaves oversized answers on screen only", () => { + NodeAssert.equal( + readSpeakableAnswer({ type: "agentMessage", text: "x".repeat(5000), phase: "final_answer" }), + undefined, + ); + }); +}); + +describe("makeCodexVoice", () => { + it.effect("starts a WebRTC realtime session and relays signaling until Codex closes it", () => + Effect.gen(function* () { + const voice = yield* startVoice; + NodeAssert.deepEqual(voice.requests[0], { + method: "thread/realtime/start", + params: { + threadId: PROVIDER_THREAD, + outputModality: "audio", + transport: { type: "webrtc", sdp: "v=offer\r\n" }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + voice: "cove", + }, + }); + + yield* voice.emit("thread/realtime/started", { threadId: PROVIDER_THREAD, version: "v3" }); + yield* voice.emit("thread/realtime/sdp", { threadId: PROVIDER_THREAD, sdp: "v=answer\r\n" }); + yield* voice.emit("thread/realtime/transcript/delta", { + threadId: "another-thread", + role: "user", + delta: "ignored", + }); + yield* voice.emit("thread/realtime/transcript/delta", { + threadId: PROVIDER_THREAD, + role: "user", + delta: "hello", + }); + yield* voice.emit("thread/realtime/transcript/done", { + threadId: PROVIDER_THREAD, + role: "assistant", + text: "Hi there", + }); + yield* voice.emit("thread/realtime/closed", { + threadId: PROVIDER_THREAD, + reason: "transport_closed", + }); + + const events = yield* Fiber.join(voice.fiber); + NodeAssert.deepEqual(events, [ + { type: "started" }, + { type: "answer", sdp: "v=answer\r\n" }, + { type: "transcript.delta", role: "user", delta: "hello" }, + { type: "transcript.done", role: "assistant", text: "Hi there" }, + { type: "closed", reason: "transport_closed" }, + ] satisfies ReadonlyArray); + // Codex already closed the session, so there is nothing to stop. + NodeAssert.ok(!voice.requests.some((request) => request.method === "thread/realtime/stop")); + }), + ); + + it.effect("stops the realtime session when the client unsubscribes", () => + Effect.gen(function* () { + const voice = yield* startVoice; + yield* Fiber.interrupt(voice.fiber); + yield* Deferred.await(voice.stopRequested); + NodeAssert.deepEqual(voice.requests.at(-1), { + method: "thread/realtime/stop", + params: { threadId: PROVIDER_THREAD }, + }); + }), + ); + + it.effect("fails the stream with Codex's realtime error", () => + Effect.gen(function* () { + const voice = yield* startVoice; + yield* voice.emit("thread/realtime/error", { + threadId: PROVIDER_THREAD, + message: "realtime unavailable", + }); + const exit = yield* Fiber.await(voice.fiber); + NodeAssert.ok(Exit.isFailure(exit)); + const error = exit.cause.reasons[0]; + NodeAssert.ok(error?._tag === "Fail" && Schema.is(CodexVoiceSessionError)(error.error)); + NodeAssert.equal(error.error.detail, "realtime unavailable"); + }), + ); + + it.effect("speaks the answer of a handoff the host resubmitted as its own turn", () => + Effect.gen(function* () { + const fake = yield* makeFakeClient; + const scope = yield* Scope.make(); + const voice = yield* makeCodexVoice(fake.client, scope); + const fiber = yield* voice + .start({ providerThreadId: PROVIDER_THREAD, offerSdp: "v=offer\r\n" }) + .pipe(Stream.runCollect, Effect.forkChild); + yield* Deferred.await(fake.startRequested); + + yield* voice.expectSpokenTurn("run the tests"); + yield* fake.emit("item/started", { + threadId: PROVIDER_THREAD, + turnId: "resubmitted", + item: { + type: "userMessage", + id: "user-resubmitted", + content: [{ type: "text", text: "\nrun the tests" }], + }, + }); + yield* fake.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId: "resubmitted", + item: { type: "agentMessage", id: "answer", text: "All green.", phase: "final_answer" }, + }); + yield* fake.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: "resubmitted", status: "completed" }, + }); + + yield* Deferred.await(fake.speechRequested); + NodeAssert.deepEqual(fake.requests.at(-1), { + method: "thread/realtime/appendSpeech", + params: { threadId: PROVIDER_THREAD, text: "All green." }, + }); + yield* Fiber.interrupt(fiber); + }), + ); + + it.effect("speaks the final answer of a voice-delegated turn, not typed turns", () => + Effect.gen(function* () { + const voice = yield* startVoice; + const userMessage = (turnId: string, text: string) => + voice.emit("item/started", { + threadId: PROVIDER_THREAD, + turnId, + item: { type: "userMessage", id: `user-${turnId}`, content: [{ type: "text", text }] }, + }); + const agentMessage = (turnId: string, text: string, phase: string) => + voice.emit("item/completed", { + threadId: PROVIDER_THREAD, + turnId, + item: { type: "agentMessage", id: `agent-${turnId}-${phase}`, text, phase }, + }); + const complete = (turnId: string) => + voice.emit("turn/completed", { + threadId: PROVIDER_THREAD, + turn: { id: turnId, status: "completed" }, + }); + + yield* userMessage("typed", "fix the build"); + yield* agentMessage("typed", "Fixed it.", "final_answer"); + yield* complete("typed"); + + yield* userMessage("voice", delegation("run the tests")); + yield* agentMessage("voice", "Running them now.", "commentary"); + yield* agentMessage("voice", "All 42 tests pass.", "final_answer"); + yield* complete("voice"); + + yield* Deferred.await(voice.speechRequested); + NodeAssert.deepEqual( + voice.requests.filter((request) => request.method === "thread/realtime/appendSpeech"), + [ + { + method: "thread/realtime/appendSpeech", + params: { threadId: PROVIDER_THREAD, text: "All 42 tests pass." }, + }, + ], + ); + yield* Fiber.interrupt(voice.fiber); + }), + ); +}); diff --git a/apps/server/src/provider/CodexVoice.ts b/apps/server/src/provider/CodexVoice.ts new file mode 100644 index 000000000000..31bec1152d25 --- /dev/null +++ b/apps/server/src/provider/CodexVoice.ts @@ -0,0 +1,294 @@ +/** + * Codex voice conversations: a thread-scoped realtime session whose audio runs + * over WebRTC between the client and OpenAI. The app-server only relays the + * SDP handshake, captions, and lifecycle; this module owns that signaling. + * + * Handoffs are client-managed, matching the Codex TUI: the realtime model + * delegates work by starting a Codex turn whose user message is wrapped in + * ``, and when that turn finishes we hand its final + * answer back with `thread/realtime/appendSpeech` so the voice reads it out. + */ +import type { ProviderVoiceSessionEvent } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import type * as CodexClient from "effect-codex-app-server/client"; +import type * as CodexErrors from "effect-codex-app-server/errors"; + +const DELEGATION_OPEN = ""; +const DELEGATION_CLOSE = ""; +// Codex rejects speech over 1,000 tokens and may add its own prefix. +const MAX_SPEAKABLE_CHARS = 990 * 4; + +function unescapeXmlText(text: string): string { + return text.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); +} + +/** The spoken request inside a Codex realtime delegation message, if `text` is one. */ +export function readRealtimeDelegationInput(text: string): string | undefined { + const trimmed = text.trim(); + if (!trimmed.startsWith(DELEGATION_OPEN) || !trimmed.endsWith(DELEGATION_CLOSE)) { + return undefined; + } + const match = /([\s\S]*?)<\/input>/.exec(trimmed); + const input = match?.[1] ? unescapeXmlText(match[1]).trim() : ""; + return input.length > 0 ? input : undefined; +} + +function readUserMessageText(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; content?: unknown }; + if (record.type !== "userMessage" || !Array.isArray(record.content)) return undefined; + return (record.content as ReadonlyArray<{ type?: unknown; text?: unknown }>) + .flatMap((entry) => + entry.type === "text" && typeof entry.text === "string" ? [entry.text] : [], + ) + .join("\n"); +} + +/** Reads the delegation input from a Codex `userMessage` item. */ +export function readRealtimeDelegationFromItem(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; content?: unknown }; + if (record.type !== "userMessage" || !Array.isArray(record.content)) return undefined; + const [only, ...rest] = record.content as ReadonlyArray<{ type?: unknown; text?: unknown }>; + if (rest.length > 0 || only?.type !== "text" || typeof only.text !== "string") return undefined; + return readRealtimeDelegationInput(only.text); +} + +/** + * Text worth speaking from a completed `agentMessage`. Commentary stays private + * to the voice handoff, and oversized answers stay on screen only. + */ +export function readSpeakableAnswer(item: unknown): string | undefined { + if (!item || typeof item !== "object") return undefined; + const record = item as { type?: unknown; text?: unknown; phase?: unknown; questions?: unknown }; + if (record.type !== "agentMessage" || typeof record.text !== "string") return undefined; + if (record.phase === "commentary" || (Array.isArray(record.questions) && record.questions.length)) + return undefined; + const text = record.text.trim(); + if ( + record.phase !== "final_answer" && + (text.startsWith("[ANALYSIS]") || text.startsWith("[COMMENTARY]")) + ) { + return undefined; + } + const spoken = text.replace(/^\[FINAL\]\s*/, ""); + if (spoken.length === 0 || spoken.length > MAX_SPEAKABLE_CHARS) return undefined; + return spoken; +} + +function transcriptRole(role: string): "user" | "assistant" { + return role === "user" ? "user" : "assistant"; +} + +/** Codex ended the realtime session with an error; `detail` is Codex's explanation for the user. */ +export class CodexVoiceSessionError extends Schema.TaggedError()( + "CodexVoiceSessionError", + { detail: Schema.String }, +) { + override get message(): string { + return this.detail; + } +} + +type VoiceQueue = Queue.Queue; + +interface ActiveVoice { + readonly providerThreadId: string; + readonly queue: VoiceQueue; + /** Delegated turn id -> latest speakable answer in that turn. */ + readonly delegatedTurns: Map; + /** Handoffs resubmitted as ordinary runs, awaiting their native turn. */ + readonly expectedHandoffs: Array; + closed: boolean; +} + +export interface CodexVoiceStartInput { + readonly providerThreadId: string; + readonly offerSdp: string; + readonly voice?: string | undefined; +} + +export const makeCodexVoice = Effect.fn("makeCodexVoice")(function* ( + client: CodexClient.CodexAppServerClient["Service"], + /** Outlives notification handlers, which must not block on requests. */ + runtimeScope: Scope.Scope, +) { + const activeRef = yield* Ref.make(undefined); + + const withActive = ( + threadId: string, + f: (active: ActiveVoice) => Effect.Effect, + ): Effect.Effect => + Effect.flatMap(Ref.get(activeRef), (active) => + active && !active.closed && active.providerThreadId === threadId ? f(active) : Effect.void, + ); + + const offer = (active: ActiveVoice, event: ProviderVoiceSessionEvent) => + Queue.offer(active.queue, event).pipe(Effect.asVoid); + + const finish = (active: ActiveVoice, event: ProviderVoiceSessionEvent) => + Effect.suspend(() => { + active.closed = true; + return offer(active, event).pipe(Effect.andThen(Queue.end(active.queue)), Effect.asVoid); + }); + + yield* client.handleServerNotification("thread/realtime/started", (payload) => + withActive(payload.threadId, (active) => offer(active, { type: "started" })), + ); + yield* client.handleServerNotification("thread/realtime/sdp", (payload) => + withActive(payload.threadId, (active) => offer(active, { type: "answer", sdp: payload.sdp })), + ); + yield* client.handleServerNotification("thread/realtime/transcript/delta", (payload) => + withActive(payload.threadId, (active) => + offer(active, { + type: "transcript.delta", + role: transcriptRole(payload.role), + delta: payload.delta, + }), + ), + ); + yield* client.handleServerNotification("thread/realtime/transcript/done", (payload) => + withActive(payload.threadId, (active) => + offer(active, { + type: "transcript.done", + role: transcriptRole(payload.role), + text: payload.text, + }), + ), + ); + yield* client.handleServerNotification("thread/realtime/error", (payload) => + withActive(payload.threadId, (active) => + Effect.suspend(() => { + active.closed = true; + return Queue.fail(active.queue, new CodexVoiceSessionError({ detail: payload.message })); + }).pipe(Effect.asVoid), + ), + ); + yield* client.handleServerNotification("thread/realtime/closed", (payload) => + withActive(payload.threadId, (active) => + finish(active, { type: "closed", ...(payload.reason ? { reason: payload.reason } : {}) }), + ), + ); + + yield* client.handleServerNotification("item/started", (payload) => + withActive(payload.threadId, (active) => + Effect.sync(() => { + if (readRealtimeDelegationFromItem(payload.item) !== undefined) { + active.delegatedTurns.set(payload.turnId, undefined); + return; + } + const text = readUserMessageText(payload.item); + const expectedIndex = + text === undefined + ? -1 + : active.expectedHandoffs.findIndex((expected) => text.includes(expected)); + if (expectedIndex !== -1) { + active.expectedHandoffs.splice(expectedIndex, 1); + active.delegatedTurns.set(payload.turnId, undefined); + } + }), + ), + ); + yield* client.handleServerNotification("item/completed", (payload) => + withActive(payload.threadId, (active) => + Effect.sync(() => { + if (!active.delegatedTurns.has(payload.turnId)) return; + const answer = readSpeakableAnswer(payload.item); + if (answer !== undefined) active.delegatedTurns.set(payload.turnId, answer); + }), + ), + ); + yield* client.handleServerNotification("turn/completed", (payload) => + withActive(payload.threadId, (active) => + Effect.suspend(() => { + const answer = active.delegatedTurns.get(payload.turn.id); + active.delegatedTurns.delete(payload.turn.id); + if (answer === undefined || payload.turn.status !== "completed") return Effect.void; + return client.raw + .request("thread/realtime/appendSpeech", { + threadId: active.providerThreadId, + text: answer, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Codex voice could not speak the delegated answer", { cause }), + ), + Effect.forkIn(runtimeScope), + Effect.asVoid, + ); + }), + ), + ); + + const stop = (active: ActiveVoice) => + Effect.gen(function* () { + yield* Ref.update(activeRef, (current) => (current === active ? undefined : current)); + if (active.closed) return; + active.closed = true; + yield* client.raw + .request("thread/realtime/stop", { threadId: active.providerThreadId }) + .pipe(Effect.ignore); + }); + + /** + * Starts a voice conversation. The stream ends when Codex closes the + * session; interrupting it (client unsubscribe or disconnect) stops it. + */ + const start = ( + input: CodexVoiceStartInput, + ): Stream.Stream< + ProviderVoiceSessionEvent, + CodexVoiceSessionError | CodexErrors.CodexAppServerError + > => + Stream.unwrap( + Effect.gen(function* () { + const queue: VoiceQueue = yield* Queue.unbounded< + ProviderVoiceSessionEvent, + CodexVoiceSessionError | Cause.Done + >(); + const active: ActiveVoice = { + providerThreadId: input.providerThreadId, + queue, + delegatedTurns: new Map(), + expectedHandoffs: [], + closed: false, + }; + const previous = yield* Ref.getAndSet(activeRef, active); + if (previous && !previous.closed) { + // One conversation per thread: a newer client takes over. + yield* finish(previous, { type: "closed", reason: "replaced" }); + yield* client.raw + .request("thread/realtime/stop", { threadId: previous.providerThreadId }) + .pipe(Effect.ignore); + } + yield* Effect.addFinalizer(() => stop(active)); + yield* client.raw.request("thread/realtime/start", { + threadId: input.providerThreadId, + outputModality: "audio", + transport: { type: "webrtc", sdp: input.offerSdp }, + version: "v3", + clientManagedHandoffs: true, + includeStartupContext: false, + ...(input.voice ? { voice: input.voice } : {}), + }); + return Stream.fromQueue(queue); + }), + ); + + /** + * Marks the next turn whose user message carries `text` as a voice handoff, + * for hosts that resubmit Codex's delegation as their own run. + */ + const expectSpokenTurn = (text: string) => + Effect.map(Ref.get(activeRef), (active) => { + if (active && !active.closed) active.expectedHandoffs.push(text); + }); + + return { start, expectSpokenTurn }; +}); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index cc400c170f4a..16c6aef73e66 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -115,6 +115,7 @@ const makeCodexConfig = (overrides: Partial): CodexSettings => ({ shadowHomePath: "", launchArgs: "", customModels: [], + voice: "", ...overrides, }); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 375c2ac78aa1..cc4e453acba0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -74,6 +74,7 @@ import { ProjectWriteFileError, ProjectMutationError, ProviderUploadFeedbackError, + ProviderVoiceSessionError, ProviderSetupError, RelayClientInstallFailedError, type RelayClientInstallProgressEvent, @@ -2307,6 +2308,49 @@ const layerWsRpc = ( }), ), ), + [WS_METHODS.providerVoiceSession]: (input) => + Stream.unwrap( + Effect.gen(function* () { + const voiceError = (detail: string) => + new ProviderVoiceSessionError({ threadId: input.threadId, detail }); + const projection = yield* threadManagement.getThreadRecords(input.threadId, [ + "providerThreads", + ]); + const providerThread = + projection.providerThreads.find( + (candidate) => candidate.id === projection.thread.activeProviderThreadId, + ) ?? projection.providerThreads.at(-1); + const providerSessionId = providerThread?.providerSessionId ?? null; + if (providerThread === undefined || providerSessionId === null) { + return yield* voiceError("Send a message before starting voice."); + } + const runtime = Option.getOrNull(yield* providerSessionsV2.get(providerSessionId)); + if (runtime === null) { + return yield* voiceError( + "The provider session is no longer running. Send a message first.", + ); + } + if (runtime.startVoiceSession === undefined) { + return yield* voiceError( + `Provider '${runtime.driver}' does not support voice conversations.`, + ); + } + return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp }); + }), + ).pipe( + Stream.mapError((cause) => + cause._tag === "ProviderVoiceSessionError" + ? cause + : new ProviderVoiceSessionError({ + threadId: input.threadId, + detail: + "detail" in cause && typeof cause.detail === "string" + ? cause.detail + : cause.message, + cause, + }), + ), + ), [WS_METHODS.serverUpdateProvider]: (input) => providerMaintenanceRunner.updateProvider(input), [WS_METHODS.providerConsumeResetCredit]: (input) => diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index b760810a3876..76e299a09413 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -187,6 +187,8 @@ import { collapseExpandedComposerCursor, parseStandaloneComposerSlashCommand, } from "../composer-logic"; +import { resolveThreadVoiceAvailability } from "../voice/voiceAvailability"; +import { toggleThreadVoice, voiceMode } from "../voice/voiceMode"; import { derivePendingApprovals, derivePendingUserInputs, @@ -9229,18 +9231,32 @@ export default function ChatView(props: ChatViewProps) { } return; } - // Providers without the legacy toggle receive their native commands unchanged. - const standaloneSlashCommand = - sendInteractionModeEnabled && + const parsedSlashCommand = composerImages.length === 0 && composerFiles.length === 0 && sendableComposerTerminalContexts.length === 0 && composerPreviewAnnotations.length === 0 && composerReviewComments.length === 0 && - composerThreadContexts.length === 0 + composerThreadContexts.length === 0 && + multipleModelSelections === null ? parseStandaloneComposerSlashCommand(trimmed) : null; - if (standaloneSlashCommand && multipleModelSelections === null) { + if (parsedSlashCommand === "voice") { + // Providers without voice receive "/voice" as a normal message. + const voiceAvailability = resolveThreadVoiceAvailability(activeThread, providerStatuses); + // An active conversation always stops, even if the provider stopped offering voice. + if (voiceAvailability.kind !== "unsupported" || voiceMode.isActiveFor(routeThreadRef)) { + toggleThreadVoice(routeThreadRef, voiceAvailability); + promptRef.current = ""; + clearComposerDraftContent(composerDraftTarget); + composerRef.current?.resetCursorState(); + return; + } + } + // Providers without the legacy toggle receive their native commands unchanged. + const standaloneSlashCommand = + sendInteractionModeEnabled && parsedSlashCommand !== "voice" ? parsedSlashCommand : null; + if (standaloneSlashCommand) { handleInteractionModeChange(standaloneSlashCommand); promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 7280f7457baa..668954b7afa8 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -58,6 +58,7 @@ import { useLocation, useNavigate, useParams } from "@tanstack/react-router"; import * as Option from "effect/Option"; import { ArrowLeftIcon, + AudioLinesIcon, ChartNoAxesColumnIcon, CheckIcon, ChevronRightIcon, @@ -69,6 +70,8 @@ import { MessageSquareDashedIcon, LinkIcon, MessageSquareIcon, + MicIcon, + MicOffIcon, MonitorIcon, MoonIcon, PaletteIcon, @@ -163,6 +166,8 @@ import { import { selectThreadTerminalUiState, useTerminalUiStateStore } from "../terminalUiStateStore"; import { buildThreadRouteParams, resolveThreadRouteTarget } from "../threadRoutes"; import { useAvailableSettingsSearchItems } from "./settings/useAvailableSettingsSearchItems"; +import { resolveThreadVoiceAvailability } from "../voice/voiceAvailability"; +import { toggleThreadVoice, useVoiceModeState, voiceMode } from "../voice/voiceMode"; import { applyWslEnvironmentConfiguration, parseWslUncPath, @@ -665,6 +670,7 @@ function OpenCommandPaletteDialog(props: { const activeThreadServerConfig = useServerConfigs().get( activeThread?.environmentId ?? ("" as EnvironmentId), ); + const voiceState = useVoiceModeState(); const activeThreadReferenceCopyTarget = referenceThreadRef === null || (pathname === "/pull-requests" && !openPanelPullRequestUrl) ? null @@ -2007,6 +2013,66 @@ function OpenCommandPaletteDialog(props: { }); } + const voiceEngaged = voiceState.phase !== "idle"; + const voiceTarget = activeThread + ? scopeThreadRef(activeThread.environmentId, activeThread.id) + : null; + const voiceOnActiveThread = voiceTarget !== null && voiceMode.isActiveFor(voiceTarget); + if (activeThread !== null && voiceTarget !== null && !voiceOnActiveThread) { + const voiceAvailability = resolveThreadVoiceAvailability( + activeThread, + activeThreadServerConfig?.providers ?? [], + ); + if (voiceAvailability.kind !== "unsupported") { + actionItems.push({ + kind: "action", + value: "action:voice-start", + searchTerms: ["voice", "talk", "speak", "conversation", "microphone", "realtime"], + title: "Start voice conversation", + description: + voiceAvailability.kind === "unavailable" ? voiceAvailability.reason : undefined, + disabled: voiceAvailability.kind === "unavailable", + icon: , + shortcutCommand: "voice.toggle", + run: async () => { + toggleThreadVoice(voiceTarget, voiceAvailability); + }, + }); + } + } + // Stop and mute act on the one conversation, wherever it runs. + if (voiceEngaged) { + actionItems.push( + { + kind: "action", + value: "action:voice-stop", + searchTerms: ["voice", "stop", "end", "conversation", "hang up"], + title: "Stop voice conversation", + icon: , + // F8 toggles the current thread, so it only stops a conversation held here. + ...(voiceOnActiveThread ? { shortcutCommand: "voice.toggle" as const } : {}), + run: async () => { + voiceMode.stop(); + }, + }, + { + kind: "action", + value: "action:voice-mute", + searchTerms: ["voice", "mute", "unmute", "microphone"], + title: voiceState.muted ? "Unmute voice" : "Mute voice", + icon: voiceState.muted ? ( + + ) : ( + + ), + shortcutCommand: "voice.toggleMute", + run: async () => { + voiceMode.toggleMuted(); + }, + }, + ); + } + actionItems.push({ kind: "action", value: "action:open-file-picker", diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 9162c4b67e0d..2073184ccb5d 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -341,6 +341,9 @@ import { } from "./composerScrollGesture"; import { prepareVideoFirstFrame } from "../../lib/videoFirstFrame"; import { observeResize } from "~/lib/observeResize"; +import { resolveVoiceAvailability } from "../../voice/voiceAvailability"; +import { toggleThreadVoice, useVoicePhaseFor, voiceMode } from "../../voice/voiceMode"; +import { VoiceModeStrip } from "./VoiceModeStrip"; function ComposerVideoThumbnail({ file }: { file: File }) { const setVideo = useCallback( @@ -1077,10 +1080,12 @@ function ComposerCommandMenuLayer(props: { anchor: HTMLElement | null; children: ); } import { Button } from "../ui/button"; +import { Toggle } from "../ui/toggle"; import { Select, SelectItem, SelectPopup, SelectValue } from "../ui/select"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { toastManager } from "../ui/toast"; import { + AudioLinesIcon, FileIcon, BotIcon, CircleAlertIcon, @@ -2240,6 +2245,34 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) [selectedProviderEntry], ); const compactCommandAvailable = providerSupportsManualCompaction(selectedProviderEntry); + // Voice follows the provider that owns the thread; a draft without a thread + // shell follows the composer's selection and waits for its first message. + const voiceThread = activeThread ?? props.activeThreadShell ?? null; + const voiceProviderInstanceId = voiceThread?.providerInstanceId ?? selectedInstanceId; + const voiceThreadHasMessages = voiceThread?.latestUserMessageAt != null; + const voiceThreadHasSession = voiceThread?.activeProviderThreadId != null; + const voiceAvailability = useMemo( + () => + resolveVoiceAvailability({ + provider: providerStatuses.find( + (provider) => provider.instanceId === voiceProviderInstanceId, + ), + hasMessages: voiceThreadHasMessages, + hasSession: voiceThreadHasSession, + }), + [providerStatuses, voiceProviderInstanceId, voiceThreadHasMessages, voiceThreadHasSession], + ); + const voicePhase = useVoicePhaseFor(routeThreadRef); + const voiceEngaged = voicePhase !== "idle"; + // A conversation stays stoppable here even if its provider stopped offering voice. + const voiceSupported = voiceAvailability.kind !== "unsupported" || voiceEngaged; + const toggleVoice = useCallback( + () => toggleThreadVoice(routeThreadRef, voiceAvailability), + [routeThreadRef, voiceAvailability], + ); + const voiceButtonDisabledReason = + voiceAvailability.kind === "unavailable" && !voiceEngaged ? voiceAvailability.reason : null; + const voiceButtonLabel = voiceEngaged ? "Stop voice conversation" : "Start voice conversation"; // Memoized so the composer menu memo below can cache between renders. const selectedProviderSkills = useMemo( () => @@ -2737,6 +2770,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }, ] as const) : []), + ...(voiceSupported + ? ([ + { + id: "slash:voice", + type: "slash-command", + command: "voice", + label: "/voice", + description: voiceEngaged + ? "Stop the voice conversation" + : "Start a voice conversation", + }, + ] as const) + : []), ] satisfies ReadonlyArray>; const slashMenuSkills = getProviderSkillsForSlashMenu( selectedProviderSkills, @@ -2856,6 +2902,8 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) selectedProviderSlashCommands, selectedProviderStatus, settings.showSkillsInSlashMenu, + voiceEngaged, + voiceSupported, workspaceEntries.entries, ]); @@ -4052,6 +4100,16 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) } return; } + if (item.command === "voice") { + const applied = applyPromptReplacement(trigger.rangeStart, trigger.rangeEnd, "", { + expectedText: snapshot.value.slice(trigger.rangeStart, trigger.rangeEnd), + }); + if (applied) { + setComposerHighlightedItemId(null); + toggleVoice(); + } + return; + } if (!planModeUiEnabled) return; void handleInteractionModeChange(item.command === "plan" ? "plan" : "default"); const applied = applyPromptReplacement(trigger.rangeStart, trigger.rangeEnd, "", { @@ -4174,6 +4232,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) planModeUiEnabled, onUsageLimitsCommand, resolveActiveComposerTrigger, + toggleVoice, ], ); @@ -5773,6 +5832,32 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) terminalOpen, ]); + useEffect(() => { + const handler = (event: globalThis.KeyboardEvent) => { + const command = resolveShortcutCommand(event, keybindings, { + context: { + terminalFocus: getTerminalFocusOwner() !== null, + terminalOpen, + modelPickerOpen: isComposerModelPickerOpen, + }, + }); + if (command === "voice.toggleMute") { + // Muting is global: the microphone belongs to the one conversation. + if (voiceMode.getState().phase === "idle") return; + event.preventDefault(); + event.stopPropagation(); + if (!event.repeat) voiceMode.toggleMuted(); + return; + } + if (command !== "voice.toggle" || !voiceSupported || isCommandPaletteOpen()) return; + event.preventDefault(); + event.stopPropagation(); + if (!event.repeat) toggleVoice(); + }; + window.addEventListener("keydown", handler, true); + return () => window.removeEventListener("keydown", handler, true); + }, [isComposerModelPickerOpen, keybindings, terminalOpen, toggleVoice, voiceSupported]); + // ------------------------------------------------------------------ // Callbacks: attachments // ------------------------------------------------------------------ @@ -6779,6 +6864,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ) : null} + {voiceEngaged ? ( + + + + ) : null} {showComposerTopDrawer && (!isTasksDrawerOpen || hasBlockingComposerTopDrawer) ? ( ) : null} + {voiceSupported && !isComposerResting ? ( + + event.preventDefault()} + onPressedChange={() => { + if (voiceButtonDisabledReason === null) toggleVoice(); + }} + aria-disabled={voiceButtonDisabledReason !== null || undefined} + aria-label={voiceButtonLabel} + /> + } + > + {/* Dim the glyph, not the Toggle, which owns its own disabled styling. */} + + + {voiceButtonDisabledReason ?? voiceButtonLabel} + + ) : null} ; + label: string; + /** Fixed bar height when levels are not animated (reduced motion). */ + staticLevel: number; +}) { + return ( + + {METER_BARS.map((bar) => ( + + ))} + + ); +} + +/** + * Live status for the thread's voice conversation: state, mic and speaker + * meters, mute and stop, and the latest captions. Meters are written straight + * to the DOM from the controller's level samples so the composer never + * re-renders per sample. + */ +export const VoiceModeStrip = memo(function VoiceModeStrip({ + target, +}: { + target: VoiceSessionTarget; +}) { + const state = useVoiceModeState(); + const isOnThread = isVoiceTargetFor(state, target) && state.phase !== "idle"; + const isActive = isOnThread && state.phase === "active"; + const prefersReducedMotion = useMediaQuery("(prefers-reduced-motion: reduce)"); + const microphoneMeterRef = useRef(null); + const speakerMeterRef = useRef(null); + const [speaking, setSpeaking] = useState(false); + + useEffect(() => { + if (!isActive) return; + // Teardown publishes a silent sample, which clears `speaking` for the next conversation. + return voiceMode.subscribeLevels((levels) => { + setSpeaking(levels.speaker > SPEAKING_LEVEL); + if (prefersReducedMotion) return; + applyMeterLevel(microphoneMeterRef.current, levels.microphone); + applyMeterLevel(speakerMeterRef.current, levels.speaker); + }); + }, [isActive, prefersReducedMotion]); + + if (!isOnThread) return null; + + const status = !isActive + ? "Connecting…" + : speaking + ? "Speaking" + : state.muted + ? "Muted" + : "Listening"; + const captions = state.captions.slice(-VISIBLE_CAPTIONS); + // Reduced motion keeps the meters still: full while a channel is live, low while muted. + const staticMicrophoneLevel = prefersReducedMotion && isActive && !state.muted ? 0.4 : 0; + const staticSpeakerLevel = prefersReducedMotion && speaking ? 0.4 : 0; + + return ( + + + {isActive ? : } + + + {status} + + {isActive ? ( + + + {state.muted ? ( + + ) : ( + + )} + + + + + + + + ) : null} + + + + + + + {captions.length > 0 ? ( + + {captions.map((caption, index) => ( +

+ {caption.text} +

+ ))} + + ) : null} + + ); +}); diff --git a/apps/web/src/components/settings/ProviderSettingsForm.test.ts b/apps/web/src/components/settings/ProviderSettingsForm.test.ts index ff1de91addc9..b7100208bb88 100644 --- a/apps/web/src/components/settings/ProviderSettingsForm.test.ts +++ b/apps/web/src/components/settings/ProviderSettingsForm.test.ts @@ -17,6 +17,7 @@ describe("ProviderSettingsForm helpers", () => { "homePath", "shadowHomePath", "launchArgs", + "voice", ]); }); diff --git a/apps/web/src/composer-logic.test.ts b/apps/web/src/composer-logic.test.ts index df3e54b4ddfc..f3c3eb75269e 100644 --- a/apps/web/src/composer-logic.test.ts +++ b/apps/web/src/composer-logic.test.ts @@ -813,8 +813,13 @@ describe("parseStandaloneComposerSlashCommand", () => { expect(parseStandaloneComposerSlashCommand("/default")).toBe("default"); }); + it("parses standalone /voice command", () => { + expect(parseStandaloneComposerSlashCommand("/Voice ")).toBe("voice"); + }); + it("ignores slash commands with extra message text", () => { expect(parseStandaloneComposerSlashCommand("/plan explain this")).toBeNull(); + expect(parseStandaloneComposerSlashCommand("/voice please")).toBeNull(); }); }); diff --git a/apps/web/src/composer-logic.ts b/apps/web/src/composer-logic.ts index a44b999f2ba6..c27a6735c71a 100644 --- a/apps/web/src/composer-logic.ts +++ b/apps/web/src/composer-logic.ts @@ -12,7 +12,7 @@ import { import { resolveShortcutCommand, type ShortcutEventLike } from "./keybindings"; export type ComposerTriggerKind = "path" | "pull-request" | "slash-command" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "voice"; export type ComposerSubmissionIntent = "foreground" | "background" | "alternate"; export interface ComposerTrigger { @@ -342,12 +342,13 @@ export function composerStateAtPromptEnd( export function parseStandaloneComposerSlashCommand( text: string, ): Exclude | null { - const match = /^\/(plan|default)\s*$/i.exec(text.trim()); + const match = /^\/(plan|default|voice)\s*$/i.exec(text.trim()); if (!match) { return null; } const command = match[1]?.toLowerCase(); if (command === "plan") return "plan"; + if (command === "voice") return "voice"; return "default"; } diff --git a/apps/web/src/keybindings.test.ts b/apps/web/src/keybindings.test.ts index d1e842205efb..cd7f43e4b45f 100644 --- a/apps/web/src/keybindings.test.ts +++ b/apps/web/src/keybindings.test.ts @@ -643,6 +643,23 @@ describe("thread navigation helpers", () => { ), ); }); + + it("toggles voice with a bare F8 unless the terminal has focus", () => { + const input = event({ key: "F8", code: "F8" }); + assert.strictEqual( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: false }, + }), + "voice.toggle", + ); + assert.isNull( + resolveShortcutCommand(input, DEFAULT_RESOLVED_KEYBINDINGS, { + platform: "MacIntel", + context: { terminalFocus: true }, + }), + ); + }); }); describe("model picker navigation helpers", () => { diff --git a/apps/web/src/voice/voiceAvailability.test.ts b/apps/web/src/voice/voiceAvailability.test.ts new file mode 100644 index 000000000000..0d5b0ff85e1f --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.test.ts @@ -0,0 +1,40 @@ +import { ProviderInstanceId, type ServerProvider } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { resolveThreadVoiceAvailability } from "./voiceAvailability"; + +const codex = ProviderInstanceId.make("codex"); +const claude = ProviderInstanceId.make("claudeAgent"); +const providers = [ + { instanceId: codex, supportsVoice: true }, + { instanceId: claude }, +] as unknown as ReadonlyArray; + +function thread(input: { readonly provider: ProviderInstanceId; readonly messages?: boolean }) { + return { + providerInstanceId: input.provider, + latestUserMessageAt: input.messages ? "2026-09-22T00:00:00.000Z" : null, + activeProviderThreadId: input.messages ? "provider-thread-1" : null, + }; +} + +describe("resolveThreadVoiceAvailability", () => { + it("hides voice for providers that do not support it", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: claude, messages: true }), providers), + ).toEqual({ kind: "unsupported" }); + }); + + it("explains that a new thread needs a first message", () => { + expect(resolveThreadVoiceAvailability(thread({ provider: codex }), providers)).toEqual({ + kind: "unavailable", + reason: "Send a message before starting voice", + }); + }); + + it("is ready once a Codex thread has run", () => { + expect( + resolveThreadVoiceAvailability(thread({ provider: codex, messages: true }), providers), + ).toEqual({ kind: "ready" }); + }); +}); diff --git a/apps/web/src/voice/voiceAvailability.ts b/apps/web/src/voice/voiceAvailability.ts new file mode 100644 index 000000000000..1e919f3b8d20 --- /dev/null +++ b/apps/web/src/voice/voiceAvailability.ts @@ -0,0 +1,46 @@ +import type { ProviderInstanceId, ServerProvider } from "@t3tools/contracts"; + +/** + * Whether a thread can hold a voice conversation. `unsupported` hides every + * voice entry point; `unavailable` shows them disabled with `reason`. + */ +export type VoiceAvailability = + | { readonly kind: "unsupported" } + | { readonly kind: "unavailable"; readonly reason: string } + | { readonly kind: "ready" }; + +const VOICE_UNSUPPORTED: VoiceAvailability = { kind: "unsupported" }; + +export function resolveVoiceAvailability(input: { + /** The provider instance that runs (or will run) the thread's turns. */ + readonly provider: Pick | null | undefined; + readonly hasMessages: boolean; + readonly hasSession: boolean; +}): VoiceAvailability { + if (input.provider?.supportsVoice !== true) return VOICE_UNSUPPORTED; + // The server hands voice requests to the thread's bound provider session, + // which only exists once a turn has been sent. + if (!input.hasMessages || !input.hasSession) { + return { kind: "unavailable", reason: "Send a message before starting voice" }; + } + return { kind: "ready" }; +} + +/** The thread fields voice availability reads (orchestrator V2 thread shells). */ +export interface VoiceThread { + readonly providerInstanceId: ProviderInstanceId; + readonly latestUserMessageAt: string | null; + readonly activeProviderThreadId: string | null; +} + +/** Voice follows the provider instance that owns the thread. */ +export function resolveThreadVoiceAvailability( + thread: VoiceThread, + providers: ReadonlyArray, +): VoiceAvailability { + return resolveVoiceAvailability({ + provider: providers.find((provider) => provider.instanceId === thread.providerInstanceId), + hasMessages: thread.latestUserMessageAt !== null, + hasSession: thread.activeProviderThreadId !== null, + }); +} diff --git a/apps/web/src/voice/voiceMode.ts b/apps/web/src/voice/voiceMode.ts new file mode 100644 index 000000000000..e0b02ee04fe4 --- /dev/null +++ b/apps/web/src/voice/voiceMode.ts @@ -0,0 +1,144 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + VoiceModeController, + describeVoiceEnd, + makeVoiceSessionOpener, + type VoiceMediaStream, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionTarget, +} from "@t3tools/client-runtime/voice-mode"; +import * as Effect from "effect/Effect"; +import { AtomRegistry } from "effect/reactivity"; +import { useSyncExternalStore } from "react"; + +import { stackedThreadToast, toastManager } from "../components/ui/toast"; +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import type { VoiceAvailability } from "./voiceAvailability"; + +function getMicrophone(): Promise { + const mediaDevices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices; + // Browsers only expose the microphone API on secure origins, so a remote + // environment opened over plain HTTP has no `mediaDevices` at all. + if (!mediaDevices?.getUserMedia) { + return Promise.reject( + new Error("Voice needs microphone access, which requires HTTPS when connecting remotely."), + ); + } + return mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); +} + +function playRemoteAudio(stream: VoiceMediaStream): () => void { + const audio = new Audio(); + audio.autoplay = true; + audio.srcObject = stream as MediaStream; + void audio.play().catch(() => undefined); + return () => { + audio.pause(); + audio.srcObject = null; + }; +} + +// The voice stream runs on the connection runtime, which owns the environment +// registry. Mounting keeps that runtime alive for the conversation. +const openSession = makeVoiceSessionOpener((effect) => + Effect.runFork( + Effect.scoped( + AtomRegistry.mount(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.andThen(AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime)), + Effect.flatMap((context) => Effect.provideContext(effect, context)), + ), + ).pipe(Effect.orDie), + ), +); + +const platform: VoiceModePlatform = { + // The DOM's overloaded addEventListener does not fit the structural `never` + // listener in VoicePeerConnection, though the runtime shape matches. + createPeerConnection: () => new RTCPeerConnection() as unknown as VoicePeerConnection, + getMicrophone, + playRemoteAudio, + openSession, +}; + +/** The app's single voice conversation. The microphone is exclusive, so there is one per client. */ +export const voiceMode = new VoiceModeController(platform); + +export function isVoiceTargetFor( + state: VoiceModeState, + target: VoiceSessionTarget | null, +): boolean { + return ( + target !== null && + state.target !== null && + state.target.environmentId === target.environmentId && + state.target.threadId === target.threadId + ); +} + +export function useVoiceModeState(): VoiceModeState { + return useSyncExternalStore(voiceMode.subscribe, voiceMode.getState, voiceMode.getState); +} + +/** The conversation phase on `target`; "idle" while voice is off or on another thread. */ +export function useVoicePhaseFor(target: VoiceSessionTarget | null): VoiceModePhase { + const getPhase = () => { + const state = voiceMode.getState(); + return isVoiceTargetFor(state, target) ? state.phase : "idle"; + }; + return useSyncExternalStore(voiceMode.subscribe, getPhase, getPhase); +} + +/** + * Starts or stops voice on `target`. An unavailable thread explains why + * instead of failing on the server. + */ +export function toggleThreadVoice(target: VoiceSessionTarget, availability: VoiceAvailability) { + if (voiceMode.isActiveFor(target)) { + voiceMode.stop(); + return; + } + if (availability.kind === "unsupported") return; + if (availability.kind === "unavailable") { + toastManager.add({ type: "info", title: availability.reason }); + return; + } + void voiceMode.start(target); +} + +// Errors and provider-side ends surface once as a toast on the conversation's +// thread. A user stop leaves neither set, so it stays silent. +let lastPhase: VoiceModePhase = "idle"; +voiceMode.subscribe((state) => { + const previousPhase = lastPhase; + lastPhase = state.phase; + if (state.phase !== "idle" || state.target === null) return; + if (state.error === null && state.endedReason === null) return; + const threadRef = scopeThreadRef( + EnvironmentId.make(state.target.environmentId), + ThreadId.make(state.target.threadId), + ); + if (state.error !== null) { + toastManager.add( + stackedThreadToast({ + type: "error", + title: previousPhase === "active" ? "Voice conversation ended" : "Couldn't start voice", + description: state.error, + data: { threadRef }, + }), + ); + } else if (state.endedReason !== null) { + toastManager.add({ + type: "info", + title: describeVoiceEnd(state.endedReason), + data: { threadRef }, + }); + } + voiceMode.dismiss(); +}); diff --git a/docs/internals/voice-conversations.md b/docs/internals/voice-conversations.md new file mode 100644 index 000000000000..1b9dc06e08ea --- /dev/null +++ b/docs/internals/voice-conversations.md @@ -0,0 +1,34 @@ +# Voice conversations + +A voice conversation is a Codex realtime session bound to one thread. The user +talks to a voice model, and when it wants work done it starts an ordinary Codex +turn on that thread. This is separate from [voice input](voice-input.md), which +only transcribes into the composer. + +Audio never touches the T3 socket. The client owns the microphone, speaker, and +WebRTC peer, and audio flows directly between the client and OpenAI. The +`provider.voice.session` stream carries only the SDP handshake, captions, and +lifecycle. That keeps remote and tunneled clients as cheap as local ones, and it +is why the [shared controller](../../packages/client-runtime/src/voice-mode/controller.ts) +takes a platform object: browsers and react-native-webrtc supply the same peer API. + +The stream's lifetime is the conversation's lifetime. When the client +unsubscribes or disconnects, the server stops the realtime session, so a dropped +client cannot leave a billed session running. + +Handoffs are client-managed, as in the Codex TUI. The +[Codex voice module](../../apps/server/src/provider/Layers/CodexVoice.ts) hands +each voice turn's final answer back with `thread/realtime/appendSpeech`, which +keeps commentary and reasoning unspoken. + +Codex starts or steers the delegated turn itself, with a user message wrapped in +``, and it cannot be told not to. Steering an active run +needs nothing special. An idle thread is different: V2 runs are +orchestrator-owned and there is no provider-initiated run yet, so +[CodexAdapterV2](../../apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts) +interrupts that turn and, once it settles, offers a `voice` continuation. The +request then runs as an ordinary user message, and Codex's history keeps a stub +of the interrupted turn. Captions are ephemeral and are not persisted. + +A thread needs an existing provider session binding before voice can start, +because Codex attaches realtime to an existing provider thread. diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index bdc87902a63b..b044b74cc8c6 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -92,6 +92,29 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques the current session, or permanently. See [Permission modes](./permission-modes.md) for command and file approvals. +## Voice conversations + +A Codex thread can hold a spoken conversation. You talk with +a voice model that hands coding requests to Codex and reads its answers back to +you. Send at least one message in the thread first. + +Start or stop it with the waveform button in the composer. On web and desktop you +can also send `/voice` or press `F8`. While it runs, the strip above the composer shows whether it is +listening or speaking, with buttons to mute your microphone and stop. The command +palette offers the same actions, and **Voice: Toggle Mute** can be bound in +**Settings > Keybindings**. + +Choose the voice in **Settings > Providers > Codex > Voice**. The change applies +to your next conversation. + +On mobile, the conversation stops when you leave the thread or the app. The mobile +app needs an update that includes voice support. + +Voice needs a microphone and permission to use it. When you open T3 Code from +another device, the page must be served over HTTPS for the browser to allow the +microphone. Live captions are not saved: the thread keeps only the requests +handed to Codex and Codex's replies. + ## Codex says I hit a usage limit When Codex stops on a usage limit, the thread names the window that ran out and diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0366c65b1ed2..c7e54c7ec9d8 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -111,6 +111,10 @@ "types": "./src/voice-input/index.ts", "default": "./src/voice-input/index.ts" }, + "./voice-mode": { + "types": "./src/voice-mode/index.ts", + "default": "./src/voice-mode/index.ts" + }, "./t3ToolSummary": { "types": "./src/t3ToolSummary.ts", "default": "./src/t3ToolSummary.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index c11a49897b88..f890e18b110c 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -76,6 +76,7 @@ export type EnvironmentStreamCommandRpcTag = | typeof WS_METHODS.chatGptHandoffSubscribe | typeof WS_METHODS.cloudInstallRelayClient | typeof WS_METHODS.serverUpdateServerWithProgress + | typeof WS_METHODS.providerVoiceSession | typeof WS_METHODS.gitRunStackedAction; export type EnvironmentStreamRpcTag = diff --git a/packages/client-runtime/src/voice-mode/controller.test.ts b/packages/client-runtime/src/voice-mode/controller.test.ts new file mode 100644 index 000000000000..1be4869bc3f2 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + appendVoiceCaption, + finishVoiceCaption, + VoiceModeController, + type VoiceMediaStream, + type VoiceModePlatform, + type VoicePeerConnection, + type VoiceSessionHandlers, +} from "./controller.ts"; + +const target = { environmentId: "env-1", threadId: "thread-1" }; + +function makeFakePlatform(options: { readonly denyMicrophone?: boolean } = {}) { + const track = { enabled: true, stopped: false, stop: () => void (track.stopped = true) }; + const microphone: VoiceMediaStream = { getTracks: () => [track], getAudioTracks: () => [track] }; + const listeners = new Map void>(); + const peer = { + connectionState: "new", + closed: false, + remoteSdp: null as string | null, + dataChannels: [] as string[], + addTrack: () => undefined, + createDataChannel: (label: string) => void peer.dataChannels.push(label), + createOffer: async () => ({ sdp: "v=offer\r\n" }), + setLocalDescription: async () => undefined, + setRemoteDescription: async (description: { sdp: string }) => { + peer.remoteSdp = description.sdp; + }, + getStats: async () => ({ forEach: () => undefined }), + addEventListener: (type: string, listener: (event: never) => void) => + void listeners.set(type, listener), + close: () => void (peer.closed = true), + connect: () => { + peer.connectionState = "connected"; + listeners.get("connectionstatechange")?.({} as never); + }, + }; + const sessions: Array<{ + readonly offerSdp: string; + readonly handlers: VoiceSessionHandlers; + cancelled: boolean; + }> = []; + const platform: VoiceModePlatform = { + createPeerConnection: () => peer as unknown as VoicePeerConnection, + getMicrophone: async () => { + if (options.denyMicrophone) { + throw Object.assign(new Error("denied"), { name: "NotAllowedError" }); + } + return microphone; + }, + playRemoteAudio: () => () => undefined, + openSession: (input, handlers) => { + const session = { offerSdp: input.offerSdp, handlers, cancelled: false }; + sessions.push(session); + return () => void (session.cancelled = true); + }, + }; + return { platform, peer, track, sessions }; +} + +async function startConversation() { + const fake = makeFakePlatform(); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + const session = fake.sessions[0]!; + return { ...fake, controller, session }; +} + +describe("VoiceModeController", () => { + it("negotiates WebRTC through the voice session and activates once both sides are up", async () => { + const { controller, peer, session } = await startConversation(); + expect(peer.dataChannels).toEqual(["oai-events"]); + expect(session.offerSdp).toBe("v=offer\r\n"); + expect(controller.getState().phase).toBe("connecting"); + + session.handlers.onEvent({ type: "answer", sdp: "v=answer\r\n" }); + await Promise.resolve(); + expect(peer.remoteSdp).toBe("v=answer\r\n"); + + session.handlers.onEvent({ type: "started" }); + expect(controller.getState().phase).toBe("connecting"); + peer.connect(); + expect(controller.getState().phase).toBe("active"); + controller.stop(); + }); + + it("keeps ephemeral captions and reports why Codex ended the conversation", async () => { + const { controller, peer, session, track } = await startConversation(); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: "run the" }); + session.handlers.onEvent({ type: "transcript.delta", role: "user", delta: " tests" }); + expect(controller.getState().captions).toEqual([ + { role: "user", text: "run the tests", final: false }, + ]); + + session.handlers.onEvent({ type: "closed", reason: "transport_closed" }); + const state = controller.getState(); + expect(state.phase).toBe("idle"); + expect(state.captions).toEqual([]); + expect(state.endedReason).toBe("transport_closed"); + expect(state.target).toEqual(target); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + }); + + it("ends the server session and releases the microphone when the user stops", async () => { + const { controller, peer, session, track } = await startConversation(); + controller.toggle(target); + expect(session.cancelled).toBe(true); + expect(peer.closed).toBe(true); + expect(track.stopped).toBe(true); + expect(controller.getState()).toMatchObject({ phase: "idle", endedReason: null, error: null }); + }); + + it("mutes by disabling the microphone track", async () => { + const { controller, track } = await startConversation(); + controller.toggleMuted(); + expect(track.enabled).toBe(false); + expect(controller.getState().muted).toBe(true); + controller.toggleMuted(); + expect(track.enabled).toBe(true); + controller.stop(); + }); + + it("explains a denied microphone without opening a session", async () => { + const fake = makeFakePlatform({ denyMicrophone: true }); + const controller = new VoiceModeController(fake.platform); + await controller.start(target); + expect(fake.sessions).toHaveLength(0); + expect(controller.getState()).toMatchObject({ + phase: "idle", + error: "Microphone access was denied.", + }); + }); +}); + +describe("voice captions", () => { + it("interleaves speakers and settles each with its final text", () => { + let captions = appendVoiceCaption([], "user", "hello"); + captions = appendVoiceCaption(captions, "assistant", "Hi"); + captions = appendVoiceCaption(captions, "user", " there"); + captions = finishVoiceCaption(captions, "user", "Hello there."); + expect(captions).toEqual([ + { role: "user", text: "Hello there.", final: true }, + { role: "assistant", text: "Hi", final: false }, + ]); + }); + + it("keeps only the most recent captions", () => { + let captions = finishVoiceCaption([], "user", "one"); + for (const text of ["two", "three", "four", "five"]) { + captions = finishVoiceCaption(captions, "assistant", text); + } + expect(captions.map((caption) => caption.text)).toEqual(["two", "three", "four", "five"]); + }); +}); diff --git a/packages/client-runtime/src/voice-mode/controller.ts b/packages/client-runtime/src/voice-mode/controller.ts new file mode 100644 index 000000000000..8605cc340ff2 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/controller.ts @@ -0,0 +1,433 @@ +// @effect-diagnostics globalTimers:off - WebRTC lifecycle runs on platform callbacks, outside an Effect runtime. +import type { ProviderVoiceSessionEvent, ProviderVoiceTranscriptRole } from "@t3tools/contracts"; + +/** + * Voice conversations (Codex realtime voice). The client owns the microphone, + * speaker, and WebRTC peer; the environment only relays the SDP handshake and + * captions over `provider.voice.session`. Audio never crosses the T3 socket, + * so voice works the same locally, over a relay, or through a tunnel. + * + * The WebRTC surface is structural so browsers and react-native-webrtc both fit. + */ + +export interface VoiceMediaTrack { + enabled: boolean; + stop(): void; +} + +export interface VoiceMediaStream { + getTracks(): ReadonlyArray; + getAudioTracks(): ReadonlyArray; + /** react-native-webrtc frees the native stream here; browsers have no equivalent. */ + release?(): void; +} + +interface VoiceSessionDescription { + readonly sdp?: string | null | undefined; +} + +interface VoiceStatsReport { + forEach(callback: (stat: { readonly type?: string; readonly kind?: string }) => void): void; +} + +export interface VoicePeerConnection { + readonly connectionState: string; + addTrack(track: never, stream: never): unknown; + createDataChannel(label: string, options?: { readonly ordered?: boolean }): unknown; + createOffer(): Promise; + setLocalDescription(description: never): Promise; + setRemoteDescription(description: never): Promise; + getStats(): Promise; + addEventListener(type: "track" | "connectionstatechange", listener: (event: never) => void): void; + close(): void; +} + +export interface VoiceSessionTarget { + readonly environmentId: string; + readonly threadId: string; +} + +export interface VoiceSessionHandlers { + readonly onEvent: (event: ProviderVoiceSessionEvent) => void; + readonly onError: (message: string) => void; + readonly onEnd: () => void; +} + +export interface VoiceModePlatform { + readonly createPeerConnection: () => VoicePeerConnection; + /** Resolves an echo-cancelled microphone stream, prompting for permission if needed. */ + readonly getMicrophone: () => Promise; + /** Plays the assistant's audio. Returns a disposer. */ + readonly playRemoteAudio: (stream: VoiceMediaStream) => () => void; + /** Opens `provider.voice.session`. Returns a cancel function that ends the conversation. */ + readonly openSession: ( + input: VoiceSessionTarget & { readonly offerSdp: string }, + handlers: VoiceSessionHandlers, + ) => () => void; +} + +export type VoiceModePhase = "idle" | "connecting" | "active"; + +export interface VoiceCaption { + readonly role: ProviderVoiceTranscriptRole; + readonly text: string; + readonly final: boolean; +} + +export interface VoiceModeState { + readonly phase: VoiceModePhase; + /** The conversation's thread; kept after it ends so its error or end notice can show there. */ + readonly target: VoiceSessionTarget | null; + readonly muted: boolean; + /** Recent captions, oldest first. Ephemeral: they vanish when the conversation ends. */ + readonly captions: ReadonlyArray; + readonly error: string | null; + /** Why the last conversation ended when the user did not end it. */ + readonly endedReason: string | null; +} + +/** Mic and speaker loudness in 0..1, sampled while a conversation is active. */ +export interface VoiceLevels { + readonly microphone: number; + readonly speaker: number; +} + +const VOICE_CONNECT_TIMEOUT_MS = 20_000; +const LEVEL_SAMPLE_INTERVAL_MS = 100; +const MAX_CAPTIONS = 4; +const MAX_CAPTION_CHARS = 1024; + +export const INITIAL_VOICE_MODE_STATE: VoiceModeState = { + phase: "idle", + target: null, + muted: false, + captions: [], + error: null, + endedReason: null, +}; + +/** Appends a streaming caption delta, continuing the speaker's open caption. */ +export function appendVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + delta: string, +): ReadonlyArray { + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return [...captions, { role, text: delta.trimStart(), final: false }].slice(-MAX_CAPTIONS); + } + const open = captions[openIndex]!; + const text = (open.text + delta).slice(-MAX_CAPTION_CHARS); + return captions.map((caption, index) => (index === openIndex ? { ...caption, text } : caption)); +} + +/** Settles the speaker's open caption with the final transcript text. */ +export function finishVoiceCaption( + captions: ReadonlyArray, + role: ProviderVoiceTranscriptRole, + text: string, +): ReadonlyArray { + const finalText = text.trim().slice(-MAX_CAPTION_CHARS); + const openIndex = captions.findLastIndex((caption) => caption.role === role && !caption.final); + if (openIndex === -1) { + return finalText + ? [...captions, { role, text: finalText, final: true }].slice(-MAX_CAPTIONS) + : captions; + } + if (!finalText) return captions.filter((_, index) => index !== openIndex); + return captions.map((caption, index) => + index === openIndex ? { role, text: finalText, final: true } : caption, + ); +} + +/** User-facing text for why a conversation ended without the user stopping it. */ +export function describeVoiceEnd(reason: string): string { + switch (reason) { + case "replaced": + return "Voice conversation moved to another window or device."; + case "ended": + return "Voice conversation ended."; + default: + return `Voice conversation ended: ${reason.replaceAll("_", " ")}.`; + } +} + +function describeError(cause: unknown): string { + if (cause && typeof cause === "object" && "name" in cause) { + const name = String((cause as { name: unknown }).name); + if (name === "NotAllowedError" || name === "SecurityError") { + return "Microphone access was denied."; + } + if (name === "NotFoundError") return "No microphone was found."; + } + if (cause instanceof Error && cause.message) return cause.message; + return "Voice conversation failed."; +} + +interface ActiveSession { + readonly target: VoiceSessionTarget; + readonly id: number; + microphone: VoiceMediaStream | null; + peer: VoicePeerConnection | null; + cancelSession: (() => void) | null; + stopRemoteAudio: (() => void) | null; + backendStarted: boolean; + peerConnected: boolean; + connectTimer: ReturnType | null; + levelTimer: ReturnType | null; + sampling: boolean; +} + +/** + * Owns at most one voice conversation. Starting on another thread ends the + * current one first, since the microphone is exclusive. + */ +export class VoiceModeController { + private state: VoiceModeState = INITIAL_VOICE_MODE_STATE; + private session: ActiveSession | null = null; + private nextSessionId = 1; + private readonly listeners = new Set<(state: VoiceModeState) => void>(); + private readonly levelListeners = new Set<(levels: VoiceLevels) => void>(); + + private readonly platform: VoiceModePlatform; + + constructor(platform: VoiceModePlatform) { + this.platform = platform; + } + + getState = (): VoiceModeState => this.state; + + subscribe = (listener: (state: VoiceModeState) => void): (() => void) => { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + }; + + subscribeLevels = (listener: (levels: VoiceLevels) => void): (() => void) => { + this.levelListeners.add(listener); + return () => this.levelListeners.delete(listener); + }; + + isActiveFor(target: VoiceSessionTarget): boolean { + const current = this.state.target; + return ( + this.state.phase !== "idle" && + current !== null && + current.environmentId === target.environmentId && + current.threadId === target.threadId + ); + } + + toggle(target: VoiceSessionTarget): void { + if (this.isActiveFor(target)) { + this.stop(); + } else { + void this.start(target); + } + } + + async start(target: VoiceSessionTarget): Promise { + this.stop(); + const session: ActiveSession = { + target, + id: this.nextSessionId++, + microphone: null, + peer: null, + cancelSession: null, + stopRemoteAudio: null, + backendStarted: false, + peerConnected: false, + connectTimer: null, + levelTimer: null, + sampling: false, + }; + this.session = session; + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + phase: "connecting", + target, + muted: this.state.muted, + }); + + try { + const microphone = await this.platform.getMicrophone(); + if (this.session !== session) { + for (const track of microphone.getTracks()) track.stop(); + microphone.release?.(); + return; + } + session.microphone = microphone; + // Timed from here so a first-run permission prompt doesn't count against it. + session.connectTimer = setTimeout( + () => this.fail(session, "Voice connection timed out."), + VOICE_CONNECT_TIMEOUT_MS, + ); + for (const track of microphone.getAudioTracks()) track.enabled = !this.state.muted; + + const peer = this.platform.createPeerConnection(); + session.peer = peer; + peer.addEventListener( + "track", + (event: { readonly streams?: ReadonlyArray }) => { + const [remote] = event.streams ?? []; + if (!remote || this.session !== session) return; + session.stopRemoteAudio?.(); + session.stopRemoteAudio = this.platform.playRemoteAudio(remote); + }, + ); + peer.addEventListener("connectionstatechange", () => { + if (this.session !== session) return; + if (peer.connectionState === "connected") { + session.peerConnected = true; + this.maybeActivate(session); + } else if (peer.connectionState === "failed") { + this.fail(session, "Voice connection was lost."); + } + }); + for (const track of microphone.getAudioTracks()) { + peer.addTrack(track as never, microphone as never); + } + // Codex's realtime transport requires the ordered events channel in the offer. + peer.createDataChannel("oai-events", { ordered: true }); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer as never); + if (this.session !== session) return; + if (!offer.sdp) throw new Error("Could not create a voice connection offer."); + + session.cancelSession = this.platform.openSession( + { ...target, offerSdp: offer.sdp }, + { + onEvent: (event) => this.handleEvent(session, event), + onError: (message) => this.fail(session, message), + onEnd: () => this.end(session, null), + }, + ); + } catch (cause) { + this.fail(session, describeError(cause)); + } + } + + stop(): void { + const session = this.session; + if (!session) return; + this.teardown(session); + this.setState({ ...INITIAL_VOICE_MODE_STATE, muted: this.state.muted }); + } + + setMuted(muted: boolean): void { + for (const track of this.session?.microphone?.getAudioTracks() ?? []) track.enabled = !muted; + this.setState({ ...this.state, muted }); + } + + toggleMuted(): void { + this.setMuted(!this.state.muted); + } + + /** Clears a finished conversation's error or end notice. */ + dismiss(): void { + if (this.state.phase !== "idle") return; + this.setState({ ...this.state, error: null, endedReason: null }); + } + + private handleEvent(session: ActiveSession, event: ProviderVoiceSessionEvent): void { + if (this.session !== session) return; + switch (event.type) { + case "answer": + session.peer + ?.setRemoteDescription({ type: "answer", sdp: event.sdp } as never) + .catch((cause: unknown) => this.fail(session, describeError(cause))); + return; + case "started": + session.backendStarted = true; + this.maybeActivate(session); + return; + case "transcript.delta": + this.setState({ + ...this.state, + captions: appendVoiceCaption(this.state.captions, event.role, event.delta), + }); + return; + case "transcript.done": + this.setState({ + ...this.state, + captions: finishVoiceCaption(this.state.captions, event.role, event.text), + }); + return; + case "closed": + this.end(session, event.reason ?? null); + return; + } + } + + private maybeActivate(session: ActiveSession): void { + if (!session.backendStarted || !session.peerConnected || this.state.phase === "active") return; + if (session.connectTimer) clearTimeout(session.connectTimer); + session.connectTimer = null; + session.levelTimer = setInterval(() => this.sampleLevels(session), LEVEL_SAMPLE_INTERVAL_MS); + this.setState({ ...this.state, phase: "active" }); + } + + private sampleLevels(session: ActiveSession): void { + // Native getStats serializes the whole report, so never stack calls. + if (this.levelListeners.size === 0 || !session.peer || session.sampling) return; + session.sampling = true; + session.peer + .getStats() + .finally(() => { + session.sampling = false; + }) + .then((report) => { + if (this.session !== session) return; + let microphone = 0; + let speaker = 0; + report.forEach((stat) => { + const level = (stat as { readonly audioLevel?: unknown }).audioLevel; + if (typeof level !== "number" || stat.kind !== "audio") return; + if (stat.type === "media-source") microphone = Math.max(microphone, level); + else if (stat.type === "inbound-rtp") speaker = Math.max(speaker, level); + }); + const levels = { microphone: this.state.muted ? 0 : microphone, speaker }; + for (const listener of this.levelListeners) listener(levels); + }) + .catch(() => undefined); + } + + private end(session: ActiveSession, reason: string | null): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + endedReason: reason ?? "ended", + }); + } + + private fail(session: ActiveSession, message: string): void { + if (this.session !== session) return; + this.teardown(session); + this.setState({ + ...INITIAL_VOICE_MODE_STATE, + target: session.target, + muted: this.state.muted, + error: message, + }); + } + + private teardown(session: ActiveSession): void { + if (this.session === session) this.session = null; + if (session.connectTimer) clearTimeout(session.connectTimer); + if (session.levelTimer) clearInterval(session.levelTimer); + session.cancelSession?.(); + session.stopRemoteAudio?.(); + session.peer?.close(); + for (const track of session.microphone?.getTracks() ?? []) track.stop(); + session.microphone?.release?.(); + for (const listener of this.levelListeners) listener({ microphone: 0, speaker: 0 }); + } + + private setState(next: VoiceModeState): void { + this.state = next; + // Read the latest state per listener: an earlier listener may have changed it. + for (const listener of [...this.listeners]) listener(this.state); + } +} diff --git a/packages/client-runtime/src/voice-mode/index.ts b/packages/client-runtime/src/voice-mode/index.ts new file mode 100644 index 000000000000..6d05d8bf22dc --- /dev/null +++ b/packages/client-runtime/src/voice-mode/index.ts @@ -0,0 +1,16 @@ +export { + INITIAL_VOICE_MODE_STATE, + VoiceModeController, + describeVoiceEnd, + type VoiceCaption, + type VoiceLevels, + type VoiceMediaStream, + type VoiceMediaTrack, + type VoiceModePhase, + type VoiceModePlatform, + type VoiceModeState, + type VoicePeerConnection, + type VoiceSessionHandlers, + type VoiceSessionTarget, +} from "./controller.ts"; +export { makeVoiceSessionOpener } from "./session.ts"; diff --git a/packages/client-runtime/src/voice-mode/session.ts b/packages/client-runtime/src/voice-mode/session.ts new file mode 100644 index 000000000000..4c316369d518 --- /dev/null +++ b/packages/client-runtime/src/voice-mode/session.ts @@ -0,0 +1,54 @@ +import { EnvironmentId, ThreadId, WS_METHODS } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import type * as Fiber from "effect/Fiber"; +import * as Stream from "effect/Stream"; + +import { EnvironmentRegistry } from "../connection/registry.ts"; +import { runStream } from "../rpc/client.ts"; +import type { VoiceModePlatform } from "./controller.ts"; + +function failureMessage(cause: Cause.Cause): string { + const error = Cause.squash(cause); + if (error && typeof error === "object") { + if ("detail" in error && typeof error.detail === "string") return error.detail; + if ("message" in error && typeof error.message === "string" && error.message) { + return error.message; + } + } + return "Voice conversation failed."; +} + +/** + * Builds `VoiceModePlatform.openSession` on an app runtime that provides the + * environment registry. Interrupting the returned fiber ends the conversation + * server-side. + */ +export function makeVoiceSessionOpener( + runFork: (effect: Effect.Effect) => Fiber.Fiber, +): VoiceModePlatform["openSession"] { + return ({ environmentId, threadId, offerSdp }, handlers) => { + const fiber = runFork( + EnvironmentRegistry.pipe( + Effect.flatMap((registry) => + registry + .runStream( + EnvironmentId.make(environmentId), + runStream(WS_METHODS.providerVoiceSession, { + threadId: ThreadId.make(threadId), + offerSdp, + }), + ) + .pipe(Stream.runForEach((event) => Effect.sync(() => handlers.onEvent(event)))), + ), + Effect.matchCause({ + onFailure: (cause) => { + if (!Cause.hasInterruptsOnly(cause)) handlers.onError(failureMessage(cause)); + }, + onSuccess: () => handlers.onEnd(), + }), + ), + ); + return () => fiber.interruptUnsafe(); + }; +} diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts index 04c3acab803d..6b57540fec72 100644 --- a/packages/contracts/src/keybindings.ts +++ b/packages/contracts/src/keybindings.ts @@ -97,6 +97,8 @@ export const STATIC_KEYBINDING_COMMANDS = [ "composer.workspace", "composer.previousWorktree", "composer.branch", + "voice.toggle", + "voice.toggleMute", "chat.new", "chat.newLocal", "chat.newWithoutProject", diff --git a/packages/contracts/src/provider.ts b/packages/contracts/src/provider.ts index f48ca86fbf08..7bf66ef4e807 100644 --- a/packages/contracts/src/provider.ts +++ b/packages/contracts/src/provider.ts @@ -142,6 +142,58 @@ export class ProviderUploadFeedbackError extends Schema.TaggedError()( + "ProviderVoiceSessionError", + { + threadId: ThreadId, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.detail; + } +} + const ProviderEventKind = Schema.Literals(["session", "notification", "request", "error"]); export const ProviderEvent = Schema.Struct({ diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 61304c52f860..4f7f6f3b57dd 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -159,6 +159,9 @@ import { ProviderUploadFeedbackError, ProviderUploadFeedbackInput, ProviderUploadFeedbackResult, + ProviderVoiceSessionError, + ProviderVoiceSessionEvent, + ProviderVoiceSessionInput, } from "./provider.ts"; import { ProviderInstanceId, ProviderInstanceMutation } from "./providerInstance.ts"; import { PullRequestReviewGuideResult } from "./pullRequestReviewGuide.ts"; @@ -404,6 +407,7 @@ export const WS_METHODS = { // Provider methods providerUploadFeedback: "provider.uploadFeedback", + providerVoiceSession: "provider.voice.session", providerAuthStart: "provider.auth.start", providerConsumeResetCredit: "provider.consumeResetCredit", providerAuthComplete: "provider.auth.complete", @@ -1345,6 +1349,13 @@ const WsProviderUploadFeedbackRpc = Rpc.make(WS_METHODS.providerUploadFeedback, error: Schema.Union([ProviderUploadFeedbackError, EnvironmentAuthorizationError]), }); +const WsProviderVoiceSessionRpc = Rpc.make(WS_METHODS.providerVoiceSession, { + payload: ProviderVoiceSessionInput, + success: ProviderVoiceSessionEvent, + error: Schema.Union([ProviderVoiceSessionError, EnvironmentAuthorizationError]), + stream: true, +}); + const WsSubscribeVcsStatusRpc = Rpc.make(WS_METHODS.subscribeVcsStatus, { payload: VcsStatusSubscriptionInput, success: VcsStatusStreamEvent, @@ -2002,6 +2013,7 @@ export const WsRpcGroup = RpcGroup.make( WsMcpAppsReadResourceRpc, WsMcpAppsUpdateModelContextRpc, WsProviderUploadFeedbackRpc, + WsProviderVoiceSessionRpc, WsSubscribeVcsStatusRpc, WsSubscribeWorktreeSetupRpc, WsWorktreeSetupCancelRpc, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 985e6bbc126e..9ae5a404e33a 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -265,6 +265,8 @@ export const ServerProvider = Schema.Struct({ requiresNewThreadForModelChange: Schema.optional(Schema.Boolean), supportsConversationRollback: Schema.optional(Schema.Boolean), supportsTextGeneration: Schema.optional(Schema.Boolean), + // The driver can hold realtime voice conversations (`provider.voice.session`). + supportsVoice: Schema.optional(Schema.Boolean), setup: Schema.optional( Schema.Struct({ canAuthenticate: Schema.Boolean, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index c0de88f4ac6e..5d69bdb36359 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -609,6 +609,21 @@ export function makeProviderSettingsSchema; + export const CodexSettings = makeProviderSettingsSchema( { setupMode: Schema.optionalKey(Schema.Literals(["managed", "existing"])).pipe( @@ -659,9 +674,21 @@ export const CodexSettings = makeProviderSettingsSchema( Schema.withDecodingDefault(Effect.succeed([])), Schema.annotateKey({ providerSettingsForm: { hidden: true } }), ), + voice: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Voice", + description: "Voice used by voice conversations. Applies to the next conversation.", + providerSettingsForm: { + control: "select", + options: CODEX_VOICE_OPTIONS, + clearWhenEmpty: "omit", + }, + }), + ), }, { - order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs"], + order: ["binaryPath", "homePath", "shadowHomePath", "launchArgs", "voice"], }, ); export type CodexSettings = typeof CodexSettings.Type; diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 809dbfac0110..5dea781c1fa5 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -1,4 +1,7 @@ -import type { OrchestrationV2HistoricalMessage } from "@t3tools/contracts"; +import type { + OrchestrationV2HistoricalMessage, + ProviderVoiceSessionEvent, +} from "@t3tools/contracts"; import { ChatAttachment, CheckpointId, @@ -635,6 +638,14 @@ export interface ProviderAdapterV2SessionRuntime { * provider are not interactive and their tool calls stay plain tool rows. */ readonly mcpApps?: ProviderAdapterV2McpApps; + /** + * Providers with realtime voice (Codex) hold a voice conversation on the + * thread. Interrupting the stream ends the conversation. + */ + readonly startVoiceSession?: (input: { + readonly providerThread: OrchestrationV2ProviderThread; + readonly offerSdp: string; + }) => Stream.Stream; readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index 5d08f22c13fb..575aee173997 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -47,6 +47,11 @@ export interface ProviderContinuationRequest { * wakes a run was dispatched for. Otherwise the service allocates one. */ readonly messageId?: MessageId; + /** + * `voice` resubmits a request the user spoke in a voice conversation, so the + * run reads as theirs: user-authored, with no background notification. + */ + readonly origin?: "voice"; readonly dispatchIfCurrent?: ( effect: Effect.Effect, ) => Effect.Effect, E, R>; diff --git a/packages/provider-core/src/server/snapshotProbe.ts b/packages/provider-core/src/server/snapshotProbe.ts index 071d9fd5ae07..45c715ac028d 100644 --- a/packages/provider-core/src/server/snapshotProbe.ts +++ b/packages/provider-core/src/server/snapshotProbe.ts @@ -92,6 +92,7 @@ export interface ServerProviderPresentation { readonly supportedRuntimeModes?: ReadonlyArray; readonly requiresNewThreadForModelChange?: boolean; readonly supportsConversationRollback?: boolean; + readonly supportsVoice?: boolean; } export type ServerProviderDraft = Omit; @@ -258,6 +259,7 @@ export function buildServerProvider(input: { ...(typeof input.presentation.requiresNewThreadForModelChange === "boolean" ? { requiresNewThreadForModelChange: input.presentation.requiresNewThreadForModelChange } : {}), + ...(input.presentation.supportsVoice ? { supportsVoice: true } : {}), enabled: input.enabled, installed: input.probe.installed, version: input.probe.version, diff --git a/packages/shared/src/composerTrigger.ts b/packages/shared/src/composerTrigger.ts index 6153978edb92..4819452a587d 100644 --- a/packages/shared/src/composerTrigger.ts +++ b/packages/shared/src/composerTrigger.ts @@ -4,7 +4,7 @@ export type ComposerTriggerKind = | "slash-command" | "slash-model" | "skill"; -export type ComposerSlashCommand = "model" | "plan" | "default"; +export type ComposerSlashCommand = "model" | "plan" | "default" | "voice"; export interface ComposerTrigger { kind: ComposerTriggerKind; diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts index 53a2aa5d3efe..7851ce03b56d 100644 --- a/packages/shared/src/keybindings.ts +++ b/packages/shared/src/keybindings.ts @@ -181,6 +181,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray = [ { key: "mod+shift+g", command: "composer.branch", when: "!terminalFocus" }, { key: "mod+shift+l", command: "composer.previousWorktree", when: "!terminalFocus" }, { key: "mod+shift+k", command: "pullRequest.copyNumber", when: "!terminalFocus" }, + { key: "f8", command: "voice.toggle", when: "!terminalFocus" }, { key: "mod+shift+arrowup", command: "modelPicker.previousProvider", when: "modelPickerOpen" }, { key: "mod+shift+arrowdown", command: "modelPicker.nextProvider", when: "modelPickerOpen" }, { key: "mod+o", command: "editor.openFavorite" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fdb699e66b36..5b82a5db8328 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -556,6 +556,9 @@ importers: react-native-svg: specifier: 15.15.5 version: 15.15.5(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0) + react-native-webrtc: + specifier: 124.0.8 + version: 124.0.8(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)) react-native-webview: specifier: 14.0.1 version: 14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0) @@ -581,6 +584,9 @@ importers: specifier: 1.12.1 version: 1.12.1(@expo/metro-config@58.0.6(patch_hash=90fa980e8b6480631fb2fbb04559834e3c612db40c550cd828f11be297d8d760)(bufferutil@4.1.0)(expo@58.0.2)(typescript@7.0.2)(utf-8-validate@6.0.6))(metro-cache@0.87.1)(metro-transform-worker@0.87.1(bufferutil@4.1.0)(utf-8-validate@6.0.6))(metro@0.87.1(bufferutil@4.1.0)(utf-8-validate@6.0.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0)(tailwindcss@4.3.3) devDependencies: + '@config-plugins/react-native-webrtc': + specifier: 15.0.2 + version: 15.0.2(expo@58.0.2) '@effect/vitest': specifier: 4.0.2 version: 4.0.2(patch_hash=595b7d26fb635caba9c3d39fcb189bb78a000192f8d3b60ead7174b310d1ade9)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.2(patch_hash=ec61ee307eaec3e17f757e0e6a16296ea4b7a8c3188df0c6ec5e66301fd78688))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) @@ -2829,6 +2835,11 @@ packages: '@coderabbitai/config@1.1.0': resolution: {integrity: sha512-NvQKaoOD7fWb7lMKhid57ABVlIBSczw/S+ePk5YvoPuWdrDkyY+PWdqVtq3TRq7VY8f3SNRlMFjmAtXVab66pw==} + '@config-plugins/react-native-webrtc@15.0.2': + resolution: {integrity: sha512-sH4T7Z4P2RowV91k9CEwEr0unn+396NBexxCNZwRXuJXwguDU1qZWpE6fcwN0730B8uiS83F7+CLTyhpC7qRHQ==} + peerDependencies: + expo: '>=56' + '@connectrpc/connect-web@1.7.0': resolution: {integrity: sha512-qyP0YOnUPRWwCc/VfsoydMJvkb7EyUPr2q9sHgBuJzbADjiqck1gKH5V5ZPzPhTLBvmz5UvG+wiZ5sMRQHU1MQ==} peerDependencies: @@ -8030,6 +8041,15 @@ packages: supports-color: optional: true + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -10141,6 +10161,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -10915,6 +10938,11 @@ packages: peerDependencies: react-native: '*' + react-native-webrtc@124.0.8: + resolution: {integrity: sha512-uuQxvmk+mvnk5U0tr+1N42sKZqgm41fJrBA+fmCvML9J9P4roSh2So82t5RHAlu/vE9vxu5AKgivAiH61clCBg==} + peerDependencies: + react-native: '>=0.60.0' + react-native-webview@14.0.1: resolution: {integrity: sha512-lBUuvFk459Mo+5vUNqwzGvn+I8rDJE6zvOulG2hWWudeEOJfWTsQwOfuYnRu+27djUIXrESFfC+EQE3PGu+FWA==} peerDependencies: @@ -13641,6 +13669,10 @@ snapshots: dependencies: zod: 4.5.4 + '@config-plugins/react-native-webrtc@15.0.2(expo@58.0.2)': + dependencies: + expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) + '@connectrpc/connect-web@1.7.0(@bufbuild/protobuf@1.10.0)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.0))': dependencies: '@bufbuild/protobuf': 1.10.0 @@ -18674,6 +18706,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.3.4: + dependencies: + ms: 2.1.2 + debug@4.4.3: dependencies: ms: 2.1.3 @@ -21161,6 +21197,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2): @@ -22130,6 +22168,14 @@ snapshots: dependencies: react-native: 0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6) + react-native-webrtc@124.0.8(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)): + dependencies: + base64-js: 1.5.1 + debug: 4.3.4 + react-native: 0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6) + transitivePeerDependencies: + - supports-color + react-native-webview@14.0.1(patch_hash=81caa279ec3dc1f81209062133b66e8f824e13f106cca57f1fa732a72c2ab378)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0): dependencies: escape-string-regexp: 4.0.0 diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 2d86a9c89d5b..ba4e43f5d082 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -1972,6 +1972,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { assert.include(entitlements, "webcredentials:clerk.example.com"); assert.include(entitlements, "webcredentials:example.clerk.accounts.dev"); assert.include(entitlements, "com.apple.security.cs.allow-jit"); + assert.include(entitlements, "com.apple.security.device.audio-input"); assert.notInclude(entitlements, "keychain-access-groups"); assert.notInclude(entitlements, "com.apple.developer.web-browser.public-key-credential"); }); diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 618aec4d6bba..bd16f2e821aa 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1413,6 +1413,8 @@ ${associatedDomains} com.apple.security.cs.disable-library-validation + com.apple.security.device.audio-input + `; @@ -2835,6 +2837,8 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( LSItemContentTypes: ["public.html", "public.xhtml"], }, ], + NSMicrophoneUsageDescription: + "T3 Code uses the microphone when you start a voice conversation with an agent.", }, protocols: [ { diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index f247cf5c17c8..b8a033368cbf 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -121,6 +121,21 @@ } ] }, + { + "bundles": ["android", "ios", "mobile"], + "license": "BSD-3-Clause", + "name": "WebRTC", + "sourceUrl": "https://webrtc.googlesource.com/src", + "generatedNotices": [ + { + "licenseId": "BSD-3-Clause", + "copyrights": ["Copyright (c) 2011, The WebRTC project authors. All rights reserved."], + "preamble": [ + "WebRTC\n\nVoice conversations use the libwebrtc build that react-native-webrtc links into the iOS and Android apps." + ] + } + ] + }, { "bundles": ["android", "assets", "mobile"], "license": "Apache-2.0",