feat: Codex voice conversations on web, desktop, and mobile

apps/desktop/src/window/DesktopWindow.test.ts

@@ -94,6 +94,10 @@ function makeFakeBrowserWindow() {
     reloadIgnoringCache: vi.fn(),
     replaceMisspelling: vi.fn(),
     send: vi.fn(),
+    session: {
+      setPermissionCheckHandler: vi.fn(),
+      setPermissionRequestHandler: vi.fn(),
+    },
     setBackgroundThrottling: vi.fn(),
     setWindowOpenHandler: vi.fn(),
     toggleDevTools: vi.fn(),

apps/desktop/src/window/DesktopWindow.ts

@@ -32,6 +32,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts";
 import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts";
 import * as ElectronApp from "../electron/ElectronApp.ts";
 import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts";
+import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts";
 import { makeQuitShortcutHandler } from "./QuitHold.ts";
 
 const TITLEBAR_HEIGHT = 40;
@@ -439,6 +440,10 @@ export const make = Effect.gen(function* () {
     if (environment.platform === "darwin") {
       window.setAutoHideCursor(false);
     }
+    installMainWindowPermissionHandlers(window.webContents.session, {
+      appUrl: applicationUrl,
+      platform: environment.platform,
+    });
     let boundsPersistFiber: Fiber.Fiber<void, never> | undefined;
     let pendingBoundsPersistFiber: Fiber.Fiber<void, never> | undefined;
     let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds;

apps/desktop/src/window/MainWindowPermissions.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { decideMainWindowMediaRequest, isAppUrl } from "./MainWindowPermissions.ts";
+
+const APP_URL = "t3code://app/";
+
+function decide(overrides: Partial<Parameters<typeof decideMainWindowMediaRequest>[0]> = {}) {
+  return decideMainWindowMediaRequest({
+    requestingUrl: "t3code://app/threads/abc",
+    appUrl: APP_URL,
+    mediaTypes: ["audio"],
+    platform: "darwin",
+    microphoneStatus: () => "granted",
+    ...overrides,
+  });
+}
+
+describe("isAppUrl", () => {
+  it("matches the app shell and nothing that merely shares its prefix", () => {
+    expect(isAppUrl("t3code://app", APP_URL)).toBe(true);
+    expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true);
+    expect(isAppUrl("t3code://application/", APP_URL)).toBe(false);
+    expect(isAppUrl("https://example.com/", APP_URL)).toBe(false);
+    expect(isAppUrl(undefined, APP_URL)).toBe(false);
+  });
+});
+
+describe("decideMainWindowMediaRequest", () => {
+  it("denies media to frames outside the app shell", () => {
+    expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny");
+  });
+
+  it("asks macOS for the microphone until the user has granted it", () => {
+    expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone");
+    expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone");
+    expect(decide()).toBe("grant");
+  });
+
+  it("grants the microphone directly off macOS", () => {
+    expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant");
+  });
+
+  it("keeps granting video-only capture such as preview tab recording", () => {
+    expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant");
+  });
+});

apps/desktop/src/window/MainWindowPermissions.ts

@@ -0,0 +1,63 @@
+import * as Electron from "electron";
+
+export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone";
+
+/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */
+export function isAppUrl(url: string | undefined, appUrl: string): boolean {
+  if (!url) return false;
+  // Custom schemes have an opaque WHATWG origin, so compare the serialized prefix.
+  const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl;
+  return url === appOrigin || url.startsWith(`${appOrigin}/`);
+}
+
+/**
+ * Media requests come from the app shell (voice conversations use the
+ * microphone; preview tab recording routes `getDisplayMedia` through the same
+ * permission). Frames from any other origin never get media. On macOS the
+ * microphone also needs the system's consent, asked for on first use.
+ */
+export function decideMainWindowMediaRequest(input: {
+  readonly requestingUrl: string;
+  readonly appUrl: string;
+  readonly mediaTypes: ReadonlyArray<"audio" | "video">;
+  readonly platform: NodeJS.Platform;
+  readonly microphoneStatus: () => string;
+}): MediaPermissionDecision {
+  if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny";
+  if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant";
+  return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone";
+}
+
+/**
+ * Installs the main window session's permission handlers. Only `media` is
+ * decided here; every other permission keeps Electron's default of allowing
+ * it, which is what this session did before it had handlers.
+ */
+export function installMainWindowPermissionHandlers(
+  session: Electron.Session,
+  options: { readonly appUrl: string; readonly platform: NodeJS.Platform },
+): void {
+  session.setPermissionRequestHandler((_webContents, permission, callback, details) => {
+    if (permission !== "media") {
+      callback(true);
+      return;
+    }
+    const decision = decideMainWindowMediaRequest({
+      requestingUrl: details.requestingUrl,
+      appUrl: options.appUrl,
+      mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [],
+      platform: options.platform,
+      microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"),
+    });
+    if (decision !== "ask-microphone") {
+      callback(decision === "grant");
+      return;
+    }
+    Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => {
+      callback(false);
+    });
+  });
+  session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) =>
+    permission === "media" ? isAppUrl(requestingOrigin, options.appUrl) : true,
+  );
+}

apps/mobile/app.config.ts

@@ -220,6 +220,10 @@ const sharingPlugin: NonNullable<ExpoConfig["plugins"]>[number] = [
   },
 ];
 
+const MICROPHONE_PERMISSION =
+  "Allow T3 Code to use your microphone for voice input and voice conversations.";
+const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes.";
+
 // These aliases match the fonts' PostScript names on iOS. Register the same
 // names on Android so React Native and the native composer use one set of
 // family names without waiting for runtime font loading.
@@ -309,8 +313,11 @@ const config: ExpoConfig = {
     // JS back handling survives it via react-native's Android 16 shim plus
     // withAndroidPredictiveBackCompat on Android 13-15.
     predictiveBackGestureEnabled: true,
-    // expo-sensors declares this for its pedometer, which the app does not use.
-    blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"],
+    // The app uses neither expo-sensors' pedometer nor WebRTC screen sharing.
+    blockedPermissions: [
+      "android.permission.ACTIVITY_RECOGNITION",
+      "android.permission.SYSTEM_ALERT_WINDOW",
+    ],
   },
   web: {
     favicon: variant.assets.appIcon,
@@ -377,22 +384,30 @@ const config: ExpoConfig = {
     [
       "expo-audio",
       {
-        microphonePermission: "Allow T3 Code to use your microphone for voice input.",
-        recordAudioAndroid: false,
+        microphonePermission: MICROPHONE_PERMISSION,
+        recordAudioAndroid: true,
         enableBackgroundPlayback: false,
         enableBackgroundRecording: false,
       },
     ],
+    // Voice conversations run over WebRTC. Pass both usage strings so its
+    // plugin cannot fall back to generic ones.
+    [
+      "@config-plugins/react-native-webrtc",
+      { cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION },
+    ],
     [
       "expo-camera",
       {
-        cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.",
+        cameraPermission: CAMERA_PERMISSION,
         microphonePermission: false,
         barcodeScannerEnabled: true,
         recordAudioAndroid: false,
       },
     ],
-    ["expo-image-picker", { photosPermission: false, microphonePermission: false }],
+    // `microphonePermission: false` would block RECORD_AUDIO on Android for every
+    // package, including voice conversations, so share the app's usage string.
+    ["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }],
     [
       "expo-splash-screen",
       {

apps/mobile/package.json

@@ -130,13 +130,15 @@
     "react-native-screens": "file:deps/react-native-screens-5.0.0-t3.7.tgz",
     "react-native-shiki-engine": "^0.3.12",
     "react-native-svg": "15.15.5",
+    "react-native-webrtc": "124.0.8",
     "react-native-webview": "14.0.1",
     "react-native-worklets": "0.13.0",
     "shiki": "4.2.0",
     "tailwind-merge": "^3.5.0",
     "uniwind": "1.12.1"
   },
   "devDependencies": {
+    "@config-plugins/react-native-webrtc": "15.0.2",
     "@effect/vitest": "catalog:",
     "@pierre/trees": "1.0.0-beta.6",
     "@types/react": "~19.3.0",

apps/mobile/src/components/AppSymbol.tsx

@@ -73,6 +73,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar";
 import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight";
 import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing";
 import IconMicrophone from "@tabler/icons-react-native/IconMicrophone";
+import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff";
 import IconLink from "@tabler/icons-react-native/IconLink";
 import IconListNumbers from "@tabler/icons-react-native/IconListNumbers";
 import IconMenu2 from "@tabler/icons-react-native/IconMenu2";
@@ -112,6 +113,7 @@ import IconUpload from "@tabler/icons-react-native/IconUpload";
 import IconUserCircle from "@tabler/icons-react-native/IconUserCircle";
 import IconUsers from "@tabler/icons-react-native/IconUsers";
 import IconWifi from "@tabler/icons-react-native/IconWifi";
+import IconWaveSine from "@tabler/icons-react-native/IconWaveSine";
 import IconWifiOff from "@tabler/icons-react-native/IconWifiOff";
 import IconWorld from "@tabler/icons-react-native/IconWorld";
 import IconX from "@tabler/icons-react-native/IconX";
@@ -194,6 +196,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
   macstudio: IconDeviceDesktop,
   magnifyingglass: IconSearch,
   mic: IconMicrophone,
+  "mic.slash": IconMicrophoneOff,
   paintbrush: IconPalette,
   pencil: IconPencil,
   "person.crop.circle": IconUserCircle,
@@ -234,6 +237,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
   "tray.and.arrow.up": IconUpload,
   trash: IconTrash,
   wifi: IconWifi,
+  waveform: IconWaveSine,
   "wifi.slash": IconWifiOff,
   xmark: IconX,
   "xmark.circle.fill": IconCircleXFilled,

apps/mobile/src/features/threads/ThreadComposer.tsx

@@ -115,6 +115,9 @@ import {
 } from "../voice-input/ComposerDictationControl";
 import { useVoiceInputController } from "../voice-input/useVoiceInputController";
 import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation";
+import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode";
+import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode";
+import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode";
 import {
   rememberModelOptions,
   withRememberedModelOptions,
@@ -509,15 +512,41 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
     onUsageLimits:
       usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined,
   });
+  const voiceTarget = useMemo(
+    () => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }),
+    [props.environmentId, props.selectedThread.id],
+  );
+  const voicePhase = useVoiceModePhase(voiceTarget);
+  useVoiceModeThreadLifecycle(voiceTarget);
+  // Dictation and a voice conversation both need the microphone.
+  const voiceConversationLive = voicePhase !== "idle";
   const voiceInput = useVoiceInputController({
     ownerKey: composerDraftKey,
     label: props.selectedThread.title || "Untitled thread",
     readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text,
     subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
     selection: composerMenu.selection,
+    disabled: voiceConversationLive,
     onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text),
     onChangeSelection: composerMenu.onSelectionChange,
   });
+  const offersVoiceConversation =
+    voiceModeSupported && selectedProviderStatus?.supportsVoice === true;
+  // The server attaches voice to the thread's provider thread, which the first
+  // run creates. An optimistic pending thread already has a message timestamp.
+  const canToggleVoiceConversation =
+    voiceConversationLive ||
+    (props.connectionState === "connected" &&
+      props.selectedThread.activeProviderThreadId !== null &&
+      !voiceInput.isBusy);
+  const toggleVoiceConversation = useCallback(() => voiceMode.toggle(voiceTarget), [voiceTarget]);
+  const voiceModeButton = offersVoiceConversation ? (
+    <ComposerVoiceModeButton
+      phase={voicePhase}
+      disabled={!canToggleVoiceConversation}
+      onPress={toggleVoiceConversation}
+    />
+  ) : null;
   const voicePresentation = resolveVoiceComposerPresentation(
     voiceInput.state,
     voiceInput.elapsedSeconds,
@@ -806,6 +835,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
             {selectedProviderStatus.compatibilityAdvisory.message}
           </Text>
         ) : null}
+
+        {voiceConversationLive ? <ComposerVoiceStrip /> : null}
+
         {modelUnavailable ? (
           <Pressable accessibilityRole="button" className="px-3 py-2" onPress={openSettings}>
             <Text className="text-xs text-foreground">Model unavailable. Open model settings.</Text>
@@ -1048,9 +1080,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
             ) : null}
             {!isExpanded ? (
               <View className="flex-row items-center">
+                {voiceModeButton}
                 <ComposerDictationStartAction
                   state={voiceInput.state}
                   isAvailable={voiceInput.isAvailable}
+                  disabled={voiceConversationLive}
                   onStart={voiceInput.start}
                   onCancel={voiceInput.cancel}
                 />
@@ -1141,10 +1175,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
                   </View>
                 )}
                 <View className="shrink-0 flex-row items-center">
+                  {isVoiceInputPresented ? null : voiceModeButton}
                   <ComposerDictationPrimaryAction
                     state={voiceInput.state}
                     presentation={voicePresentation}
                     isAvailable={voiceInput.isAvailable}
+                    disabled={voiceConversationLive}
                     onStart={voiceInput.start}
                     onConfirm={voiceInput.stop}
                     onCancel={voiceInput.cancel}

apps/mobile/src/features/threads/ThreadDetailScreen.tsx

@@ -145,6 +145,8 @@ import { useThreadQueuedCount } from "./ThreadQueueControl";
 import type { ThreadContentPresentation } from "./threadContentPresentation";
 import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow";
 import { useGlobalVoiceInput } from "../voice-input/VoiceInputProvider";
+import { ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode";
+import { useVoiceModePhase } from "../voice-mode/useVoiceMode";
 
 export interface ThreadDetailScreenProps {
   readonly worktreeSetup?: WorktreeSetupCardProps | null;
@@ -563,6 +565,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
   // A pending user-input request or a failed creation owns the composer slot.
   const composerSlotHidden =
     activeUserInputRequestId !== null || props.creationState?.kind === "failed";
+  const voiceTarget = useMemo(
+    () => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }),
+    [props.environmentId, props.selectedThread.id],
+  );
+  const voiceLiveHere = useVoiceModePhase(voiceTarget) !== "idle";
   // The open /usage-limits panel for this thread, model and turn. Only the open
   // moment is stored: the rows read live provider data, so a redeemed reset
   // credit or refreshed probe shows through. Anything that spends quota closes
@@ -1333,7 +1340,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
                   ) : null}
                 </View>
 
-                {/* Hidden (not unmounted) while a user-input request owns the
+              {/* The composer below hides for a question card; keep a live
+                voice conversation's mute and stop controls reachable. */}
+              {activeUserInputRequestId !== null && voiceLiveHere ? <ComposerVoiceStrip /> : null}
+              {/* Hidden (not unmounted) while a user-input request owns the
                 composer slot, so composer drafts and editor state survive.
                 A rejected creation has no thread to send to; the failure card
                 owns the slot instead. */}