diff --git a/Sources/App/VoicePipeline+Recording.swift b/Sources/App/VoicePipeline+Recording.swift index a3637c3..1d52a9c 100644 --- a/Sources/App/VoicePipeline+Recording.swift +++ b/Sources/App/VoicePipeline+Recording.swift @@ -126,8 +126,23 @@ extension VoicePipeline { } activeCapture.thresholds = snapshot.audioActivityThresholds + activeCapture.onAutoSwitch = { [weak self] in + Task { @MainActor in + guard let self, self.sessionLease == lease, self.appState.isRecording else { return } + self.appState.statusMessage = L("pipeline.mic_switched") + } + } + activeCapture.onInputUnavailable = { [weak self] in + Task { @MainActor in + guard let self, self.sessionLease == lease, self.appState.isRecording else { return } + self.appState.phase = .error(L("pipeline.mic_clamshell_no_input")) + self.appState.statusMessage = L("pipeline.mic_clamshell_no_input") + self.overlay.hide() + self.cancel() + } + } - let micStarted = activeCapture.start( + let micFailure = activeCapture.start( deviceID: microphoneID, levelUpdate: { [weak self] level in Task { @MainActor in @@ -140,12 +155,18 @@ extension VoicePipeline { self?.currentEngine?.appendAudioBuffer(buffer) } ) - guard micStarted else { + if let micFailure { currentEngine?.cancelListening() cancelScreenContextCapture() recordingTargetApp = nil - appState.phase = .error(L("pipeline.mic_failed_permissions")) - appState.statusMessage = L("pipeline.mic_unavailable") + switch micFailure { + case .noUsableInput: + appState.phase = .error(L("pipeline.mic_clamshell_no_input")) + appState.statusMessage = L("pipeline.mic_clamshell_no_input") + case .permissionDenied, .engineFailed: + appState.phase = .error(L("pipeline.mic_failed_permissions")) + appState.statusMessage = L("pipeline.mic_unavailable") + } overlay.hide() return } diff --git a/Sources/Audio/AudioCaptureManager.swift b/Sources/Audio/AudioCaptureManager.swift index d172d0a..a8a000c 100644 --- a/Sources/Audio/AudioCaptureManager.swift +++ b/Sources/Audio/AudioCaptureManager.swift @@ -41,6 +41,14 @@ struct AudioCaptureActivity: Equatable { } } +/// Why a local capture could not start. Remote wiring maps these to localized +/// messages; the pipeline distinguishes "no usable input" from permission loss. +enum AudioCaptureStartFailure: Error, Equatable { + case permissionDenied + case noUsableInput + case engineFailed +} + final class AudioCaptureManager { private let engine = AVAudioEngine() private var audioFile: AVAudioFile? @@ -52,11 +60,23 @@ final class AudioCaptureManager { /// A voice-key session from the connected remote can supply audio instead of /// the selected CoreAudio device. Keyboard/API sessions remain local. var remoteMicSource: RemoteMicCaptureManager? + /// Called on the main run loop after capture switches to a fallback input + /// because the active device became unusable (for example, the lid closed). + var onAutoSwitch: (() -> Void)? + /// Called on the main run loop when the active input was lost and no other + /// usable input exists. + var onInputUnavailable: (() -> Void)? private var usesRemoteMic = false private var levelCallback: ((Float) -> Void)? private var bufferCallback: ((AVAudioPCMBuffer) -> Void)? private var isRunning = false + private var preferredInputUID: String? + private var activeInputUID: String? + private var failoverTimer: Timer? + private var recordingFormat: AVAudioFormat? + private var converter: AVAudioConverter? + private var converterSourceFormat: AVAudioFormat? var lastRecordingURL: URL? { usesRemoteMic ? remoteMicSource?.lastRecordingURL : localLastRecordingURL @@ -83,7 +103,7 @@ final class AudioCaptureManager { deviceID: String?, levelUpdate: @escaping (Float) -> Void, bufferUpdate: ((AVAudioPCMBuffer) -> Void)? = nil - ) -> Bool { + ) -> AudioCaptureStartFailure? { if isRunning { stop() } cleanupLastRecording() usesRemoteMic = false @@ -99,7 +119,7 @@ final class AudioCaptureManager { if remoteMicSource.start(token: token, levelUpdate: levelUpdate, bufferUpdate: bufferUpdate) { usesRemoteMic = true isRunning = true - return true + return nil } Log.info("[AudioCapture] wireless remote unavailable; using the system input") } @@ -107,10 +127,23 @@ final class AudioCaptureManager { let authStatus = AVCaptureDevice.authorizationStatus(for: .audio) guard authStatus == .authorized else { Log.error("[AudioCapture] microphone not authorized (status: \(authStatus.rawValue))") - return false + return .permissionDenied } - if let deviceID, let uid = findDevice(id: deviceID) { + let devices = AudioInputDevices.available() + let lidClosed = ClamshellState.isClosed + switch AudioInputResolver.resolve( + devices: devices, + preferredUID: deviceID, + systemDefaultUID: AudioInputDevices.systemDefaultUID(), + lidClosed: lidClosed + ) { + case .unavailable: + Log.error("[AudioCapture] no usable input (lid closed: \(lidClosed))") + return .noUsableInput + case .use(let uid): + preferredInputUID = deviceID + activeInputUID = uid setInputDevice(uid: uid) } @@ -118,7 +151,7 @@ final class AudioCaptureManager { let format = inputNode.outputFormat(forBus: 0) guard format.sampleRate > 0, format.channelCount > 0 else { Log.error("[AudioCapture] invalid input format: \(format)") - return false + return .engineFailed } let url = FileManager.default.temporaryDirectory @@ -126,51 +159,42 @@ final class AudioCaptureManager { localLastRecordingURL = url do { - audioFile = try AVAudioFile( + let file = try AVAudioFile( forWriting: url, settings: format.settings, commonFormat: format.commonFormat, interleaved: format.isInterleaved ) + audioFile = file + recordingFormat = file.processingFormat } catch { Log.error("[AudioCapture] cannot create audio file: \(error.localizedDescription)") - return false + return .engineFailed } - inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in - guard let self else { return } - try? self.audioFile?.write(from: buffer) - - let rms = Self.calculateRMS(buffer: buffer) - self.localLastActivity.record(rms: rms, frameCount: Int(buffer.frameLength)) - - let level = Self.visualLevel(fromRMS: rms) - self.levelCallback?(level) - - if let bufferCallback = self.bufferCallback { - if let copiedBuffer = buffer.copied() { - bufferCallback(copiedBuffer) - } else { - Log.error("[AudioCapture] unsupported format \(buffer.format.commonFormat.rawValue); dropping streaming buffer") - } - } + guard installCaptureTap() else { + audioFile = nil + return .engineFailed } engine.prepare() do { try engine.start() - isRunning = true - return true } catch { engine.inputNode.removeTap(onBus: 0) audioFile = nil Log.error("[AudioCapture] engine start failed: \(error.localizedDescription)") - return false + return .engineFailed } + + isRunning = true + startFailoverMonitor() + return nil } func stop() { guard isRunning else { return } + stopFailoverMonitor() if usesRemoteMic { remoteMicSource?.stop() levelCallback = nil @@ -181,11 +205,152 @@ final class AudioCaptureManager { engine.inputNode.removeTap(onBus: 0) engine.stop() audioFile = nil + recordingFormat = nil + converter = nil + converterSourceFormat = nil + preferredInputUID = nil + activeInputUID = nil levelCallback = nil bufferCallback = nil isRunning = false } + // MARK: - Capture tap and file writing + + private func installCaptureTap() -> Bool { + let inputNode = engine.inputNode + let format = inputNode.outputFormat(forBus: 0) + guard format.sampleRate > 0, format.channelCount > 0 else { + Log.error("[AudioCapture] invalid input format: \(format)") + return false + } + + inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in + guard let self else { return } + self.write(buffer) + + let rms = Self.calculateRMS(buffer: buffer) + self.localLastActivity.record(rms: rms, frameCount: Int(buffer.frameLength)) + self.levelCallback?(Self.visualLevel(fromRMS: rms)) + + if let bufferCallback = self.bufferCallback { + if let copiedBuffer = buffer.copied() { + bufferCallback(copiedBuffer) + } else { + Log.error("[AudioCapture] unsupported format \(buffer.format.commonFormat.rawValue); dropping streaming buffer") + } + } + } + return true + } + + /// Writes a buffer to the recording file, converting when a fallback device + /// produced a different sample rate or channel layout. + private func write(_ buffer: AVAudioPCMBuffer) { + guard let audioFile, let target = recordingFormat else { return } + if buffer.format == target { + try? audioFile.write(from: buffer) + return + } + guard let converted = convert(buffer, to: target) else { return } + try? audioFile.write(from: converted) + } + + private func convert(_ buffer: AVAudioPCMBuffer, to target: AVAudioFormat) -> AVAudioPCMBuffer? { + if converterSourceFormat != buffer.format { + converter = AVAudioConverter(from: buffer.format, to: target) + converterSourceFormat = buffer.format + } + guard let converter else { return nil } + + let ratio = target.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 1024 + guard let output = AVAudioPCMBuffer(pcmFormat: target, frameCapacity: capacity) else { + return nil + } + + var supplied = false + var error: NSError? + converter.convert(to: output, error: &error) { _, status in + if supplied { + status.pointee = .noDataNow + return nil + } + supplied = true + status.pointee = .haveData + return buffer + } + if let error { + Log.error("[AudioCapture] format conversion failed: \(error.localizedDescription)") + return nil + } + return output + } + + // MARK: - Mid-session failover + + private func startFailoverMonitor() { + stopFailoverMonitor() + let timer = Timer(timeInterval: 0.5, repeats: true) { [weak self] _ in + self?.evaluateInputHealth() + } + RunLoop.main.add(timer, forMode: .common) + failoverTimer = timer + } + + private func stopFailoverMonitor() { + failoverTimer?.invalidate() + failoverTimer = nil + } + + private func evaluateInputHealth() { + guard isRunning, !usesRemoteMic else { return } + let action = MicFailoverDecision.decide( + activeUID: activeInputUID, + devices: AudioInputDevices.available(), + preferredUID: preferredInputUID, + systemDefaultUID: AudioInputDevices.systemDefaultUID(), + lidClosed: ClamshellState.isClosed + ) + switch action { + case .keep: + break + case .switchTo(let uid): + switchInput(to: uid) + case .fail: + handleInputUnavailable() + } + } + + private func switchInput(to uid: String) { + let name = AudioInputDevices.available().first(where: { $0.uid == uid })?.name ?? uid + Log.info("[AudioCapture] switching to fallback input") + engine.inputNode.removeTap(onBus: 0) + engine.stop() + setInputDevice(uid: uid) + guard installCaptureTap() else { + handleInputUnavailable() + return + } + engine.prepare() + do { + try engine.start() + } catch { + Log.error("[AudioCapture] fallback engine start failed: \(error.localizedDescription)") + handleInputUnavailable() + return + } + activeInputUID = uid + Log.info("[AudioCapture] fallback input active: \(name)") + onAutoSwitch?() + } + + private func handleInputUnavailable() { + Log.error("[AudioCapture] active input lost with no fallback") + stop() + onInputUnavailable?() + } + private static func calculateRMS(buffer: AVAudioPCMBuffer) -> Float { let count = Int(buffer.frameLength) guard count > 0 else { return 0 } @@ -225,32 +390,11 @@ final class AudioCaptureManager { // MARK: - Device Management static func availableMicrophones() -> [(id: String, name: String)] { - var propertyAddress = AudioObjectPropertyAddress( - mSelector: kAudioHardwarePropertyDevices, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMain - ) - var dataSize: UInt32 = 0 - AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &propertyAddress, 0, nil, &dataSize) - - let count = Int(dataSize) / MemoryLayout.size - var deviceIDs = [AudioDeviceID](repeating: 0, count: count) - AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &propertyAddress, 0, nil, &dataSize, &deviceIDs) - - return deviceIDs.compactMap { deviceID -> (id: String, name: String)? in - guard hasInputChannels(deviceID: deviceID) else { return nil } - let name = deviceName(deviceID: deviceID) ?? "Unknown" - let uid = deviceUID(deviceID: deviceID) ?? "\(deviceID)" - return (id: uid, name: name) - } - } - - private func findDevice(id: String) -> String? { - AudioCaptureManager.availableMicrophones().first { $0.id == id }?.id + AudioInputDevices.available().map { (id: $0.uid, name: $0.name) } } private func setInputDevice(uid: String) { - guard let deviceID = Self.audioDeviceID(forUID: uid) else { return } + guard let deviceID = AudioInputDevices.deviceID(forUID: uid) else { return } guard let audioUnit = engine.inputNode.audioUnit else { return } var id = deviceID AudioUnitSetProperty( @@ -262,54 +406,4 @@ final class AudioCaptureManager { UInt32(MemoryLayout.size) ) } - - private static func audioDeviceID(forUID uid: String) -> AudioDeviceID? { - var address = AudioObjectPropertyAddress( - mSelector: kAudioHardwarePropertyDevices, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMain - ) - var dataSize: UInt32 = 0 - AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &dataSize) - let count = Int(dataSize) / MemoryLayout.size - var deviceIDs = [AudioDeviceID](repeating: 0, count: count) - AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &dataSize, &deviceIDs) - return deviceIDs.first { deviceUID(deviceID: $0) == uid } - } - - private static func hasInputChannels(deviceID: AudioDeviceID) -> Bool { - var address = AudioObjectPropertyAddress( - mSelector: kAudioDevicePropertyStreamConfiguration, - mScope: kAudioDevicePropertyScopeInput, - mElement: kAudioObjectPropertyElementMain - ) - var size: UInt32 = 0 - AudioObjectGetPropertyDataSize(deviceID, &address, 0, nil, &size) - let bufferListPointer = UnsafeMutablePointer.allocate(capacity: Int(size)) - defer { bufferListPointer.deallocate() } - AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, bufferListPointer) - let bufferList = UnsafeMutableAudioBufferListPointer(bufferListPointer) - return bufferList.reduce(0) { $0 + Int($1.mNumberChannels) } > 0 - } - - private static func deviceName(deviceID: AudioDeviceID) -> String? { - getStringProperty(deviceID: deviceID, selector: kAudioDevicePropertyDeviceNameCFString) - } - - private static func deviceUID(deviceID: AudioDeviceID) -> String? { - getStringProperty(deviceID: deviceID, selector: kAudioDevicePropertyDeviceUID) - } - - private static func getStringProperty(deviceID: AudioDeviceID, selector: AudioObjectPropertySelector) -> String? { - var address = AudioObjectPropertyAddress( - mSelector: selector, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMain - ) - var value: Unmanaged? - var size = UInt32(MemoryLayout?>.size) - let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &value) - guard status == noErr, let cf = value?.takeUnretainedValue() else { return nil } - return cf as String - } } diff --git a/Sources/Audio/AudioInputDevice.swift b/Sources/Audio/AudioInputDevice.swift new file mode 100644 index 0000000..ad101ee --- /dev/null +++ b/Sources/Audio/AudioInputDevice.swift @@ -0,0 +1,127 @@ +import CoreAudio +import Foundation + +/// One CoreAudio input device, classified by transport so callers can tell the +/// built-in microphone apart from external inputs (USB, Bluetooth, display, +/// aggregate, and Continuity/iPhone). +struct AudioInputDevice: Equatable, Sendable { + let uid: String + let name: String + let isBuiltIn: Bool +} + +enum AudioInputDevices { + static func available() -> [AudioInputDevice] { + deviceIDs().compactMap { deviceID in + guard hasInputChannels(deviceID: deviceID) else { return nil } + let name = deviceName(deviceID: deviceID) ?? "Unknown" + let uid = deviceUID(deviceID: deviceID) ?? "\(deviceID)" + return AudioInputDevice( + uid: uid, + name: name, + isBuiltIn: isBuiltIn(transportType: transportType(deviceID: deviceID)) + ) + } + } + + static func systemDefaultUID() -> String? { + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var deviceID = AudioDeviceID(0) + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData( + AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceID + ) + guard status == noErr, deviceID != 0 else { return nil } + return deviceUID(deviceID: deviceID) + } + + static func deviceID(forUID uid: String) -> AudioDeviceID? { + deviceIDs().first { deviceUID(deviceID: $0) == uid } + } + + static func isBuiltIn(transportType: UInt32?) -> Bool { + transportType == kAudioDeviceTransportTypeBuiltIn + } + + // MARK: - CoreAudio property helpers + + private static func deviceIDs() -> [AudioDeviceID] { + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard AudioObjectGetPropertyDataSize( + AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size + ) == noErr else { return [] } + + let count = Int(size) / MemoryLayout.size + var deviceIDs = [AudioDeviceID](repeating: 0, count: count) + guard AudioObjectGetPropertyData( + AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceIDs + ) == noErr else { return [] } + return deviceIDs + } + + private static func hasInputChannels(deviceID: AudioDeviceID) -> Bool { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard AudioObjectGetPropertyDataSize(deviceID, &address, 0, nil, &size) == noErr, + size > 0 else { return false } + + let bufferListPointer = UnsafeMutablePointer.allocate(capacity: Int(size)) + defer { bufferListPointer.deallocate() } + guard AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, bufferListPointer) == noErr + else { return false } + + let bufferList = UnsafeMutableAudioBufferListPointer(bufferListPointer) + return bufferList.reduce(0) { $0 + Int($1.mNumberChannels) } > 0 + } + + private static func transportType(deviceID: AudioDeviceID) -> UInt32? { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyTransportType, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var value: UInt32 = 0 + var size = UInt32(MemoryLayout.size) + guard AudioObjectGetPropertyData( + deviceID, &address, 0, nil, &size, &value + ) == noErr else { return nil } + return value + } + + private static func deviceName(deviceID: AudioDeviceID) -> String? { + getStringProperty(deviceID: deviceID, selector: kAudioDevicePropertyDeviceNameCFString) + } + + private static func deviceUID(deviceID: AudioDeviceID) -> String? { + getStringProperty(deviceID: deviceID, selector: kAudioDevicePropertyDeviceUID) + } + + private static func getStringProperty( + deviceID: AudioDeviceID, + selector: AudioObjectPropertySelector + ) -> String? { + var address = AudioObjectPropertyAddress( + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var value: Unmanaged? + var size = UInt32(MemoryLayout?>.size) + let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &value) + guard status == noErr, let cf = value?.takeUnretainedValue() else { return nil } + return cf as String + } +} diff --git a/Sources/Audio/AudioInputResolver.swift b/Sources/Audio/AudioInputResolver.swift new file mode 100644 index 0000000..b2bdbca --- /dev/null +++ b/Sources/Audio/AudioInputResolver.swift @@ -0,0 +1,79 @@ +import Foundation + +enum AudioInputResolution: Equatable { + case use(uid: String) + case unavailable +} + +enum MicFailoverAction: Equatable { + case keep + case switchTo(uid: String) + case fail +} + +/// Chooses a usable input device without touching CoreAudio, so the policy is +/// deterministic and unit-testable. +/// +/// A device is unusable only when it is the built-in microphone and the lid is +/// closed, which is the case Apple disconnects in hardware. +enum AudioInputResolver { + static func resolve( + devices: [AudioInputDevice], + preferredUID: String?, + systemDefaultUID: String?, + lidClosed: Bool + ) -> AudioInputResolution { + if let preferredUID, + let device = devices.first(where: { $0.uid == preferredUID }), + isUsable(device, lidClosed: lidClosed) { + return .use(uid: device.uid) + } + if let systemDefaultUID, + let device = devices.first(where: { $0.uid == systemDefaultUID }), + isUsable(device, lidClosed: lidClosed) { + return .use(uid: device.uid) + } + if let external = devices.first(where: { !$0.isBuiltIn && isUsable($0, lidClosed: lidClosed) }) { + return .use(uid: external.uid) + } + if let fallback = devices.first(where: { isUsable($0, lidClosed: lidClosed) }) { + return .use(uid: fallback.uid) + } + return .unavailable + } + + static func isUsable(_ device: AudioInputDevice, lidClosed: Bool) -> Bool { + !(device.isBuiltIn && lidClosed) + } +} + +/// Decides what a running capture should do when re-checking its input. +/// +/// Keeps the active device whenever it is still usable so a reopened lid does +/// not cause churn; switches only when the active device became unusable. +enum MicFailoverDecision { + static func decide( + activeUID: String?, + devices: [AudioInputDevice], + preferredUID: String?, + systemDefaultUID: String?, + lidClosed: Bool + ) -> MicFailoverAction { + if let activeUID, + let active = devices.first(where: { $0.uid == activeUID }), + AudioInputResolver.isUsable(active, lidClosed: lidClosed) { + return .keep + } + switch AudioInputResolver.resolve( + devices: devices, + preferredUID: preferredUID, + systemDefaultUID: systemDefaultUID, + lidClosed: lidClosed + ) { + case .unavailable: + return .fail + case .use(let uid): + return uid == activeUID ? .keep : .switchTo(uid: uid) + } + } +} diff --git a/Sources/Audio/ClamshellState.swift b/Sources/Audio/ClamshellState.swift new file mode 100644 index 0000000..cbf1a3e --- /dev/null +++ b/Sources/Audio/ClamshellState.swift @@ -0,0 +1,35 @@ +import Foundation +import IOKit + +/// Reads the MacBook lid state from the power-management root domain. +/// +/// Apple hardware disconnects the built-in microphone in hardware while the lid +/// is closed, so capture must route elsewhere instead of trying to re-enable it. +/// Desktops without a lid report no clamshell entry and are treated as open. +enum ClamshellState { + static var isClosed: Bool { + isClosed(fromClamshellState: registryClamshellValue) + } + + /// Pure parse of the `AppleClamshellState` registry value; unit-testable. + static func isClosed(fromClamshellState value: Any?) -> Bool { + if let number = value as? NSNumber { return number.boolValue } + if let flag = value as? Bool { return flag } + return false + } + + private static var registryClamshellValue: Any? { + let service = IOServiceGetMatchingService( + kIOMainPortDefault, + IOServiceMatching("IOPMrootDomain") + ) + guard service != 0 else { return nil } + defer { IOObjectRelease(service) } + return IORegistryEntryCreateCFProperty( + service, + "AppleClamshellState" as CFString, + kCFAllocatorDefault, + 0 + )?.takeRetainedValue() + } +} diff --git a/Sources/Integration/InputSessionCoordinator.swift b/Sources/Integration/InputSessionCoordinator.swift index e6bd71b..1aaf1cf 100644 --- a/Sources/Integration/InputSessionCoordinator.swift +++ b/Sources/Integration/InputSessionCoordinator.swift @@ -91,14 +91,14 @@ final class InputSessionCoordinator { audioCapture.thresholds = thresholds - let micStarted = audioCapture.start( + let micFailure = audioCapture.start( deviceID: microphoneID, levelUpdate: { _ in }, bufferUpdate: effective.streamingEnabled && engine.supportsStreaming ? { buffer in engine.appendAudioBuffer(buffer) } : nil ) - guard micStarted else { + guard micFailure == nil else { engine.cancelListening() audioCapture.stop() audioCapture.cleanupLastRecording() diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 8864034..d507980 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -85,6 +85,8 @@ "pipeline.insert_failed_body" = "The text has been copied to your clipboard. Press ⌘V to paste it manually.\n\nReason: "; "pipeline.mic_failed_permissions" = "Microphone failed — check permissions"; "pipeline.mic_unavailable" = "Microphone unavailable"; +"pipeline.mic_clamshell_no_input" = "The MacBook lid is closed, so the built-in microphone is disabled. Connect an external microphone (display, USB, Bluetooth, or iPhone)."; +"pipeline.mic_switched" = "Switched to an external microphone"; "pipeline.download_failed" = "The model download did not finish. Existing files are preserved; check your network and resume."; "pipeline.compile_failed" = "The model was downloaded but could not be compiled. Retry, then redownload or switch models if it continues."; "pipeline.load_failed" = "The model was downloaded but could not be loaded. Retry, then redownload or switch models if it continues."; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index 31156df..d91283b 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -85,6 +85,8 @@ "pipeline.insert_failed_body" = "文本已复制到剪贴板，请按 ⌘V 手动粘贴。\n\n原因："; "pipeline.mic_failed_permissions" = "麦克风启动失败，请检查权限"; "pipeline.mic_unavailable" = "麦克风不可用"; +"pipeline.mic_clamshell_no_input" = "笔记本已合盖，内置麦克风被禁用。请连接外接麦克风（显示器、USB、蓝牙或 iPhone）。"; +"pipeline.mic_switched" = "已切换到外接麦克风"; "pipeline.download_failed" = "模型下载未完成。已下载内容会保留，请检查网络后继续下载"; "pipeline.compile_failed" = "模型文件已下载，但编译失败。请重试；若仍失败，请重新下载或切换模型"; "pipeline.load_failed" = "模型文件已下载，但加载失败。请重试；若仍失败，请重新下载或切换模型"; diff --git a/Tests/OpenTypeTests/AudioInputResolverTests.swift b/Tests/OpenTypeTests/AudioInputResolverTests.swift new file mode 100644 index 0000000..6c3a507 --- /dev/null +++ b/Tests/OpenTypeTests/AudioInputResolverTests.swift @@ -0,0 +1,179 @@ +import CoreAudio +import XCTest +@testable import OpenType + +final class AudioInputResolverTests: XCTestCase { + private let builtIn = AudioInputDevice(uid: "builtin", name: "MacBook Microphone", isBuiltIn: true) + private let display = AudioInputDevice(uid: "display", name: "Studio Display Microphone", isBuiltIn: false) + private let usb = AudioInputDevice(uid: "usb", name: "USB Microphone", isBuiltIn: false) + private let iphone = AudioInputDevice(uid: "iphone", name: "iPhone Microphone", isBuiltIn: false) + + // MARK: - Clamshell parsing + + func testClamshellParseTreatsOnlyTrueAsClosed() { + XCTAssertTrue(ClamshellState.isClosed(fromClamshellState: NSNumber(value: true))) + XCTAssertFalse(ClamshellState.isClosed(fromClamshellState: NSNumber(value: false))) + XCTAssertFalse(ClamshellState.isClosed(fromClamshellState: nil)) + XCTAssertFalse(ClamshellState.isClosed(fromClamshellState: "not-a-bool")) + } + + // MARK: - Transport classification + + func testBuiltInClassificationUsesTransportType() { + XCTAssertTrue(AudioInputDevices.isBuiltIn(transportType: kAudioDeviceTransportTypeBuiltIn)) + XCTAssertFalse(AudioInputDevices.isBuiltIn(transportType: kAudioDeviceTransportTypeUSB)) + XCTAssertFalse(AudioInputDevices.isBuiltIn(transportType: nil)) + } + + // MARK: - Start resolution + + func testLidOpenHonorsPreferredBuiltIn() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn, display], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: false + ), + .use(uid: "builtin") + ) + } + + func testLidClosedReplacesPreferredBuiltInWithExternal() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn, display], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: true + ), + .use(uid: "display") + ) + } + + func testLidClosedWithOnlyBuiltInIsUnavailable() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: true + ), + .unavailable + ) + } + + func testPreferredExternalAlwaysWinsWhenUsable() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn, display, usb], + preferredUID: "usb", + systemDefaultUID: "builtin", + lidClosed: true + ), + .use(uid: "usb") + ) + } + + func testMissingPreferredFallsBackToSystemDefault() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn, display], + preferredUID: "gone", + systemDefaultUID: "display", + lidClosed: false + ), + .use(uid: "display") + ) + } + + func testContinuityIPhoneIsEligibleFallback() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn, iphone], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: true + ), + .use(uid: "iphone") + ) + } + + func testLidOpenWithNoDefaultStillUsesBuiltIn() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [builtIn], + preferredUID: nil, + systemDefaultUID: nil, + lidClosed: false + ), + .use(uid: "builtin") + ) + } + + func testNoDevicesIsUnavailable() { + XCTAssertEqual( + AudioInputResolver.resolve( + devices: [], + preferredUID: nil, + systemDefaultUID: nil, + lidClosed: false + ), + .unavailable + ) + } + + // MARK: - Mid-session failover + + func testFailoverKeepsUsableActiveDevice() { + XCTAssertEqual( + MicFailoverDecision.decide( + activeUID: "display", + devices: [builtIn, display], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: false + ), + .keep + ) + } + + func testFailoverSwitchesWhenLidClosesOverBuiltIn() { + XCTAssertEqual( + MicFailoverDecision.decide( + activeUID: "builtin", + devices: [builtIn, display], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: true + ), + .switchTo(uid: "display") + ) + } + + func testFailoverFailsWhenActiveLostWithNoAlternative() { + XCTAssertEqual( + MicFailoverDecision.decide( + activeUID: "builtin", + devices: [builtIn], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: true + ), + .fail + ) + } + + func testFailoverSwitchesWhenActiveDeviceRemoved() { + XCTAssertEqual( + MicFailoverDecision.decide( + activeUID: "removed", + devices: [builtIn, display], + preferredUID: "builtin", + systemDefaultUID: "builtin", + lidClosed: false + ), + .switchTo(uid: "builtin") + ) + } +} diff --git a/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/intent.md b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/intent.md new file mode 100644 index 0000000..c6c7a8f --- /dev/null +++ b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/intent.md @@ -0,0 +1,81 @@ +# Intent: Record through an external mic when the MacBook lid is closed + +**Status:** approved +**Approved-by:** User (conversation) +**Approved-date:** 2026-10-01 +**Upstream:** User report in this conversation, 2026-10-01 + +## Problem + +When a MacBook is used with an external display and the lid closed (clamshell +mode), the built-in microphone is physically disconnected in hardware, but +macOS still lists it in CoreAudio as a connected input device. Utter therefore +selects it, records silence, and never surfaces an error: the user speaks, sees +the overlay, and gets no text. This is the docked-at-a-desk case, which is a +primary usage context for a menu-bar dictation app. + +## Outcome + +While docked with the lid closed, Utter records from a usable external input +instead of the disconnected built-in microphone. If the lid closes mid-session, +capture moves to a working external input and the session continues. If no +usable input exists at all, the session ends immediately with a clear, localized +message instead of silently producing nothing. + +## Scope + +- Detect that the MacBook lid is closed, both before a session starts and while + one is recording. +- Treat the built-in microphone as unavailable when the lid is closed, and + classify CoreAudio input devices as built-in or external (including + Continuity/iPhone, USB, Bluetooth, display, and aggregate inputs). +- Resolve a fallback: when the preferred input is the built-in mic and the lid + is closed, automatically choose an available external input; otherwise honor + the user's selected or system-default input. +- Fail over mid-recording when the active device becomes unusable, preserving + the session so speech on both sides of the switch is transcribed. +- Fail fast with a localized error when no usable microphone exists. +- Leave the existing wireless-remote (Xiaomi) capture path unchanged. +- No change to model, LLM, insertion, or release behavior. + +## Constraints + +- Apple's built-in-mic disconnect on lid close is a hardware guarantee; no + software (including Utter) can re-enable it. The fix is failover, not override. +- Never present silence as success: an unusable input must end in an error or a + successful switch, never a silent no-op. +- Do not persist or log audio samples or transcript content; diagnostics stay + numeric, consistent with `AudioCaptureDiagnostics`. +- Preserve existing consent and permission behavior; no new TCC prompts. +- Behavior on desktops without a lid (Mac mini/Studio) and on unlocked lids must + not regress. + +## Acceptance criteria + +- With the built-in mic selected and the lid closed, a started session records + from an available external input rather than the built-in mic (deterministic + test over the resolver decision plus a docked real-device check). +- When the lid is closed and no external input is available, the session fails + fast with a distinct localized message; it does not insert text, write the + clipboard, or create a history entry. +- During a local session, if the active input stops being usable (lid closes + over the built-in mic, or the device is removed), capture switches to an + available external input without ending the session, and the final transcript + includes speech captured on both sides of the switch. +- Continuity/iPhone and other non-built-in inputs are eligible fallback targets. +- With the lid open, the configured/system-default input is still used exactly + as before. +- The wireless-remote path and its selection precedence are unchanged. +- `bash scripts/sdlc-checks.sh`, `bash scripts/ci-basic-checks.sh`, and + `swift test` pass; en and zh-Hans localization keys stay in parity. + +## Open questions + +- When several external inputs exist, which should win? Current decision: the + first available non-built-in input in CoreAudio enumeration order. +- Should an automatic switch show a transient status message, or only be logged? + Current decision: log always; surface a brief status message only when the + session continues, to be confirmed in spec. +- Mid-recording format change (device sample rate/channels differ): append to a + single recording via conversion, or keep segments and concatenate at stop. + Design detail for the spec stage. diff --git a/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/plan.md b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/plan.md new file mode 100644 index 0000000..b125219 --- /dev/null +++ b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/plan.md @@ -0,0 +1,47 @@ +# Plan: Record through an external mic when the MacBook lid is closed + +**Status:** approved +**Approved-by:** User (conversation) +**Approved-date:** 2026-10-01 +**Upstream:** `spec.md` (approved 2026-10-01) + +## Work items + +- [ ] Add `Sources/Audio/ClamshellState.swift` with `isClosed` and a pure + `isClosed(fromClamshellState:)` parser. +- [ ] Add `Sources/Audio/AudioInputDevice.swift` with `AudioInputDevice`, + `AudioInputDevices.available()`, `systemDefaultUID()`, and transport-type + classification. +- [ ] Add `Sources/Audio/AudioInputResolver.swift` with the pure `resolve(...)` + rules and a `MicFailoverDecision` helper. +- [ ] Extend `AudioCaptureManager` to resolve the start device via the resolver + and return `CaptureStartFailure` instead of `Bool`. +- [ ] Add the local-capture watchdog (0.5s) plus + `AVAudioEngineConfigurationChange` observer for mid-session failover, + including `AVAudioConverter` handling when the device format changes. +- [ ] Update `VoicePipeline+Recording.swift` (and the remote-spy seam) to map the + new failure cases to localized messages and to surface a brief status + message when a switch succeeds. +- [ ] Add `pipeline.mic_clamshell_no_input` to `en.lproj` and `zh-Hans.lproj`. +- [ ] Add unit tests for the resolver, clamshell parser, device classification, + and failover decision. +- [ ] Run the automated checks and the manual docked-device verification. + +## Verification plan + +- [ ] `bash scripts/sdlc-checks.sh` +- [ ] `bash scripts/ci-basic-checks.sh` +- [ ] `swift test` +- [ ] Real docked MacBook: built-in mic + lid closed records from an external + input; mid-session lid close continues the session; no external input + produces the localized error. +- [ ] `bash scripts/build-app.sh` if the capture change alters packaging or + runtime dependencies (not expected; run if in doubt). + +## Human gates + +- Intent approval before design or implementation starts. +- Spec approval (medium risk: user-visible audio behavior and a new failure + message) before implementation starts. +- Plan approval before implementation. +- Verification approval and PR review before merge. diff --git a/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/spec.md b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/spec.md new file mode 100644 index 0000000..23afe47 --- /dev/null +++ b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/spec.md @@ -0,0 +1,137 @@ +# Spec: Record through an external mic when the MacBook lid is closed + +**Status:** approved +**Approved-by:** User (conversation) +**Approved-date:** 2026-10-01 +**Upstream:** `intent.md` (approved 2026-10-01) + +## Context + +- `AudioCaptureManager.start(deviceID:levelUpdate:bufferUpdate:)` resolves the + selected UID to a CoreAudio device, sets it on `AVAudioEngine.inputNode`'s + audio unit, installs a tap, and writes buffers to one `AVAudioFile`. The same + tap feeds `bufferUpdate` for streaming recognition. +- `AudioCaptureManager.availableMicrophones()` enumerates every device with + input channels and reports the built-in mic even when the lid is closed, which + is the root cause. +- `VoiceInputSettings.microphoneID` comes from `AppSettings` and is chosen in + `GeneralSettingsView` via a picker. +- The wireless-remote path takes precedence when a remote session token is + latched (`remoteMicSource.currentSessionToken`), and must stay untouched. +- `DeviceCapability` already imports IOKit, so IOKit is available without new + dependencies or entitlements. + +## Design + +### 1. Lid state (`Sources/Audio/ClamshellState.swift`) + +- `enum ClamshellState` exposes `static var isClosed: Bool` by reading the + `AppleClamshellState` boolean from the `IOPMrootDomain` IORegistry entry. +- A pure `static func isClosed(fromClamshellState: Any?) -> Bool` parses the + registry value so it is unit-testable without hardware. +- Missing entry (desktops, or inaccessible) yields `false`, so non-lid Macs are + unaffected. +- No notification wiring: mid-session detection uses a bounded watchdog (below), + which is simpler and deterministic to test. + +### 2. Device catalog and classification + +- `struct AudioInputDevice: Equatable { let uid: String; let name: String; let isBuiltIn: Bool }`. +- `AudioInputDevices.available() -> [AudioInputDevice]` enumerates devices with + input channels and sets `isBuiltIn` from + `kAudioDevicePropertyTransportType == kAudioDeviceTransportTypeBuiltIn`. +- `AudioInputDevices.systemDefaultUID() -> String?` reads + `kAudioHardwarePropertyDefaultInputDevice`. +- External means `!isBuiltIn`; Continuity/iPhone, USB, Bluetooth, display, and + aggregate inputs all qualify. + +### 3. Pure resolver (`AudioInputResolver`) + +`static func resolve(devices:preferredUID:systemDefaultUID:lidClosed:) -> Resolution` +where `Resolution = .use(uid: String) | .unavailable`. + +Rules, in order: + +1. A device is *usable* unless it is built-in and `lidClosed`. +2. If `preferredUID` names a usable device, use it. +3. Otherwise, if the system default is a usable device, use it. +4. Otherwise, if any non-built-in device is usable, use the first one. +5. Otherwise, `.unavailable`. + +This keeps the user's explicit choice authoritative when it works, and only +substitutes when the chosen input is the disconnected built-in microphone (or +the chosen device is gone). + +### 4. Capture start (`AudioCaptureManager`) + +- `start(...)` adds a `forcedUID` resolution step before `setInputDevice`: + resolve from `AudioInputDevices.available()`, `preferredUID = deviceID`, + `systemDefaultUID`, and `ClamshellState.isClosed`. +- On `.unavailable`, return a typed failure instead of a bare `false`: + `enum CaptureStartFailure { case permissionDenied, noUsableInput, engineFailed }`. + Existing `Bool` call sites migrate to `Result<(), CaptureStartFailure>`; the + remote-spy seam maps `noUsableInput`/`permissionDenied` to the current + `pipeline.mic_unavailable` behavior. +- On success, record the resolved UID as `activeInputUID` and that a fallback + was used, for numeric diagnostics only. + +### 5. Mid-session failover + +- While a local capture is running, a serial-queue watchdog re-evaluates the + resolver every 0.5s with current devices and lid state (plus an + `AVAudioEngineConfigurationChange` observer for device removal). +- If the active device is no longer usable and an alternative resolves, switch: + stop the engine and tap, set the new device, restart, and re-install the tap. + Level and buffer callbacks stay wired so streaming and the overlay are + uninterrupted. +- Recording file handling: append into the existing `AVAudioFile` when the new + device's format matches the file's. When it differs, convert buffers with + `AVAudioConverter` to the file's processing format before writing. If + conversion cannot be created, fail the session rather than lose audio. +- If no alternative resolves mid-session, stop capture and report the same + `noUsableInput` failure so the pipeline ends with a localized error. + +### 6. Pipeline and UI + +- `VoicePipeline+Recording.swift` maps `noUsableInput` to a new localized + message `pipeline.mic_clamshell_no_input`; `permissionDenied` keeps + `pipeline.mic_failed_permissions`. Mid-session failure calls `stopRecording` + and shows the message. +- Add a brief status message when a mid-session switch succeeds, reusing the + overlay status channel. (Confirms intent open question: signal, not silent.) +- No new settings screen; the existing microphone picker stays as the preference + source. Localization keys are added to both `en.lproj` and `zh-Hans.lproj`. + +## Safety and failure modes + +- Apple's hardware disconnect cannot be overridden; this change only reroutes. +- Never treat a silent built-in capture as success: every unusable-input path + ends in a switch or an error. +- Falling back to Bluetooth/Continuity can add latency and reduce quality; the + substitution is automatic but only when the selected input is unusable. +- A switch may drop up to the watchdog interval (≤0.5s) of audio at the seam; + acceptable and recorded as residual risk. +- Diagnostics stay numeric (`AudioCaptureDiagnostics`); no samples, no + transcripts, no new permissions or TCC prompts. +- If engine restart or format conversion fails mid-failover, the session ends + with an error instead of continuing silently. + +## Test strategy + +- `AudioInputResolver` unit tests: lid open/closed, `preferredUID` nil, preferred + external, preferred built-in with an external present, preferred built-in with + none present, chosen device missing, desktop (no built-in). +- `ClamshellState.isClosed(fromClamshellState:)` parse tests: true/false/nil. +- `AudioInputDevice` classification test with an injected transport type. +- Pure `MicFailoverDecision.shouldSwitch(activeUID:resolution:)` tests covering + continue vs. switch vs. fail. +- Manual docked-device check: start with lid closed, close lid mid-session, + remove all external inputs. + +## Rollout and rollback + +- Internal behavior change; ships with the next normal release. No migration. +- Rollback: revert the change set. No persisted state is written, so reverting is + clean and immediate. +- Observation: numeric diagnostics confirm which input was used and whether a + fallback occurred; no user audio is logged. diff --git a/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/verification.md b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/verification.md new file mode 100644 index 0000000..f247132 --- /dev/null +++ b/docs/sdlc/changes/2026-10-01-clamshell-mic-failover/verification.md @@ -0,0 +1,52 @@ +# Verification: Record through an external mic when the MacBook lid is closed + +**Status:** pending approval +**Approved-by:** — +**Approved-date:** — +**Upstream:** `plan.md` (approved 2026-10-01) + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| `bash scripts/sdlc-checks.sh` | Pass | `SDLC checks passed.` | +| `bash scripts/ci-basic-checks.sh` | Pass | `Basic CI checks passed.`; both `Localizable.strings` lint OK and key parity holds | +| `swift test` | Pass | `808 tests, 18 skipped, 0 failures` (20.4s); run with `DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer` because the CommandLineTools SDK has no XCTest | +| Docked MacBook manual check | Not run | Requires physical docked hardware; see residual risk | + +## Acceptance criteria + +- Built-in mic selected + lid closed records from an external input — covered by + the resolver tests (`testLidClosedReplacesPreferredBuiltInWithExternal`) and + the `MicFailoverDecision` test; real-device check not run. +- Lid closed + no external input fails fast with a localized message and no + insertion/clipboard/history side effect — `testLidClosedWithOnlyBuiltInIsUnavailable` + and `testFailoverFailsWhenActiveLostWithNoAlternative`; the pipeline sets + `.error` and returns before `commitRecording`, so no output path runs. Real + device not run. +- Mid-session lid close continues the session and transcribes both sides — + decision logic covered by `testFailoverSwitchesWhenLidClosesOverBuiltIn`; the + engine restart and `AVAudioConverter` path is not exercised by hardware here. +- Continuity/iPhone and other non-built-in inputs are eligible fallbacks — + `testContinuityIPhoneIsEligibleFallback` and + `testPreferredExternalAlwaysWinsWhenUsable`. +- Lid open preserves the configured/default input; remote path unchanged — + `testLidOpenHonorsPreferredBuiltIn`, `testLidOpenWithNoDefaultStillUsesBuiltIn`; + the remote branch in `start` is untouched. +- Localization parity and repository checks pass — Pass (`ci-basic-checks.sh`). + +## Residual risk + +- Bluetooth/Continuity fallback may add latency or lower fidelity. +- A failover seam may drop up to the watchdog interval (≤0.5s) of audio. +- The real capture failover (engine restart + format conversion) and the + localized no-input message have not been exercised on physical docked + hardware in this environment; a maintainer must run the docked check before + merge. +- `swift test` needs the Xcode toolchain in this environment; plain `swift test` + fails before compiling because the CommandLineTools SDK lacks XCTest. + +## Decision + +Implementation complete; awaiting human verification approval and the real +docked-device check before PR approval.