Skip to content

fix: improve Android transcription failure diagnostics #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
grapeot wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: improve Android transcription failure diagnostics #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 135 additions & 19 deletions app/src/main/java/com/yage/opencode_client/data/audio/AIBuildersAudioClient.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,22 @@ import java.net.URI
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean

internal enum class SpeechFailureKind {
LOCAL_AUDIO,
CLIENT,
CONFIG,
NETWORK,
SERVER,
UNKNOWN
}

internal class SpeechTranscriptionException(
val kind: SpeechFailureKind,
val stage: String,
detail: String,
cause: Throwable? = null
) : IOException(detail, cause)

data class TranscriptionResponse(
val requestId: String,
val text: String
Expand Down Expand Up @@ -73,7 +89,11 @@ object AIBuildersAudioClient {
val normalizedBase = normalizedBaseURL(baseURL)
val cleanedToken = sanitizeBearerToken(token)
if (cleanedToken.isEmpty()) {
throw IOException("AI Builder token is empty")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.CONFIG,
stage = "speech.config.token",
detail = "AI Builder token is empty"
)
}

val session = createRealtimeSession(
Expand Down Expand Up @@ -203,20 +223,55 @@ object AIBuildersAudioClient {
.post(payload.toString().toRequestBody(AudioTranscriptionConfig.jsonMediaType.toMediaType()))
.build()

client.newCall(request).execute().use { response ->
if (response.code >= 400) {
throw IOException("Create session failed with status ${response.code}")
}
try {
client.newCall(request).execute().use { response ->
if (response.code >= 400) {
val message = "HTTP ${response.code} while creating realtime session"
Log.e(TAG, "speech.server.session.create failed: $message")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.SERVER,
stage = "speech.server.session.create",
detail = message
)
}

val bodyText = response.body?.string()
?: throw IOException("Create session returned empty body")
val bodyJson = JSONObject(bodyText)
val sessionId = bodyJson.getString("session_id")
val wsUrl = bodyJson.getString("ws_url")
val bodyText = response.body?.string()
if (bodyText.isNullOrBlank()) {
Log.e(TAG, "speech.server.session.create failed: empty response body")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.SERVER,
stage = "speech.server.session.create",
detail = "Empty response body while creating realtime session"
)
}
val bodyJson = try {
JSONObject(bodyText)
} catch (error: Exception) {
Log.e(TAG, "speech.server.session.create failed: invalid JSON response", error)
throw SpeechTranscriptionException(
kind = SpeechFailureKind.SERVER,
stage = "speech.server.session.create",
detail = "Invalid JSON response while creating realtime session",
cause = error
)
}
val sessionId = bodyJson.getString("session_id")
val wsUrl = bodyJson.getString("ws_url")

return RealtimeSessionResponse(
sessionId = sessionId,
wsUrl = wsUrl
return RealtimeSessionResponse(
sessionId = sessionId,
wsUrl = wsUrl
)
}
} catch (error: SpeechTranscriptionException) {
throw error
} catch (error: Exception) {
Log.e(TAG, "speech.net.session.create failed", error)
throw SpeechTranscriptionException(
kind = SpeechFailureKind.NETWORK,
stage = "speech.net.session.create",
detail = error.message ?: "Failed to create realtime session",
cause = error
)
}
}
Expand All @@ -228,6 +283,15 @@ object AIBuildersAudioClient {
pcmAudio: ByteArray,
onPartialTranscript: ((String) -> Unit)?
): TranscriptionResponse {
if (pcmAudio.isEmpty()) {
Log.e(TAG, "speech.audio.empty failed for session=$sessionId: PCM payload is empty")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.LOCAL_AUDIO,
stage = "speech.audio.empty",
detail = "Recorded audio produced an empty PCM payload"
)
}

val readySignal = CompletableDeferred<Unit>()
val resultSignal = CompletableDeferred<TranscriptionResponse>()
val closed = AtomicBoolean(false)
Expand Down Expand Up @@ -290,24 +354,66 @@ object AIBuildersAudioClient {
val message = event.optString("message").ifEmpty {
event.optString("code", "Unknown websocket error")
}
Log.e(TAG, "speech.server.transcription failed for session=$sessionId: $message")
if (closed.compareAndSet(false, true)) {
resultSignal.completeExceptionally(IOException(message))
resultSignal.completeExceptionally(
SpeechTranscriptionException(
kind = SpeechFailureKind.SERVER,
stage = "speech.server.transcription",
detail = message
)
)
}
}
}
} catch (error: Exception) {
Log.e(TAG, "speech.client.ws.parse failed for session=$sessionId", error)
if (closed.compareAndSet(false, true)) {
resultSignal.completeExceptionally(error)
resultSignal.completeExceptionally(
SpeechTranscriptionException(
kind = SpeechFailureKind.CLIENT,
stage = "speech.client.ws.parse",
detail = error.message ?: "Failed to parse websocket event",
cause = error
)
)
}
}
}

override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
val responseCode = response?.code
val kind = if (responseCode != null && responseCode >= 400) {
SpeechFailureKind.SERVER
} else {
SpeechFailureKind.NETWORK
}
val stage = if (kind == SpeechFailureKind.SERVER) {
"speech.server.ws.connect"
} else {
"speech.net.ws.connect"
}
val detail = if (responseCode != null && responseCode >= 400) {
"WebSocket upgrade failed with HTTP $responseCode"
} else {
t.message ?: "WebSocket connection failed"
}
Log.e(
TAG,
"$stage failed for session=$sessionId responseCode=${responseCode ?: "none"}",
t
)
val failure = SpeechTranscriptionException(
kind = kind,
stage = stage,
detail = detail,
cause = t
)
if (!readySignal.isCompleted) {
readySignal.completeExceptionally(t)
readySignal.completeExceptionally(failure)
}
if (closed.compareAndSet(false, true)) {
resultSignal.completeExceptionally(t)
resultSignal.completeExceptionally(failure)
}
}
}
Expand All @@ -334,13 +440,23 @@ object AIBuildersAudioClient {
.toByteString()
)
if (!sent) {
throw IOException("Failed to send audio chunk")
Log.e(TAG, "speech.net.ws.send failed for session=$sessionId: audio chunk send returned false")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.NETWORK,
stage = "speech.net.ws.send",
detail = "Failed to send audio chunk"
)
}
chunkStart = chunkEnd
}

if (!webSocket.send("{\"type\":\"commit\"}")) {
throw IOException("Failed to send commit event")
Log.e(TAG, "speech.net.ws.send failed for session=$sessionId: commit send returned false")
throw SpeechTranscriptionException(
kind = SpeechFailureKind.NETWORK,
stage = "speech.net.ws.send",
detail = "Failed to send commit event"
)
}
Log.d(TAG, "Commit event sent")

Expand Down
54 changes: 32 additions & 22 deletions app/src/main/java/com/yage/opencode_client/data/audio/AudioRecorderManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,30 +85,40 @@ class AudioRecorderManager @Inject constructor(
}

suspend fun convertToPCM(m4aFile: File): ByteArray = withContext(Dispatchers.Default) {
Log.d(TAG, "Converting M4A to PCM: ${m4aFile.absolutePath}")
val decodeResult = decodeM4aToPCM(m4aFile)
val pcmSamples = if (decodeResult.sampleRate != AudioRecorderConfig.targetPcmSampleRate) {
resamplePCM(
decodeResult.samples,
decodeResult.sampleRate,
AudioRecorderConfig.targetPcmSampleRate
)
} else {
decodeResult.samples
}
try {
Log.d(TAG, "Converting M4A to PCM: ${m4aFile.absolutePath}")
val decodeResult = decodeM4aToPCM(m4aFile)
val pcmSamples = if (decodeResult.sampleRate != AudioRecorderConfig.targetPcmSampleRate) {
resamplePCM(
decodeResult.samples,
decodeResult.sampleRate,
AudioRecorderConfig.targetPcmSampleRate
)
} else {
decodeResult.samples
}

val pcmBytes = ByteBuffer
.allocate(pcmSamples.size * 2)
.order(ByteOrder.LITTLE_ENDIAN)
pcmSamples.forEach { sample ->
pcmBytes.putShort(sample)
}
val pcmBytes = ByteBuffer
.allocate(pcmSamples.size * 2)
.order(ByteOrder.LITTLE_ENDIAN)
pcmSamples.forEach { sample ->
pcmBytes.putShort(sample)
}

Log.d(
TAG,
"PCM conversion complete. inputRate=${decodeResult.sampleRate}, outputRate=${AudioRecorderConfig.targetPcmSampleRate}, bytes=${pcmBytes.array().size}"
)
pcmBytes.array()
Log.d(
TAG,
"PCM conversion complete. inputRate=${decodeResult.sampleRate}, outputRate=${AudioRecorderConfig.targetPcmSampleRate}, bytes=${pcmBytes.array().size}"
)
pcmBytes.array()
} catch (error: Exception) {
Log.e(TAG, "speech.audio.decode failed for ${m4aFile.absolutePath}", error)
throw SpeechTranscriptionException(
kind = SpeechFailureKind.LOCAL_AUDIO,
stage = "speech.audio.decode",
detail = error.message ?: "Failed to decode recorded audio",
cause = error
)
}
}

private fun decodeM4aToPCM(m4aFile: File): DecodedPCM {
Expand Down
8 changes: 4 additions & 4 deletions app/src/main/java/com/yage/opencode_client/ui/MainViewModelSpeechActions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,22 +62,22 @@ internal fun launchSpeechTranscription(
)
}
}.onFailure { error ->
Log.e(tag, "Transcription failed", error)
Log.e(tag, "Speech transcription failed at ${speechLogStage(error, "speech.remote")}", error)
state.update {
it.copy(
inputText = existingInput,
isTranscribing = false,
speechError = errorMessageOrFallback(error, "Transcription failed")
speechError = speechErrorMessage(error, "Transcription failed")
)
}
}
} catch (error: Exception) {
Log.e(tag, "Speech processing failed", error)
Log.e(tag, "Speech processing failed at ${speechLogStage(error, "speech.local")}", error)
state.update {
it.copy(
inputText = existingInput,
isTranscribing = false,
speechError = errorMessageOrFallback(error, "Transcription failed")
speechError = speechErrorMessage(error, "Transcription failed")
)
}
}
Expand Down
24 changes: 24 additions & 0 deletions app/src/main/java/com/yage/opencode_client/ui/MainViewModelSupport.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.yage.opencode_client.ui

import android.util.Log
import com.yage.opencode_client.data.audio.SpeechFailureKind
import com.yage.opencode_client.data.audio.SpeechTranscriptionException
import com.yage.opencode_client.data.model.Part
import com.yage.opencode_client.data.model.QuestionRequest
import com.yage.opencode_client.data.model.SSEEvent
Expand Down Expand Up @@ -48,6 +50,28 @@ internal fun errorMessageOrFallback(throwable: Throwable?, fallback: String): St
return if (message.isEmpty()) fallback else message
}

internal fun speechLogStage(throwable: Throwable?, fallback: String): String {
return (throwable as? SpeechTranscriptionException)?.stage ?: fallback
}

internal fun speechErrorMessage(throwable: Throwable?, fallback: String): String {
val stage = speechLogStage(throwable, "speech.unknown")
val detail = errorMessageOrFallback(throwable, fallback)
val prefix = when ((throwable as? SpeechTranscriptionException)?.kind ?: SpeechFailureKind.UNKNOWN) {
SpeechFailureKind.LOCAL_AUDIO -> "Local audio processing failed"
SpeechFailureKind.CLIENT -> "Client-side speech handling failed"
SpeechFailureKind.CONFIG -> "Speech recognition configuration failed"
SpeechFailureKind.NETWORK -> "Network upload or connection failed"
SpeechFailureKind.SERVER -> "Server transcription failed"
SpeechFailureKind.UNKNOWN -> fallback
}
return if (prefix == fallback) {
detail
} else {
"$prefix ($stage): $detail"
}
}

internal fun parseSessionCreatedEvent(event: SSEEvent): SessionCreatedEvent? {
val sessionJson = event.payload.getJsonObject("session") ?: return null
return runCatching {
Expand Down
Loading