Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions Sources/App/VoicePipeline+Models.swift
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,18 @@ extension VoicePipeline {
let engine = QwenNativeASREngine(modelPath: modelPath)
qwenSpeechEngine = engine
Task { await engine.prepare() }
case .mimo:
appState.settings.speechEngine = .apple
await ensureEngineLoaded(requestPermission: requestPermission)
case .firered, .megaASR:
let settings = appState.settings
guard let modelID = settings.speechEngine.asrModelID else { return }
guard localASRIsAvailable(modelID) else {
mlxSTTEngine = nil
markSpeechModelDownloadRequired(showInStatus: requestPermission)
return
}
if mlxSTTEngine?.modelID == modelID { return }
let engine = MLXSTTEngine(modelID: modelID)
mlxSTTEngine = engine
Task { await engine.prepare() }
}
}

Expand Down
3 changes: 2 additions & 1 deletion Sources/App/VoicePipeline.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ final class VoicePipeline {
var appleSpeechEngine: AppleSpeechEngine?
var volcSpeechEngine: VolcSpeechEngine?
var qwenSpeechEngine: QwenNativeASREngine?
var mlxSTTEngine: MLXSTTEngine?
var screenOCRTask: Task<ScreenContextSnapshot, Never>?
var screenOCRStartedAt: CFAbsoluteTime?
var processingTask: Task<Void, Never>?
Expand All @@ -29,7 +30,7 @@ final class VoicePipeline {
case .apple: return appleSpeechEngine
case .volc: return volcSpeechEngine
case .qwen3: return qwenSpeechEngine
case .mimo: return nil
case .firered, .megaASR: return mlxSTTEngine
}
}

Expand Down
18 changes: 15 additions & 3 deletions Sources/Config/AppSettingTypes.swift
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@ enum SpeechEngineType: String, Codable, CaseIterable {
case apple = "apple"
case volc = "volc"
case qwen3 = "qwen3"
case mimo = "mimo"
case firered = "firered"
case megaASR = "megaASR"

static var selectableCases: [SpeechEngineType] {
[.qwen3, .whisper, .apple, .volc]
[.qwen3, .firered, .megaASR, .whisper, .apple, .volc]
}

var label: String {
Expand All @@ -43,7 +44,18 @@ enum SpeechEngineType: String, Codable, CaseIterable {
case .apple: return L("engine.apple_speech")
case .volc: return L("engine.volc_asr")
case .qwen3: return L("engine.qwen3_asr")
case .mimo: return L("engine.mimo_asr")
case .firered: return L("engine.firered_asr")
case .megaASR: return L("engine.mega_asr")
}
}

/// The ASR model ID associated with this engine, if any.
var asrModelID: String? {
switch self {
case .qwen3: return QwenASRModel.defaultID
case .firered: return "mlx-community/FireRedASR2-AED-mlx"
case .megaASR: return "mlx-community/Mega-ASR-6bit"
default: return nil
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions Sources/Config/AppSettings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ final class AppSettings: ObservableObject {
let loadedSpeechEngine = SpeechEngineType(rawValue: savedEngine)
?? (savedEngine.contains("Whisper") || savedEngine.contains("whisper") ? .whisper : nil)
?? .apple
speechEngine = loadedSpeechEngine == .mimo ? .apple : loadedSpeechEngine
if loadedSpeechEngine == .mimo {
speechEngine = savedEngine == "mimo" ? .apple : loadedSpeechEngine
if savedEngine == "mimo" {
ud.set(SpeechEngineType.apple.rawValue, forKey: Key.speechEngine.rawValue)
}
[
Expand Down
36 changes: 32 additions & 4 deletions Sources/Config/ModelCatalogASR.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,34 @@ extension ModelCatalog {
QwenASRModel.defaultID,
"Qwen3-ASR 1.7B",
L("model.qwen3_asr_quality")
)
),
(
"mlx-community/FireRedASR2-AED-mlx",
"FireRedASR2-AED",
L("model.firered_asr")
),
(
"mlx-community/Mega-ASR-6bit",
"Mega-ASR 6bit",
L("model.mega_asr")
),
]
}

/// All ASR model IDs that use the generic MLX STT engine.
static let mlxSTTModelIDs: Set<String> = [
"mlx-community/FireRedASR2-AED-mlx",
"mlx-community/Mega-ASR-6bit",
]

func asrModels(for engine: SpeechEngineType) -> [ModelEntry] {
switch engine {
case .qwen3:
return asrModels.filter { $0.id == QwenASRModel.defaultID }
case .mimo:
return []
case .firered:
return asrModels.filter { $0.id == "mlx-community/FireRedASR2-AED-mlx" }
case .megaASR:
return asrModels.filter { $0.id == "mlx-community/Mega-ASR-6bit" }
default:
return []
}
Expand Down Expand Up @@ -165,7 +183,7 @@ extension ModelCatalog {
}
}

static func asrRequiredFiles(for id: String) -> [String] {
nonisolated static func asrRequiredFiles(for id: String) -> [String] {
switch id {
case QwenASRModel.defaultID:
return [
Expand All @@ -177,6 +195,16 @@ extension ModelCatalog {
"vocab.json",
"merges.txt",
]
case "mlx-community/FireRedASR2-AED-mlx":
return [
"config.json",
"tokenizer.json",
]
case "mlx-community/Mega-ASR-6bit":
return [
"config.json",
"tokenizer_config.json",
]
default:
return ["config.json"]
}
Expand Down
2 changes: 2 additions & 0 deletions Sources/Config/ModelCatalogDownloadEstimates.swift
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ extension ModelCatalog {
"mlx-community/Llama-4-Scout-17B-16E-Instruct-4bit": 61_143_654_248,
"mlx-community/Llama-4-Maverick-17B-128E-Instruct-4bit": 225_923_469_800,
QwenASRModel.defaultID: 4_080_707_826,
"mlx-community/FireRedASR2-AED-mlx": 4_570_000_000,
"mlx-community/Mega-ASR-6bit": 2_040_000_000,
]

private static let downloadEstimateRegex = try! NSRegularExpression(
Expand Down
9 changes: 9 additions & 0 deletions Sources/Processing/TranscriptionSanitizer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ enum TranscriptionSanitizer {
audioActivity: AudioCaptureActivity?
) -> String? {
guard audioActivity?.hasWeakSpeechEvidence == true else { return text }

if weakAudioWholeTranscriptHallucinations.contains(normalizedPhrase(text)) {
return nil
}

var cleaned = text.trimmingCharacters(in: .whitespacesAndNewlines)
for pattern in trailingHallucinationPatterns {
let candidate = cleaned.replacingOccurrences(
Expand Down Expand Up @@ -173,6 +178,10 @@ enum TranscriptionSanitizer {
"[blank_audio]", "<blank_audio>",
]

private static let weakAudioWholeTranscriptHallucinations: Set<String> = [
"do anything",
]

private static let trailingHallucinationPatterns = [
#"\s*(?:thank you for watching|thanks for watching)[.!?。!?]*\s*$"#,
#"\s*(?:感谢观看|謝謝觀看|谢谢观看|谢谢收看|感謝收看)[。.!!!??]*\s*$"#,
Expand Down
8 changes: 6 additions & 2 deletions Sources/Resources/en.lproj/Localizable.strings
Original file line number Diff line number Diff line change
Expand Up @@ -290,11 +290,13 @@
"engine.apple_short" = "Apple";
"engine.volc_short" = "Doubao";
"engine.qwen3_short" = "Qwen";
"engine.mimo_short" = "MiMo";
"engine.firered_short" = "FireRed";
"engine.mega_short" = "Mega";
"engine.apple_speech" = "Apple Speech";
"engine.volc_asr" = "Doubao ASR";
"engine.qwen3_asr" = "Qwen3-ASR (Local)";
"engine.mimo_asr" = "MiMo-V2.5-ASR (Local)";
"engine.firered_asr" = "FireRedASR2-AED (Local)";
"engine.mega_asr" = "Mega-ASR (Local)";
"screen_context_mode.ocr" = "OCR";
"screen_context_mode.multimodal" = "Multimodal";
"style.prompt.concise" = "Minimalist. Keep only core information, remove repetition and filler, break long sentences short.";
Expand Down Expand Up @@ -490,6 +492,8 @@
/* ── Local ASR ── */
"qwen_asr.config_hint" = "Qwen3-ASR runs locally with native Swift and MLX. Download the model once, then recognition works offline.";
"model.qwen3_asr_quality" = "Local ASR through MLX, ~4.1 GB";
"model.firered_asr" = "Robust Chinese ASR, ~4.6 GB";
"model.mega_asr" = "Noise-robust ASR (Qwen3-ASR + LoRA), ~2.0 GB";
"model.asr_incomplete" = "Only part of the model was downloaded. Select Resume to finish.";
"error.llm_not_loaded" = "The model is stored locally but is not loaded into memory. Run the action again and Utter will retry.";
"error.llm_not_downloaded" = "The model files have not been downloaded. Open Settings → Models and confirm the data usage first.";
Expand Down
8 changes: 6 additions & 2 deletions Sources/Resources/zh-Hans.lproj/Localizable.strings
Original file line number Diff line number Diff line change
Expand Up @@ -290,11 +290,13 @@
"engine.apple_short" = "Apple";
"engine.volc_short" = "豆包";
"engine.qwen3_short" = "Qwen";
"engine.mimo_short" = "MiMo";
"engine.firered_short" = "FireRed";
"engine.mega_short" = "Mega";
"engine.apple_speech" = "Apple 语音识别";
"engine.volc_asr" = "豆包语音识别";
"engine.qwen3_asr" = "Qwen3-ASR(本地)";
"engine.mimo_asr" = "MiMo-V2.5-ASR(本地)";
"engine.firered_asr" = "FireRedASR2-AED(本地)";
"engine.mega_asr" = "Mega-ASR(本地)";
"screen_context_mode.ocr" = "OCR";
"screen_context_mode.multimodal" = "多模态";
"style.prompt.concise" = "极简。只保留核心信息,删掉修饰、重复和过渡,长句拆短。";
Expand Down Expand Up @@ -490,6 +492,8 @@
/* ── 本地语音识别 ── */
"qwen_asr.config_hint" = "Qwen3-ASR 使用原生 Swift 和 MLX 在本机运行。模型下载一次后即可离线识别。";
"model.qwen3_asr_quality" = "本地 MLX 语音识别,约 4.1 GB";
"model.firered_asr" = "强鲁棒中文 ASR,约 4.6 GB";
"model.mega_asr" = "抗噪 ASR(Qwen3-ASR + LoRA),约 2.0 GB";
"model.asr_incomplete" = "模型只下载了一部分。点击“继续下载”即可接着完成";
"error.llm_not_loaded" = "模型文件已在本地,但当前尚未加载到内存。请重新执行;Utter 会再次尝试加载";
"error.llm_not_downloaded" = "模型文件尚未下载。请前往 设置 → 模型,确认流量后下载";
Expand Down
Loading
Loading