Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Sources/App/DeferredReplacement.swift
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@ struct DeferredReplacement {
var state: DeferredReplacementState
var message: String
var context: InputContext?
let formatKind: TextFormatKind

init(
rawText: String,
insertedText: String,
targetApp: NSRunningApplication?,
message: String,
context: InputContext? = nil,
formatKind: TextFormatKind = .plainParagraph,
createdAt: Date = Date(),
expirationInterval: TimeInterval = DeferredReplacementPolicy.expirationInterval
) {
Expand All @@ -44,6 +46,7 @@ struct DeferredReplacement {
self.state = .formatting
self.message = message
self.context = context
self.formatKind = formatKind
}

var targetApplication: NSRunningApplication? {
Expand Down
19 changes: 19 additions & 0 deletions Sources/App/VoicePipeline+CorrectionCapture.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import Foundation

@MainActor
extension VoicePipeline {
func beginCorrectionCapture(
recordID: UUID,
insertedText: String,
context: InputContext
) {
guard appState.settings.enableCorrectionLearning,
let seed = textInserter.correctionCaptureSeed(
expectedText: insertedText,
context: context
) else {
return
}
correctionCapture.start(seed: seed, recordID: recordID)
}
}
21 changes: 18 additions & 3 deletions Sources/App/VoicePipeline+EditCommands.swift
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,18 @@ extension VoicePipeline {
return
}

InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: replacementText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = replacementText
beginCorrectionCapture(
recordID: recordID,
insertedText: replacementText,
context: context
)
}

private func replaceSelectedText(
Expand Down Expand Up @@ -142,13 +147,18 @@ extension VoicePipeline {
return
}

InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: replacementText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = replacementText
beginCorrectionCapture(
recordID: recordID,
insertedText: replacementText,
context: context
)
}

private func replacementInputContext(
Expand Down Expand Up @@ -287,12 +297,17 @@ extension VoicePipeline {
return
}

InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: rewrittenText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = rewrittenText
beginCorrectionCapture(
recordID: recordID,
insertedText: rewrittenText,
context: context
)
}
}
24 changes: 21 additions & 3 deletions Sources/App/VoicePipeline+Processing.swift
Original file line number Diff line number Diff line change
Expand Up @@ -174,17 +174,20 @@ extension VoicePipeline {
memoryWindowMinutes: memoryWindowMinutes,
currentContext: inputContext
)
let formatDecision = TextFormatClassifier.classify(text: raw, context: inputContext)
Log.info("[VoicePipeline] format kind \(formatDecision.kind.rawValue) (\(formatDecision.reason.rawValue))")
let text = await textProcessor.process(
text: raw,
options: processingOptions,
screenContext: screenContext.text,
screenImage: screenContext.image,
memoryContext: memoryContext,
inputContext: inputContext,
formatKind: formatDecision.kind,
dictionarySnapshot: dictionarySnapshot
)
recordFormattingDuration(started, label: "Smart Format")
return VoicePipelineOutput(text: text, context: inputContext)
return VoicePipelineOutput(text: text, context: inputContext, formatKind: formatDecision.kind)
}

private func processCommand(
Expand Down Expand Up @@ -273,19 +276,34 @@ extension VoicePipeline {
let wasProcessed = inputMode.isTranslation
|| settings.outputMode == .processed
|| settings.outputMode == .command
InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: finalText,
wasProcessed: wasProcessed,
context: output.context
context: output.context,
formatKind: output.formatKind
)
appState.lastInsertedText = finalText
if !inputMode.isTranslation {
beginCorrectionCapture(
recordID: recordID,
insertedText: finalText,
context: output.context
)
}
}
}

struct VoicePipelineOutput {
let text: String
let context: InputContext
let formatKind: TextFormatKind?

init(text: String, context: InputContext, formatKind: TextFormatKind? = nil) {
self.text = text
self.context = context
self.formatKind = formatKind
}
}

private enum VoicePipelineStop: Error {
Expand Down
33 changes: 28 additions & 5 deletions Sources/App/VoicePipeline+Replacement.swift
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ extension VoicePipeline {
inputLanguage: processingOptions.inputLanguage,
source: .menuBar
)
let formatDecision = TextFormatClassifier.classify(text: raw, context: quickContext)
let ocrTask = screenOCRTask
let ocrStartedAt = screenOCRStartedAt
screenOCRTask = nil
Expand Down Expand Up @@ -88,20 +89,27 @@ extension VoicePipeline {
return
}

InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: quickText,
wasProcessed: false,
context: quickContext
context: quickContext,
formatKind: formatDecision.kind
)
appState.lastInsertedText = quickText
beginCorrectionCapture(
recordID: recordID,
insertedText: quickText,
context: quickContext
)

let replacement = DeferredReplacement(
rawText: raw,
insertedText: quickText,
targetApp: targetApp,
message: L("pipeline.background_formatting"),
context: quickContext
context: quickContext,
formatKind: formatDecision.kind
)
appState.pendingReplacement = replacement

Expand Down Expand Up @@ -131,6 +139,8 @@ extension VoicePipeline {
appState.phase = .inserting
appState.statusMessage = L("pipeline.replacing")

correctionCapture.finishCurrentSession()

let result = await textInserter.replaceRecentInsertion(
text: formattedText,
previouslyInserted: replacement.insertedText,
Expand All @@ -150,11 +160,23 @@ extension VoicePipeline {

appState.processedText = formattedText
appState.lastInsertedText = formattedText
InputHistory.shared.replaceLatestRecord(
let recordID = InputHistory.shared.replaceLatestRecord(
rawText: replacement.rawText,
processedText: formattedText,
wasProcessed: true,
context: replacement.context
context: replacement.context,
formatKind: replacement.formatKind
)
beginCorrectionCapture(
recordID: recordID,
insertedText: formattedText,
context: replacement.context ?? InputContext(
appName: replacement.targetAppName,
bundleIdentifier: replacement.targetBundleIdentifier,
outputMode: .processed,
inputLanguage: appState.settings.inputLanguage,
source: .menuBar
)
)
appState.clearPendingReplacement()
appState.phase = .done
Expand Down Expand Up @@ -208,6 +230,7 @@ extension VoicePipeline {
screenImage: screenContext.image,
memoryContext: memoryContext,
inputContext: inputContext,
formatKind: currentReplacement.formatKind,
allowsPreparedFallback: false,
allowsGuardFallback: false,
dictionarySnapshot: dictionarySnapshot
Expand Down
7 changes: 6 additions & 1 deletion Sources/App/VoicePipeline+RewriteLast.swift
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,17 @@ extension VoicePipeline {
return
}

InputHistory.shared.addRecord(
let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: rewrittenText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = rewrittenText
beginCorrectionCapture(
recordID: recordID,
insertedText: rewrittenText,
context: context
)
}
}
3 changes: 3 additions & 0 deletions Sources/App/VoicePipeline.swift
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ final class VoicePipeline {
let soundPlayer = SoundPlayer()
let audioCapture = AudioCaptureManager()
let textInserter = TextInserter()
let correctionCapture = CorrectionCaptureService()
let textProcessor = TextProcessor()
let overlay = OverlayPanel()
var whisperEngine: WhisperEngine?
Expand Down Expand Up @@ -77,6 +78,8 @@ final class VoicePipeline {

if appState.isDownloading { return }

correctionCapture.finishCurrentSession()

if !(currentEngine?.isReady ?? false) {
await ensureEngineLoaded(requestPermission: true)
}
Expand Down
7 changes: 6 additions & 1 deletion Sources/Config/AppSettings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ final class AppSettings: ObservableObject {
@Published var historyRetention: HistoryRetention
@Published var enableMemory: Bool
@Published var memoryWindowMinutes: Int
@Published var enableCorrectionLearning: Bool
@Published var useCustomSystemPrompt: Bool
@Published var customSystemPrompt: String
@Published var useRemoteLLM: Bool
Expand Down Expand Up @@ -309,7 +310,7 @@ final class AppSettings: ObservableObject {
case enableStreamingRecognitionBeta
case inputLanguage, translationTargetLanguage
case useScreenContext, screenContextMode, enableInstantInsert, hasCompletedOnboarding, uiLanguage, historyRetention
case enableMemory, memoryWindowMinutes
case enableMemory, memoryWindowMinutes, enableCorrectionLearning
case useCustomSystemPrompt, customSystemPrompt
case useRemoteLLM, remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel
case menuBarIcon, appIconAppearance
Expand Down Expand Up @@ -373,6 +374,7 @@ final class AppSettings: ObservableObject {
historyRetention = HistoryRetention(rawValue: ud.string(forKey: Key.historyRetention.rawValue) ?? "") ?? .forever
enableMemory = ud.object(forKey: Key.enableMemory.rawValue) as? Bool ?? true
memoryWindowMinutes = (ud.integer(forKey: Key.memoryWindowMinutes.rawValue)).nonZeroInt ?? 30
enableCorrectionLearning = ud.object(forKey: Key.enableCorrectionLearning.rawValue) as? Bool ?? true
useCustomSystemPrompt = ud.bool(forKey: Key.useCustomSystemPrompt.rawValue)
customSystemPrompt = ud.string(forKey: Key.customSystemPrompt.rawValue) ?? ""
useRemoteLLM = ud.bool(forKey: Key.useRemoteLLM.rawValue)
Expand Down Expand Up @@ -439,6 +441,9 @@ final class AppSettings: ObservableObject {
$historyRetention.dropFirst().sink { [defaults] in defaults.set($0.rawValue, forKey: Key.historyRetention.rawValue) }.store(in: &cancellables)
$enableMemory.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.enableMemory.rawValue) }.store(in: &cancellables)
$memoryWindowMinutes.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.memoryWindowMinutes.rawValue) }.store(in: &cancellables)
$enableCorrectionLearning.dropFirst().sink {
[defaults] in defaults.set($0, forKey: Key.enableCorrectionLearning.rawValue)
}.store(in: &cancellables)
$useCustomSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useCustomSystemPrompt.rawValue) }.store(in: &cancellables)
$customSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.customSystemPrompt.rawValue) }.store(in: &cancellables)
$useRemoteLLM.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useRemoteLLM.rawValue) }.store(in: &cancellables)
Expand Down
9 changes: 8 additions & 1 deletion Sources/Integration/InputSessionCoordinator+Output.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ extension InputSessionCoordinator {
let memoryWindowMinutes = settings.memoryWindowMinutes
let text: String
let context: InputContext
let formatKind: TextFormatKind?

switch active.mode {
case .direct:
Expand All @@ -19,6 +20,7 @@ extension InputSessionCoordinator {
inputLanguage: active.inputLanguage,
dictionarySnapshot: dictionarySnapshot
)
formatKind = nil
case .processed:
let screenContext = await screenContext(from: active)
context = inputContext(for: active, screenContext: screenContext.text, mode: .processed)
Expand All @@ -28,13 +30,16 @@ extension InputSessionCoordinator {
memoryWindowMinutes: memoryWindowMinutes,
currentContext: context
)
let decision = TextFormatClassifier.classify(text: raw, context: context)
formatKind = decision.kind
text = await textProcessor.process(
text: raw,
options: options,
screenContext: screenContext.text,
screenImage: screenContext.image,
memoryContext: memoryContext,
inputContext: context,
formatKind: decision.kind,
dictionarySnapshot: dictionarySnapshot
)
case .command:
Expand All @@ -46,6 +51,7 @@ extension InputSessionCoordinator {
memoryWindowMinutes: memoryWindowMinutes,
currentContext: context
)
formatKind = nil
text = await textProcessor.processCommand(
text: raw,
options: options,
Expand All @@ -66,7 +72,8 @@ extension InputSessionCoordinator {
rawText: raw,
processedText: text,
wasProcessed: active.mode != .direct,
context: context
context: context,
formatKind: formatKind
)
return text
}
Expand Down
Loading
Loading