Skip to content

Stabilize prompt prefix: move volatile context to the user turn #118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IchenDEV merged 1 commit into from
Sep 30, 2026
Merged

Stabilize prompt prefix: move volatile context to the user turn #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 14 additions & 9 deletions Sources/Processing/TextProcessor+EditCommandResolution.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,21 @@ extension TextProcessor {

do {
let generationOptions = editCommandResolutionOptions(for: transcript)
let baseUserPrompt = PromptBuilder.buildEditCommandResolverUserPrompt(
text: transcript,
inputLanguage: options.inputLanguage,
context: context
)
let personal = personalContextSections(
inputLanguage: options.inputLanguage,
transcript: transcript
)
let userPrompt = personal.isEmpty
? baseUserPrompt
: personal.joined(separator: "\n\n") + "\n\n" + baseUserPrompt
let result = try await generateText(
prompt: PromptBuilder.buildEditCommandResolverUserPrompt(
text: transcript,
inputLanguage: options.inputLanguage,
context: context
),
systemPrompt: systemPromptWithPersonalContext(
PromptBuilder.buildEditCommandResolverSystemPrompt(
inputLanguage: options.inputLanguage
),
prompt: userPrompt,
systemPrompt: PromptBuilder.buildEditCommandResolverSystemPrompt(
inputLanguage: options.inputLanguage
),
options: options,
Expand Down
91 changes: 88 additions & 3 deletions Sources/Processing/TextProcessor+PromptConstruction.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,22 @@ extension TextProcessor {
dictionarySnapshot: PersonalDictionarySnapshot? = nil,
transcript: String = ""
) -> String {
let extraSections = personalContextSections(
inputLanguage: inputLanguage,
dictionarySnapshot: dictionarySnapshot,
transcript: transcript
)
guard !extraSections.isEmpty else { return systemPrompt }
return ([systemPrompt] + extraSections).joined(separator: "\n\n")
}

func personalContextSections(
inputLanguage: InputLanguage,
dictionarySnapshot: PersonalDictionarySnapshot? = nil,
transcript: String = ""
) -> [String] {
let snapshot = dictionarySnapshot ?? PersonalDictionary.shared.snapshot(settings: .shared)
let extraSections = [
return [
PromptCatalog.activeIndustryLexiconSection(
snapshot.industryLexicon.promptDescription(matching: transcript),
industry: snapshot.industryLexicon.pack?.id,
Expand All @@ -93,8 +107,79 @@ extension TextProcessor {
inputLanguage: inputLanguage
),
].compactMap { $0 }
}

guard !extraSections.isEmpty else { return systemPrompt }
return ([systemPrompt] + extraSections).joined(separator: "\n\n")
/// Cacheable formatting assembly: only settings-derived instructions stay in
/// `stablePrefix`; processing context and personal context become the
/// volatile tail that travels in the user turn.
func formattingAssembly(
options: TextProcessingOptions,
screenContext: String,
screenImageAvailable: Bool,
memoryContext: String,
inputContext: InputContext?,
formatKind: TextFormatKind? = nil,
dictionarySnapshot: PersonalDictionarySnapshot? = nil,
transcript: String = ""
) -> PromptAssembly {
let base = PromptBuilder.buildFormattingAssembly(
style: options.languageStyle,
stylePrompt: options.customStylePrompt,
screenContext: screenContext,
screenImageAvailable: screenImageAvailable,
memoryContext: memoryContext,
inputContext: inputContext,
formatKind: formatKind,
inputLanguage: options.inputLanguage,
useCustomSystemPrompt: options.useCustomSystemPrompt,
customSystemPrompt: options.customSystemPrompt
)
return augmentingVolatileContext(
base,
inputLanguage: options.inputLanguage,
dictionarySnapshot: dictionarySnapshot,
transcript: transcript
)
}

func commandAssembly(
options: TextProcessingOptions,
screenContext: String,
screenImageAvailable: Bool,
memoryContext: String,
inputContext: InputContext?,
dictionarySnapshot: PersonalDictionarySnapshot? = nil,
transcript: String = ""
) -> PromptAssembly {
let base = PromptBuilder.buildCommandAssembly(
screenContext: screenContext,
screenImageAvailable: screenImageAvailable,
memoryContext: memoryContext,
inputContext: inputContext,
inputLanguage: options.inputLanguage
)
return augmentingVolatileContext(
base,
inputLanguage: options.inputLanguage,
dictionarySnapshot: dictionarySnapshot,
transcript: transcript
)
}

private func augmentingVolatileContext(
_ assembly: PromptAssembly,
inputLanguage: InputLanguage,
dictionarySnapshot: PersonalDictionarySnapshot?,
transcript: String
) -> PromptAssembly {
let personal = personalContextSections(
inputLanguage: inputLanguage,
dictionarySnapshot: dictionarySnapshot,
transcript: transcript
)
let volatile = ([assembly.volatileContext] + personal)
.filter { !$0.isEmpty }
.joined(separator: "\n\n")
return PromptAssembly(stablePrefix: assembly.stablePrefix, volatileContext: volatile)
}
}
26 changes: 17 additions & 9 deletions Sources/Processing/TextProcessor+SelectionEdit.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,24 @@ extension TextProcessor {
let generationOptions = selectionEditOptions(for: trimmedSelection, intent: intent)

do {
let baseUserPrompt = selectionEditPrompt(
selectedText: trimmedSelection,
intent: intent,
inputLanguage: options.inputLanguage,
spokenCommand: spokenCommand,
memoryContext: memoryContext,
inputContext: inputContext
)
let personal = personalContextSections(
inputLanguage: options.inputLanguage,
transcript: trimmedSelection
)
let userPrompt = personal.isEmpty
? baseUserPrompt
: personal.joined(separator: "\n\n") + "\n\n" + baseUserPrompt
let result = try await generateText(
prompt: selectionEditPrompt(
selectedText: trimmedSelection,
intent: intent,
inputLanguage: options.inputLanguage,
spokenCommand: spokenCommand,
memoryContext: memoryContext,
inputContext: inputContext
),
systemPrompt: selectionEditSystemPromptWithPersonalContext(inputLanguage: options.inputLanguage),
prompt: userPrompt,
systemPrompt: selectionEditSystemPrompt(inputLanguage: options.inputLanguage),
options: options,
maxTokens: generationOptions.maxTokens,
temperature: generationOptions.temperature
Expand Down
42 changes: 22 additions & 20 deletions Sources/Processing/TextProcessor.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,11 @@ final class TextProcessor {
guard !cleanedText.isEmpty else { return "" }

let useScreenImage = shouldUseScreenImage(options: options, image: screenImage)
let systemPrompt = formattingSystemPrompt(
let baseUserPrompt = formattingUserPrompt(
text: cleanedText,
options: options
)
let assembly = formattingAssembly(
options: options,
screenContext: screenContext,
screenImageAvailable: useScreenImage,
Expand All @@ -97,11 +101,8 @@ final class TextProcessor {
dictionarySnapshot: dictionarySnapshot,
transcript: cleanedText
)
let userPrompt = assembly.userPrompt(containing: baseUserPrompt)

let userPrompt = formattingUserPrompt(
text: cleanedText,
options: options
)
let generationOptions = formattingOptions(for: cleanedText, style: options.languageStyle)

do {
Expand All @@ -112,15 +113,15 @@ final class TextProcessor {
do {
return try await generateWithScreenImage(
prompt: userPrompt,
systemPrompt: systemPrompt,
systemPrompt: assembly.stablePrefix,
model: options.llmModel,
image: screenImage,
maxTokens: generationOptions.maxTokens,
temperature: generationOptions.temperature
)
} catch {
Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)")
let textFallbackSystemPrompt = formattingSystemPrompt(
let textFallback = formattingAssembly(
options: options,
screenContext: screenContext,
screenImageAvailable: false,
Expand All @@ -131,8 +132,8 @@ final class TextProcessor {
transcript: cleanedText
)
return try await generateText(
prompt: userPrompt,
systemPrompt: textFallbackSystemPrompt,
prompt: textFallback.userPrompt(containing: baseUserPrompt),
systemPrompt: textFallback.stablePrefix,
options: options,
maxTokens: generationOptions.maxTokens,
temperature: generationOptions.temperature
Expand All @@ -142,7 +143,7 @@ final class TextProcessor {
} else {
result = try await generateText(
prompt: userPrompt,
systemPrompt: systemPrompt,
systemPrompt: assembly.stablePrefix,
options: options,
maxTokens: generationOptions.maxTokens,
temperature: generationOptions.temperature
Expand Down Expand Up @@ -206,7 +207,11 @@ final class TextProcessor {
) async -> String {
let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot(settings: .shared)
let useScreenImage = shouldUseScreenImage(options: options, image: screenImage)
let systemPrompt = commandSystemPrompt(
let baseUserPrompt = PromptBuilder.buildCommandUserPrompt(
text: text,
inputLanguage: options.inputLanguage
)
let assembly = commandAssembly(
options: options,
screenContext: screenContext,
screenImageAvailable: useScreenImage,
Expand All @@ -215,10 +220,7 @@ final class TextProcessor {
dictionarySnapshot: dictionarySnapshot,
transcript: text
)
let userPrompt = PromptBuilder.buildCommandUserPrompt(
text: text,
inputLanguage: options.inputLanguage
)
let userPrompt = assembly.userPrompt(containing: baseUserPrompt)

do {
var result: String
Expand All @@ -227,15 +229,15 @@ final class TextProcessor {
do {
return try await generateWithScreenImage(
prompt: userPrompt,
systemPrompt: systemPrompt,
systemPrompt: assembly.stablePrefix,
model: options.llmModel,
image: screenImage,
maxTokens: 4096,
temperature: 0.3
)
} catch {
Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)")
let textFallbackSystemPrompt = commandSystemPrompt(
let textFallback = commandAssembly(
options: options,
screenContext: screenContext,
screenImageAvailable: false,
Expand All @@ -245,8 +247,8 @@ final class TextProcessor {
transcript: text
)
return try await generateText(
prompt: userPrompt,
systemPrompt: textFallbackSystemPrompt,
prompt: textFallback.userPrompt(containing: baseUserPrompt),
systemPrompt: textFallback.stablePrefix,
options: options,
maxTokens: 4096,
temperature: 0.3
Expand All @@ -256,7 +258,7 @@ final class TextProcessor {
} else {
result = try await generateText(
prompt: userPrompt,
systemPrompt: systemPrompt,
systemPrompt: assembly.stablePrefix,
options: options,
maxTokens: 4096,
temperature: 0.3
Expand Down
30 changes: 30 additions & 0 deletions Sources/Prompts/PromptAssembly.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/// Splits an assembled LLM prompt into a byte-stable, cacheable instruction
/// prefix and the per-request context that follows it.
///
/// `stablePrefix` depends only on settings (language, style, format kind,
/// custom prompt, main vs command path) and is identical across requests.
/// `volatileContext` carries screen, memory, input-target, time, and personal
/// dictionary context; keeping it out of the prefix lets a prompt/KV cache key
/// on `(modelID, stablePrefix)` and lets provider-side prefix caching engage.
struct PromptAssembly: Sendable, Equatable {
let stablePrefix: String
let volatileContext: String

init(stablePrefix: String, volatileContext: String = "") {
self.stablePrefix = stablePrefix
self.volatileContext = volatileContext
}

/// The legacy single-string system prompt: stable prefix then volatile
/// context. Prefer passing `stablePrefix` as the system prompt and
/// `userPrompt(containing:)` as the user turn for cacheable requests.
var systemPrompt: String {
volatileContext.isEmpty ? stablePrefix : "\(stablePrefix)\n\n\(volatileContext)"
}

/// The user turn: volatile context (when present) ahead of the payload,
/// preserving the original content order while keeping it out of the prefix.
func userPrompt(containing content: String) -> String {
volatileContext.isEmpty ? content : "\(volatileContext)\n\n\(content)"
}
}
Loading
Loading