From 88c55e8d05fc3a0b7204d98d02d11bb810b8f961 Mon Sep 17 00:00:00 2001 From: savasturkoglu1 Date: Thu, 10 Sep 2026 15:54:16 +0300 Subject: [PATCH 1/2] feat(session): transcribe images for models without vision support When the selected model cannot read images and the prompt contains one, transcribe it with a vision-capable model from the same provider and inject the text into the request. The selected model and the visible timeline stay unchanged, and no error is shown. Add an optional experimental.vision_model override to pick the transcription model explicitly. Cross-provider fallback is intentionally not implemented; local OCR is tracked separately. --- packages/core/src/v1/config/config.ts | 4 + packages/opencode/src/session/prompt.ts | 6 +- packages/opencode/src/session/vision.ts | 168 +++++++++++++++ packages/opencode/test/session/vision.test.ts | 194 ++++++++++++++++++ 4 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 packages/opencode/src/session/vision.ts create mode 100644 packages/opencode/test/session/vision.test.ts diff --git a/packages/core/src/v1/config/config.ts b/packages/core/src/v1/config/config.ts index 7ebb4b69b023..8d1d9675de08 100644 --- a/packages/core/src/v1/config/config.ts +++ b/packages/core/src/v1/config/config.ts @@ -182,6 +182,10 @@ export const Info = Schema.Struct({ mcp_timeout: Schema.optional(PositiveInt).annotate({ description: "Timeout in milliseconds for model context protocol (MCP) requests", }), + vision_model: Schema.optional(Schema.String).annotate({ + description: + "Model used to transcribe images when the selected model does not support image input, e.g. provider/model", + }), policies: Schema.optional(Schema.mutable(Schema.Array(ConfigExperimental.Policy))).annotate({ description: "Policy statements applied to supported resources, such as provider access", }), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 0f85d44f209b..df0242b387e0 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -55,6 +55,7 @@ import { eq } from "drizzle-orm" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionReminders } from "./reminders" import { SessionTools } from "./tools" +import { SessionVision } from "./vision" import { LLMEvent } from "@opencode-ai/llm" // @ts-ignore @@ -137,6 +138,7 @@ const layer = Layer.effect( const summary = yield* SessionSummary.Service const sys = yield* SystemPrompt.Service const llm = yield* LLM.Service + const vision = yield* SessionVision.Service const events = yield* EventV2Bridge.Service const flags = yield* RuntimeFlags.Service const database = yield* Database.Service @@ -1254,12 +1256,13 @@ const layer = Layer.effect( yield* plugin.trigger("experimental.chat.messages.transform", {}, { messages: msgs }) + const bridged = yield* vision.bridge({ messages: msgs, model, user: lastUser }) const [skills, env, instructions, mcpInstructions, modelMsgs] = yield* Effect.all([ sys.skills(agent), sys.environment(model), instruction.system().pipe(Effect.orDie), sys.mcp(agent, session.permission), - MessageV2.toModelMessagesEffect(msgs, model), + MessageV2.toModelMessagesEffect(bridged, model), ]) const system = [ ...env, @@ -1622,6 +1625,7 @@ export const node = LayerNode.make({ SessionSummary.node, SystemPrompt.node, LLM.node, + SessionVision.node, EventV2Bridge.node, RuntimeFlags.node, Database.node, diff --git a/packages/opencode/src/session/vision.ts b/packages/opencode/src/session/vision.ts new file mode 100644 index 000000000000..b0d0da9d4e91 --- /dev/null +++ b/packages/opencode/src/session/vision.ts @@ -0,0 +1,168 @@ +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { Context, Effect, Layer } from "effect" +import * as Stream from "effect/Stream" +import { convertToModelMessages, type UIMessage } from "ai" +import { LLMEvent } from "@opencode-ai/llm" +import { Provider } from "@/provider/provider" +import { Config } from "@/config/config" +import { Agent } from "@/agent/agent" +import { isImageAttachment } from "@/util/media" +import { LLM } from "./llm" + +const TRANSCRIBE_INSTRUCTION = `Transcribe every piece of text visible in this image, preserving order and structure. +Include code, error messages, terminal output, and UI labels verbatim. +Output only the transcription, with no commentary or summary.` + +const TRANSCRIBE_AGENT: Agent.Info = { + name: "vision", + mode: "primary", + permission: [], + options: {}, + prompt: "You convert images into faithful text. Never summarize or omit content.", +} + +function supportsVision(model: Provider.Model) { + return model.capabilities.input.image +} + +function imageParts(messages: SessionV1.WithParts[]) { + return messages.flatMap((message) => + message.info.role === "user" + ? message.parts.filter((part): part is SessionV1.FilePart => part.type === "file" && isImageAttachment(part.mime)) + : [], + ) +} + +export interface Interface { + readonly bridge: (input: { + messages: SessionV1.WithParts[] + model: Provider.Model + user: SessionV1.User + }) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/SessionVision") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const provider = yield* Provider.Service + const config = yield* Config.Service + const llm = yield* LLM.Service + // Transcripts are keyed by part id so a multi-step session only pays for + // each image once. Part ids are globally unique, so one map is safe here. + const cache = new Map() + + const transcribe = Effect.fnUntraced(function* (input: { + part: SessionV1.FilePart + model: Provider.Model + user: SessionV1.User + }) { + const cached = cache.get(input.part.id) + if (cached !== undefined) return cached + const message: Omit = { + role: "user", + parts: [ + { type: "text", text: TRANSCRIBE_INSTRUCTION }, + { type: "file", mediaType: input.part.mime, filename: input.part.filename, url: input.part.url }, + ], + } + const messages = yield* Effect.promise(() => convertToModelMessages([message])) + const text = yield* llm + .stream({ + user: input.user, + sessionID: input.user.sessionID, + model: input.model, + agent: TRANSCRIBE_AGENT, + system: [], + small: true, + tools: {}, + retries: 1, + messages, + }) + .pipe( + Stream.filter(LLMEvent.is.textDelta), + Stream.map((event) => event.text), + Stream.mkString, + ) + .pipe(Effect.catch(() => Effect.succeed(""))) + const result = text.trim() + if (result) cache.set(input.part.id, result) + return result + }) + + const selectModel = Effect.fnUntraced(function* (current: Provider.Model) { + const override = (yield* config.get()).experimental?.vision_model + if (override) { + const parsed = Provider.parseModel(override) + const model = yield* provider + .getModel(parsed.providerID, parsed.modelID) + .pipe(Effect.catch(() => Effect.succeed(undefined))) + if (model && supportsVision(model)) return model + } + + const providers = yield* provider.list() + const candidates = Object.values(providers).flatMap((info) => + Object.values(info.models).filter((model) => model.status === "active" && supportsVision(model)), + ) + // Only the selected model's own provider is used. Sending the user's image + // to a different provider would be surprising, so OCR (opt-in) is the only + // cross-provider-safe fallback. + return candidates + .filter((model) => model.providerID === current.providerID && model.id !== current.id) + .toSorted((a, b) => b.release_date.localeCompare(a.release_date))[0] + }) + + const bridge = Effect.fn("SessionVision.bridge")(function* (input: { + messages: SessionV1.WithParts[] + model: Provider.Model + user: SessionV1.User + }) { + if (supportsVision(input.model)) return input.messages + const parts = imageParts(input.messages) + if (parts.length === 0) return input.messages + + const visionModel = yield* selectModel(input.model) + if (!visionModel) return input.messages + + const transcripts = yield* Effect.forEach( + parts, + (part) => + transcribe({ part, model: visionModel, user: input.user }).pipe( + Effect.map((text) => ({ id: part.id, text })), + ), + { concurrency: 2 }, + ) + const byID = new Map(transcripts.filter((item) => item.text).map((item) => [item.id, item.text])) + if (byID.size === 0) return input.messages + + return input.messages.map((message) => ({ + info: message.info, + parts: message.parts.map((part): SessionV1.Part => { + if (part.type !== "file") return part + const text = byID.get(part.id) + if (!text) return part + return { + id: part.id, + sessionID: part.sessionID, + messageID: part.messageID, + type: "text", + text: `[Image${part.filename ? `: ${part.filename}` : ""}]\n${text}`, + synthetic: true, + } + }), + })) + }) + + return Service.of({ bridge }) + }), +) + +export const node = LayerNode.make({ + service: Service, + layer, + deps: [Provider.node, Config.node, LLM.node], +}) + +export * as SessionVision from "./vision" diff --git a/packages/opencode/test/session/vision.test.ts b/packages/opencode/test/session/vision.test.ts new file mode 100644 index 000000000000..416b2311b654 --- /dev/null +++ b/packages/opencode/test/session/vision.test.ts @@ -0,0 +1,194 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer, Stream } from "effect" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { ModelV2 } from "@opencode-ai/core/model" +import { LLMEvent } from "@opencode-ai/llm" +import { Provider } from "@/provider/provider" +import { LLM } from "@/session/llm" +import { SessionVision } from "@/session/vision" +import { MessageID, PartID, SessionID } from "@/session/schema" +import { testEffect } from "../lib/effect" +import { TestConfig } from "../fixture/config" + +const sessionID = SessionID.make("session-vision") +const messageID = MessageID.make("msg_vision") + +function model(input: { + providerID: string + id: string + image?: boolean + status?: "alpha" | "beta" | "deprecated" | "active" + cost?: number + release?: string +}): Provider.Model { + return { + id: ModelV2.ID.make(input.id), + providerID: ProviderV2.ID.make(input.providerID), + api: { id: input.id, url: "https://example.com", npm: "@ai-sdk/openai-compatible" }, + name: input.id, + capabilities: { + temperature: false, + reasoning: false, + attachment: input.image ?? false, + toolcall: true, + input: { text: true, audio: false, image: input.image ?? false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + cost: { input: input.cost ?? 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 100000, output: 10000 }, + status: input.status ?? "active", + options: {}, + headers: {}, + release_date: input.release ?? "2025-01-01", + } +} + +function info(id: string, models: Provider.Model[]): Provider.Info { + return { + id: ProviderV2.ID.make(id), + name: id, + source: "config", + env: [], + options: {}, + models: Object.fromEntries(models.map((item) => [item.id, item])), + } +} + +const textModel = model({ providerID: "test", id: "text-only" }) +const visionModel = model({ providerID: "test", id: "vision", image: true, release: "2025-06-01" }) +const otherVision = model({ providerID: "other", id: "vision", image: true, release: "2025-06-01" }) + +function user(): SessionV1.User { + return { + id: messageID, + sessionID, + role: "user", + time: { created: 0 }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("text-only") }, + } +} + +function imagePart(id: string): SessionV1.FilePart { + return { + id: PartID.make(id), + sessionID, + messageID, + type: "file", + mime: "image/png", + filename: "shot.png", + url: "data:image/png;base64,AAAA", + } +} + +function textPart(): SessionV1.TextPart { + return { id: PartID.make("prt_text"), sessionID, messageID, type: "text", text: "look at this" } +} + +function messages(parts: SessionV1.Part[]): SessionV1.WithParts[] { + return [{ info: user(), parts }] +} + +function env(input: { providers: Provider.Info[]; transcription?: string }) { + const calls = { count: 0 } + const provider = Layer.mock(Provider.Service, { + list: () => Effect.succeed(Object.fromEntries(input.providers.map((item) => [item.id, item]))), + getModel: (providerID, modelID) => + Effect.sync(() => { + const found = input.providers + .flatMap((item) => Object.values(item.models)) + .find((item) => item.providerID === providerID && item.id === modelID) + if (!found) throw new Error("model not found") + return found + }), + }) + const config = TestConfig.layer() + const llm = Layer.mock(LLM.Service, { + stream: () => { + calls.count++ + return Stream.make(LLMEvent.textDelta({ id: "text-0", text: input.transcription ?? "transcribed text" })) + }, + }) + return { + calls, + layer: SessionVision.layer.pipe(Layer.provide(Layer.mergeAll(provider, config, llm))), + } +} + +const available = env({ providers: [info("test", [textModel, visionModel])] }) +const visionCapable = env({ providers: [info("test", [textModel, visionModel])] }) +const onlyText = env({ providers: [info("test", [textModel])] }) +const cached = env({ providers: [info("test", [textModel, visionModel])] }) +const crossProvider = env({ providers: [info("test", [textModel]), info("other", [otherVision])] }) + +const itAvailable = testEffect(available.layer) +const itVisionCapable = testEffect(visionCapable.layer) +const itOnlyText = testEffect(onlyText.layer) +const itCached = testEffect(cached.layer) +const itCrossProvider = testEffect(crossProvider.layer) + +describe("SessionVision.bridge", () => { + itAvailable.effect("transcribes images for models without vision support", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([textPart(), imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).not.toBe(input) + expect(output[0]?.parts).toHaveLength(2) + expect(output[0]?.parts[1]).toMatchObject({ type: "text", synthetic: true }) + const part = output[0]?.parts[1] + const text = part && part.type === "text" ? part.text : "" + expect(text).toContain("transcribed text") + expect(text).toContain("shot.png") + expect(input[0]?.parts[1]?.type).toBe("file") + expect(available.calls.count).toBe(1) + }), + ) + + itVisionCapable.effect("skips transcription when the model already supports vision", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: visionModel, user: user() }) + + expect(output).toBe(input) + expect(visionCapable.calls.count).toBe(0) + }), + ) + + itOnlyText.effect("leaves messages untouched when no vision model is available", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).toBe(input) + expect(onlyText.calls.count).toBe(0) + }), + ) + + itCached.effect("caches transcripts per image part", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + yield* vision.bridge({ messages: input, model: textModel, user: user() }) + yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(cached.calls.count).toBe(1) + }), + ) + + itCrossProvider.effect("never uses another provider's vision model", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).toBe(input) + expect(crossProvider.calls.count).toBe(0) + }), + ) +}) From 9bfe2cf3da4af77b748e7aaf935f78fc44eb24b7 Mon Sep 17 00:00:00 2001 From: savasturkoglu1 Date: Thu, 10 Sep 2026 16:21:40 +0300 Subject: [PATCH 2/2] feat(session): describe images with little or no text The transcription prompt only asked for visible text, so design-only images produced nothing useful. Ask the vision model to describe visual content when there is little or no text. --- packages/opencode/src/session/vision.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/session/vision.ts b/packages/opencode/src/session/vision.ts index b0d0da9d4e91..0e7717c5936a 100644 --- a/packages/opencode/src/session/vision.ts +++ b/packages/opencode/src/session/vision.ts @@ -10,16 +10,16 @@ import { Agent } from "@/agent/agent" import { isImageAttachment } from "@/util/media" import { LLM } from "./llm" -const TRANSCRIBE_INSTRUCTION = `Transcribe every piece of text visible in this image, preserving order and structure. -Include code, error messages, terminal output, and UI labels verbatim. -Output only the transcription, with no commentary or summary.` +const TRANSCRIBE_INSTRUCTION = `Describe this image for a developer who cannot see it. +Transcribe all visible text verbatim, including code, error messages, terminal output, and UI labels. +If there is little or no text, describe the visual content instead: layout, components, colors, and anything a developer would need to understand it.` const TRANSCRIBE_AGENT: Agent.Info = { name: "vision", mode: "primary", permission: [], options: {}, - prompt: "You convert images into faithful text. Never summarize or omit content.", + prompt: "You convert images into faithful text and descriptions. Never summarize away content.", } function supportsVision(model: Provider.Model) {