diff --git a/packages/core/src/v1/config/config.ts b/packages/core/src/v1/config/config.ts index 7ebb4b69b023..8d1d9675de08 100644 --- a/packages/core/src/v1/config/config.ts +++ b/packages/core/src/v1/config/config.ts @@ -182,6 +182,10 @@ export const Info = Schema.Struct({ mcp_timeout: Schema.optional(PositiveInt).annotate({ description: "Timeout in milliseconds for model context protocol (MCP) requests", }), + vision_model: Schema.optional(Schema.String).annotate({ + description: + "Model used to transcribe images when the selected model does not support image input, e.g. provider/model", + }), policies: Schema.optional(Schema.mutable(Schema.Array(ConfigExperimental.Policy))).annotate({ description: "Policy statements applied to supported resources, such as provider access", }), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 0f85d44f209b..df0242b387e0 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -55,6 +55,7 @@ import { eq } from "drizzle-orm" import { SessionTable } from "@opencode-ai/core/session/sql" import { SessionReminders } from "./reminders" import { SessionTools } from "./tools" +import { SessionVision } from "./vision" import { LLMEvent } from "@opencode-ai/llm" // @ts-ignore @@ -137,6 +138,7 @@ const layer = Layer.effect( const summary = yield* SessionSummary.Service const sys = yield* SystemPrompt.Service const llm = yield* LLM.Service + const vision = yield* SessionVision.Service const events = yield* EventV2Bridge.Service const flags = yield* RuntimeFlags.Service const database = yield* Database.Service @@ -1254,12 +1256,13 @@ const layer = Layer.effect( yield* plugin.trigger("experimental.chat.messages.transform", {}, { messages: msgs }) + const bridged = yield* vision.bridge({ messages: msgs, model, user: lastUser }) const [skills, env, instructions, mcpInstructions, modelMsgs] = yield* Effect.all([ sys.skills(agent), sys.environment(model), instruction.system().pipe(Effect.orDie), sys.mcp(agent, session.permission), - MessageV2.toModelMessagesEffect(msgs, model), + MessageV2.toModelMessagesEffect(bridged, model), ]) const system = [ ...env, @@ -1622,6 +1625,7 @@ export const node = LayerNode.make({ SessionSummary.node, SystemPrompt.node, LLM.node, + SessionVision.node, EventV2Bridge.node, RuntimeFlags.node, Database.node, diff --git a/packages/opencode/src/session/vision.ts b/packages/opencode/src/session/vision.ts new file mode 100644 index 000000000000..0e7717c5936a --- /dev/null +++ b/packages/opencode/src/session/vision.ts @@ -0,0 +1,168 @@ +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { Context, Effect, Layer } from "effect" +import * as Stream from "effect/Stream" +import { convertToModelMessages, type UIMessage } from "ai" +import { LLMEvent } from "@opencode-ai/llm" +import { Provider } from "@/provider/provider" +import { Config } from "@/config/config" +import { Agent } from "@/agent/agent" +import { isImageAttachment } from "@/util/media" +import { LLM } from "./llm" + +const TRANSCRIBE_INSTRUCTION = `Describe this image for a developer who cannot see it. +Transcribe all visible text verbatim, including code, error messages, terminal output, and UI labels. +If there is little or no text, describe the visual content instead: layout, components, colors, and anything a developer would need to understand it.` + +const TRANSCRIBE_AGENT: Agent.Info = { + name: "vision", + mode: "primary", + permission: [], + options: {}, + prompt: "You convert images into faithful text and descriptions. Never summarize away content.", +} + +function supportsVision(model: Provider.Model) { + return model.capabilities.input.image +} + +function imageParts(messages: SessionV1.WithParts[]) { + return messages.flatMap((message) => + message.info.role === "user" + ? message.parts.filter((part): part is SessionV1.FilePart => part.type === "file" && isImageAttachment(part.mime)) + : [], + ) +} + +export interface Interface { + readonly bridge: (input: { + messages: SessionV1.WithParts[] + model: Provider.Model + user: SessionV1.User + }) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/SessionVision") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const provider = yield* Provider.Service + const config = yield* Config.Service + const llm = yield* LLM.Service + // Transcripts are keyed by part id so a multi-step session only pays for + // each image once. Part ids are globally unique, so one map is safe here. + const cache = new Map() + + const transcribe = Effect.fnUntraced(function* (input: { + part: SessionV1.FilePart + model: Provider.Model + user: SessionV1.User + }) { + const cached = cache.get(input.part.id) + if (cached !== undefined) return cached + const message: Omit = { + role: "user", + parts: [ + { type: "text", text: TRANSCRIBE_INSTRUCTION }, + { type: "file", mediaType: input.part.mime, filename: input.part.filename, url: input.part.url }, + ], + } + const messages = yield* Effect.promise(() => convertToModelMessages([message])) + const text = yield* llm + .stream({ + user: input.user, + sessionID: input.user.sessionID, + model: input.model, + agent: TRANSCRIBE_AGENT, + system: [], + small: true, + tools: {}, + retries: 1, + messages, + }) + .pipe( + Stream.filter(LLMEvent.is.textDelta), + Stream.map((event) => event.text), + Stream.mkString, + ) + .pipe(Effect.catch(() => Effect.succeed(""))) + const result = text.trim() + if (result) cache.set(input.part.id, result) + return result + }) + + const selectModel = Effect.fnUntraced(function* (current: Provider.Model) { + const override = (yield* config.get()).experimental?.vision_model + if (override) { + const parsed = Provider.parseModel(override) + const model = yield* provider + .getModel(parsed.providerID, parsed.modelID) + .pipe(Effect.catch(() => Effect.succeed(undefined))) + if (model && supportsVision(model)) return model + } + + const providers = yield* provider.list() + const candidates = Object.values(providers).flatMap((info) => + Object.values(info.models).filter((model) => model.status === "active" && supportsVision(model)), + ) + // Only the selected model's own provider is used. Sending the user's image + // to a different provider would be surprising, so OCR (opt-in) is the only + // cross-provider-safe fallback. + return candidates + .filter((model) => model.providerID === current.providerID && model.id !== current.id) + .toSorted((a, b) => b.release_date.localeCompare(a.release_date))[0] + }) + + const bridge = Effect.fn("SessionVision.bridge")(function* (input: { + messages: SessionV1.WithParts[] + model: Provider.Model + user: SessionV1.User + }) { + if (supportsVision(input.model)) return input.messages + const parts = imageParts(input.messages) + if (parts.length === 0) return input.messages + + const visionModel = yield* selectModel(input.model) + if (!visionModel) return input.messages + + const transcripts = yield* Effect.forEach( + parts, + (part) => + transcribe({ part, model: visionModel, user: input.user }).pipe( + Effect.map((text) => ({ id: part.id, text })), + ), + { concurrency: 2 }, + ) + const byID = new Map(transcripts.filter((item) => item.text).map((item) => [item.id, item.text])) + if (byID.size === 0) return input.messages + + return input.messages.map((message) => ({ + info: message.info, + parts: message.parts.map((part): SessionV1.Part => { + if (part.type !== "file") return part + const text = byID.get(part.id) + if (!text) return part + return { + id: part.id, + sessionID: part.sessionID, + messageID: part.messageID, + type: "text", + text: `[Image${part.filename ? `: ${part.filename}` : ""}]\n${text}`, + synthetic: true, + } + }), + })) + }) + + return Service.of({ bridge }) + }), +) + +export const node = LayerNode.make({ + service: Service, + layer, + deps: [Provider.node, Config.node, LLM.node], +}) + +export * as SessionVision from "./vision" diff --git a/packages/opencode/test/session/vision.test.ts b/packages/opencode/test/session/vision.test.ts new file mode 100644 index 000000000000..416b2311b654 --- /dev/null +++ b/packages/opencode/test/session/vision.test.ts @@ -0,0 +1,194 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer, Stream } from "effect" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { ModelV2 } from "@opencode-ai/core/model" +import { LLMEvent } from "@opencode-ai/llm" +import { Provider } from "@/provider/provider" +import { LLM } from "@/session/llm" +import { SessionVision } from "@/session/vision" +import { MessageID, PartID, SessionID } from "@/session/schema" +import { testEffect } from "../lib/effect" +import { TestConfig } from "../fixture/config" + +const sessionID = SessionID.make("session-vision") +const messageID = MessageID.make("msg_vision") + +function model(input: { + providerID: string + id: string + image?: boolean + status?: "alpha" | "beta" | "deprecated" | "active" + cost?: number + release?: string +}): Provider.Model { + return { + id: ModelV2.ID.make(input.id), + providerID: ProviderV2.ID.make(input.providerID), + api: { id: input.id, url: "https://example.com", npm: "@ai-sdk/openai-compatible" }, + name: input.id, + capabilities: { + temperature: false, + reasoning: false, + attachment: input.image ?? false, + toolcall: true, + input: { text: true, audio: false, image: input.image ?? false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + cost: { input: input.cost ?? 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 100000, output: 10000 }, + status: input.status ?? "active", + options: {}, + headers: {}, + release_date: input.release ?? "2025-01-01", + } +} + +function info(id: string, models: Provider.Model[]): Provider.Info { + return { + id: ProviderV2.ID.make(id), + name: id, + source: "config", + env: [], + options: {}, + models: Object.fromEntries(models.map((item) => [item.id, item])), + } +} + +const textModel = model({ providerID: "test", id: "text-only" }) +const visionModel = model({ providerID: "test", id: "vision", image: true, release: "2025-06-01" }) +const otherVision = model({ providerID: "other", id: "vision", image: true, release: "2025-06-01" }) + +function user(): SessionV1.User { + return { + id: messageID, + sessionID, + role: "user", + time: { created: 0 }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("text-only") }, + } +} + +function imagePart(id: string): SessionV1.FilePart { + return { + id: PartID.make(id), + sessionID, + messageID, + type: "file", + mime: "image/png", + filename: "shot.png", + url: "data:image/png;base64,AAAA", + } +} + +function textPart(): SessionV1.TextPart { + return { id: PartID.make("prt_text"), sessionID, messageID, type: "text", text: "look at this" } +} + +function messages(parts: SessionV1.Part[]): SessionV1.WithParts[] { + return [{ info: user(), parts }] +} + +function env(input: { providers: Provider.Info[]; transcription?: string }) { + const calls = { count: 0 } + const provider = Layer.mock(Provider.Service, { + list: () => Effect.succeed(Object.fromEntries(input.providers.map((item) => [item.id, item]))), + getModel: (providerID, modelID) => + Effect.sync(() => { + const found = input.providers + .flatMap((item) => Object.values(item.models)) + .find((item) => item.providerID === providerID && item.id === modelID) + if (!found) throw new Error("model not found") + return found + }), + }) + const config = TestConfig.layer() + const llm = Layer.mock(LLM.Service, { + stream: () => { + calls.count++ + return Stream.make(LLMEvent.textDelta({ id: "text-0", text: input.transcription ?? "transcribed text" })) + }, + }) + return { + calls, + layer: SessionVision.layer.pipe(Layer.provide(Layer.mergeAll(provider, config, llm))), + } +} + +const available = env({ providers: [info("test", [textModel, visionModel])] }) +const visionCapable = env({ providers: [info("test", [textModel, visionModel])] }) +const onlyText = env({ providers: [info("test", [textModel])] }) +const cached = env({ providers: [info("test", [textModel, visionModel])] }) +const crossProvider = env({ providers: [info("test", [textModel]), info("other", [otherVision])] }) + +const itAvailable = testEffect(available.layer) +const itVisionCapable = testEffect(visionCapable.layer) +const itOnlyText = testEffect(onlyText.layer) +const itCached = testEffect(cached.layer) +const itCrossProvider = testEffect(crossProvider.layer) + +describe("SessionVision.bridge", () => { + itAvailable.effect("transcribes images for models without vision support", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([textPart(), imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).not.toBe(input) + expect(output[0]?.parts).toHaveLength(2) + expect(output[0]?.parts[1]).toMatchObject({ type: "text", synthetic: true }) + const part = output[0]?.parts[1] + const text = part && part.type === "text" ? part.text : "" + expect(text).toContain("transcribed text") + expect(text).toContain("shot.png") + expect(input[0]?.parts[1]?.type).toBe("file") + expect(available.calls.count).toBe(1) + }), + ) + + itVisionCapable.effect("skips transcription when the model already supports vision", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: visionModel, user: user() }) + + expect(output).toBe(input) + expect(visionCapable.calls.count).toBe(0) + }), + ) + + itOnlyText.effect("leaves messages untouched when no vision model is available", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).toBe(input) + expect(onlyText.calls.count).toBe(0) + }), + ) + + itCached.effect("caches transcripts per image part", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + yield* vision.bridge({ messages: input, model: textModel, user: user() }) + yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(cached.calls.count).toBe(1) + }), + ) + + itCrossProvider.effect("never uses another provider's vision model", () => + Effect.gen(function* () { + const vision = yield* SessionVision.Service + const input = messages([imagePart("prt_image")]) + const output = yield* vision.bridge({ messages: input, model: textModel, user: user() }) + + expect(output).toBe(input) + expect(crossProvider.calls.count).toBe(0) + }), + ) +})