From 8769eba2002a838abd92638558be52e339c52189 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Mon, 31 Aug 2026 10:12:01 -0700 Subject: [PATCH 1/3] Let a Bot draw the answer, instead of describing markup it cannot show A Bot asked for a chart could write the HTML for one and never put it on screen. The renderer was already here, wired to the stored components the playground publishes, but the runtime half was not: `openGenerativeUI` was never passed, and that middleware is the only thing that turns a streamed `generateSandboxedUi` call into the activity events that paint. Without it the tool's own renderer shows the waiting message and then returns nothing, so the Bot wrote a whole interface into a transcript that stayed empty. The switch is one deployment capability beside `accessibility` and the computer, on unless `OPENBOT_GENERATIVE_UI_DISABLED` says otherwise. It is not a per-Bot grant because the SDK has no seam for one: the agent list narrows only the event transform while the browser keeps offering the tool to every Bot, so naming some Bots would leave the rest able to call it and draw nothing, which is worse than not offering it at all. The flag has to reach the browser as well as the runtime, which is why it is projected on /api/capabilities. The SDK reads the capability as on when either the runtime says so or the provider prop is merely present, so a deployment that switched the server half off while the app kept passing the prop would land back in exactly the state above. Channels could not have shown one either. The transcript projection named the roles it understood and dropped the rest, so a turn whose whole answer was a drawing rendered as silence. Activities are carried through it now, and the call that produces one is dropped from the transcript rather than left as an empty row under every interface. --- .env.example | 13 ++ app/src/components/channels/chat-messages.ts | 43 ++++- .../components/channels/chat-transcript.tsx | 60 ++++++- app/src/lib/copilot/generative-ui.ts | 50 ++++++ app/src/lib/copilot/provider.tsx | 27 ++- app/src/lib/deployment/queries.ts | 59 +++++++ app/tests/chat-messages.test.ts | 157 ++++++++++++++++++ docs/configuration.md | 23 +++ server/src/app.ts | 10 ++ server/src/config.ts | 37 +++++ server/src/copilot.ts | 17 ++ server/tests/config.test.ts | 39 +++++ server/tests/health.test.ts | 25 +++ 13 files changed, 555 insertions(+), 5 deletions(-) create mode 100644 app/src/lib/copilot/generative-ui.ts create mode 100644 app/src/lib/deployment/queries.ts create mode 100644 app/tests/chat-messages.test.ts diff --git a/.env.example b/.env.example index cb3c4ac62..6c5c79036 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,19 @@ KEY_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= PORT=3001 SERVER_PORT=3001 TENANT_PACKAGE_DIR=../examples/fintech +# Whether a Bot may answer with an interface it wrote itself: markup, styles and a script it +# generates for that one answer, streamed into the transcript and rendered in a sandboxed iframe with +# no same-origin access to this app. On unless this says otherwise. +# +# This is not the component catalogue. A component is something this deployment holds and an +# administrator grants per Bot; this has nothing to grant, because the Bot writes it on the spot and +# it is gone when the conversation moves on. It is a deployment switch rather than a per-Bot grant +# because the SDK offers no seam for one — see DeploymentConfig.generativeUi in server/src/config.ts. +# +# What the interface can reach is what the sandbox hands it, and this deployment hands it nothing: no +# session, no same-origin access, and no route into your data. It can load libraries from a CDN, so a +# deployment that must not reach the public internet from a browser tab is a reason to turn this off. +# OPENBOT_GENERATIVE_UI_DISABLED=true # What this deployment calls itself, when more than one shares an Intelligence project. A copy of a # deployment made for development uses the same project key, and threads are listed per Bot with # nothing to say which deployment a conversation came from. The name goes into every thread id this diff --git a/app/src/components/channels/chat-messages.ts b/app/src/components/channels/chat-messages.ts index aba4af7ef..7dce9acf8 100644 --- a/app/src/components/channels/chat-messages.ts +++ b/app/src/components/channels/chat-messages.ts @@ -1,4 +1,4 @@ -import type { Message, ToolCall } from "@ag-ui/core"; +import type { ActivityMessage, Message, ToolCall } from "@ag-ui/core"; /** * Transcript projection that pairs assistant tool calls with later tool-result messages. @@ -12,7 +12,25 @@ export type VisibleChatItem = toolCall: ToolCall; /** The result, once there is one. Absent means the call is still in flight. */ result?: string; - }; + } + /** + * Something a Bot is drawing rather than saying. + * + * Carried whole rather than projected into fields of our own, because what is inside an activity + * belongs to whoever renders it: an interface a Bot generated arrives here as partial HTML that + * grows on every chunk, and the renderer that paints it is the one that knows what a half-finished + * one looks like. Reshaping it on the way past would mean this file had to understand every + * activity type anybody registers. + */ + | { kind: "activity"; id: string; message: ActivityMessage }; + +/** + * The SDK's own tool for drawing an interface, whose output is an activity rather than a result. + * + * Named here rather than imported because the SDK exports the renderer and the argument schema but + * not the tool name; it is the string the runtime middleware matches on to emit the activity. + */ +const GENERATE_SANDBOXED_UI = "generateSandboxedUi"; /** A tool result, as it arrives, its own message, pointing back at the call it answers. */ type ToolResultMessage = { role: "tool"; toolCallId: string; content?: string }; @@ -44,6 +62,15 @@ export function toVisibleChatItems( }); } for (const toolCall of message.toolCalls ?? []) { + /* + * The call that draws an interface is not a row of its own; the interface is. + * + * Its renderer shows the waiting message and then returns nothing, so once the interface has + * arrived this leaves an empty item behind — invisible in itself, but still a child of a + * `gap-6` column, so every generated interface gained a stray gap under it. The activity + * beside it already shows its own progress while it is being written. + */ + if (toolCall.function.name === GENERATE_SANDBOXED_UI) continue; items.push({ kind: "tool", // One assistant message can carry multiple tool calls. @@ -57,6 +84,18 @@ export function toVisibleChatItems( return items; } + /* + * Activities are their own messages, in order, beside the prose. + * + * Kept rather than dropped, which is what this projection used to do with every role it did not + * name. A Bot that draws its own interface says nothing in `content` and calls no tool the + * transcript can pair a result with — the whole answer is the activity. Falling through to the + * bail below meant the turn rendered as silence. + */ + if (message.role === "activity") { + return [{ kind: "activity", id: message.id, message }]; + } + if (message.role !== "user") return []; const text = diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsx index 548bc600f..2a6a1fddb 100644 --- a/app/src/components/channels/chat-transcript.tsx +++ b/app/src/components/channels/chat-transcript.tsx @@ -1,5 +1,8 @@ -import type { Message } from "@ag-ui/core"; -import { useRenderToolCall } from "@copilotkit/react-core/v2"; +import type { ActivityMessage, Message } from "@ag-ui/core"; +import { + useRenderActivityMessage, + useRenderToolCall, +} from "@copilotkit/react-core/v2"; import { IconBox } from "@tabler/icons-react"; import { motion, useReducedMotion } from "motion/react"; import { memo, useEffect, useMemo, useRef } from "react"; @@ -464,6 +467,52 @@ const TranscriptMessage = memo(function TranscriptMessage({ ); }); +/** + * What a failed activity is called on screen. + * + * An `activityType` is a protocol name and reads like one, so the boundary's sentence gets a phrase + * a person can read instead. An unknown type falls back to its own name rather than to something + * vague: whoever registered it will recognise it, and nobody else can act on either wording. + */ +function activityName(activityType: string): string { + return activityType === "open-generative-ui" + ? "This Bot's generated interface" + : activityType; +} + +/** + * One activity, drawn by whichever renderer claims its type. + * + * The renderer comes from the SDK's registry, so an activity nobody registered draws nothing and + * this returns null rather than an empty row — unlike a tool call, which always has a line to fall + * back to because a call that happened is worth reporting even undrawn. An activity is the drawing; + * with no renderer there is nothing to say about it. + * + * The memo boundary earns its place differently here than on a tool call. It cannot spare this + * component its own churn — a generated interface streams, and `content` is a new object on every + * chunk, which is exactly when it must re-render — but it does stop the sentence being typed after it + * from re-rendering a mounted iframe. + */ +const TranscriptActivity = memo(function TranscriptActivity({ + delay, + message, +}: { + delay: number; + message: ActivityMessage; +}) { + const { renderActivityMessage } = useRenderActivityMessage(); + const drawn = renderActivityMessage(message); + if (!drawn) return null; + + return ( + + + {drawn} + + + ); +}); + /** * One drawn tool call, memoised on the same terms. * @@ -653,6 +702,13 @@ export function ChatTranscript({ toolCallId={item.toolCall.id} /> + ) : item.kind === "activity" ? ( + + + ) : ( and tags do load, so Chart.js, D3 and similar are available when a chart genuinely needs them; prefer plain SVG or CSS for anything simple. +- No network calls to this deployment. The iframe has no session and no same-origin access; a fetch to /api will fail. +- Guard every read of browser storage in try/catch. It throws outright in some contexts. +- Interactive controls need a visible focus ring, real button elements, and hit targets of at least 32px. +- Prefer one clear interface over a dashboard of panels. A single legible chart beats four cramped ones.`; diff --git a/app/src/lib/copilot/provider.tsx b/app/src/lib/copilot/provider.tsx index 14e0b196e..46001b609 100644 --- a/app/src/lib/copilot/provider.tsx +++ b/app/src/lib/copilot/provider.tsx @@ -1,9 +1,12 @@ import { CopilotKitProvider } from "@copilotkit/react-core/v2"; +import { useQuery } from "@tanstack/react-query"; import type { ReactNode } from "react"; +import { deploymentCapabilitiesQueryOptions } from "@/lib/deployment/queries"; import { ActiveBotProvider } from "./active-bot"; import { ComputerTools } from "./computer-tools"; import { EscalationTool } from "./escalation-tool"; import { GalleryTools } from "./gallery-tools"; +import { GENERATIVE_UI_DESIGN_SKILL } from "./generative-ui"; import { HandoffTool } from "./handoff-tool"; import { SandboxedTools } from "./sandboxed-tools"; @@ -22,8 +25,30 @@ import { SandboxedTools } from "./sandboxed-tools"; * runtime rather than returning it). */ export function CopilotProvider({ children }: { children: ReactNode }) { + const { data: capabilities } = useQuery(deploymentCapabilitiesQueryOptions()); + return ( - + {/* Computer tools target the Bot declared by the mounted surface. */} diff --git a/app/src/lib/deployment/queries.ts b/app/src/lib/deployment/queries.ts new file mode 100644 index 000000000..cee31dc13 --- /dev/null +++ b/app/src/lib/deployment/queries.ts @@ -0,0 +1,59 @@ +import { queryOptions } from "@tanstack/react-query"; +import { client } from "@/lib/client"; + +/** + * What this deployment can do, as the server is willing to say it. + * + * A projection, not the runtime. `/api/capabilities` is reachable before anybody has signed in, so + * only fields somebody may know unauthenticated appear on it; the Intelligence contract and every + * deployment secret stay on the server. See server/src/app.ts. + */ +export type DeploymentCapabilities = { + /** + * Whether a Bot may answer with an interface it wrote itself. + * + * Read by the browser because the browser owns half of this capability: the SDK's provider is what + * offers the model the tool that generates one. A deployment that turned the server half off while + * the browser kept offering the tool would have Bots writing whole interfaces that nothing draws, + * so both halves read this one answer. + */ + generativeUi: boolean; +}; + +export const deploymentKeys = { + all: ["deployment"] as const, + capabilities: () => ["deployment", "capabilities"] as const, +}; + +/** + * What this deployment can do. + * + * From the server rather than from the build, like the sign-in options beside it: the container image + * is built once and knows nothing about the deployment that will run it, so a capability compiled + * into the bundle can only ever describe the build machine. + * + * Absent fields read as off. A server too old to answer, or one that failed to, is a server this app + * should not assume a capability of — and for generated interfaces the fail-closed direction is the + * safe one, because claiming it wrongly is what makes a Bot generate something nothing renders. + */ +export function deploymentCapabilitiesQueryOptions() { + return queryOptions({ + queryKey: deploymentKeys.capabilities(), + // Configuration, not data. It cannot change without the process restarting. + staleTime: Number.POSITIVE_INFINITY, + queryFn: async (): Promise => { + /* + * The whole body, then one field off it. `/api/capabilities` answers with a bare object rather + * than the envelope most endpoints use, which is why this reads the Response itself instead of + * naming a key — the same shape the sign-in query uses. + */ + const body = (await ( + await client("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/api/capabilities", { + fallback: "This deployment's capabilities could not be loaded.", + }) + ).json()) as { generativeUi?: boolean }; + + return { generativeUi: body.generativeUi === true }; + }, + }); +} diff --git a/app/tests/chat-messages.test.ts b/app/tests/chat-messages.test.ts new file mode 100644 index 000000000..b637c3ecf --- /dev/null +++ b/app/tests/chat-messages.test.ts @@ -0,0 +1,157 @@ +import type { Message } from "@ag-ui/core"; +import { describe, expect, test } from "bun:test"; +import { toVisibleChatItems } from "../src/components/channels/chat-messages"; + +/** + * What a channel transcript shows, out of the messages a run produced. + * + * The projection used to name the roles it understood and drop everything else, which was correct + * while every answer was prose or a tool call. It stopped being correct once a Bot could answer by + * drawing: a generated interface arrives as an activity message, says nothing in `content`, and + * pairs with no tool result — so the turn rendered as silence. These cases hold that shut. + */ + +const PROSE: Message = { + id: "assistant-1", + role: "assistant", + content: "Here is how those issues group.", +}; + +/** A generated interface, mid-stream: the HTML grows on every chunk and `generating` is still true. */ +const DRAWING: Message = { + id: "activity-1", + role: "activity", + activityType: "open-generative-ui", + content: { + css: ".card { color: #0a0a0a }", + cssComplete: true, + html: ['
'], + htmlComplete: false, + generating: true, + }, +}; + +describe("toVisibleChatItems", () => { + test("keeps an activity, carrying the message whole", () => { + expect(toVisibleChatItems([DRAWING])).toEqual([ + { kind: "activity", id: "activity-1", message: DRAWING }, + ]); + }); + + /* + * The regression this projection had. A Bot that answers only by drawing produces exactly this + * one message, so dropping it left a turn that had plainly happened showing nothing at all. + */ + test("does not render a drawing-only turn as silence", () => { + expect(toVisibleChatItems([DRAWING])).not.toEqual([]); + }); + + test("keeps an activity in its place beside the prose", () => { + expect( + toVisibleChatItems([PROSE, DRAWING]).map((item) => item.kind), + ).toEqual(["text", "activity"]); + }); + + /* + * An activity is a message in its own right, not something folded into the assistant turn that + * preceded it: it renders as its own row, and both survive. + */ + test("draws prose and an activity as two items", () => { + const items = toVisibleChatItems([PROSE, DRAWING]); + + expect(items).toHaveLength(2); + expect(items[0]).toEqual({ + kind: "text", + id: "assistant-1", + role: "assistant", + text: "Here is how those issues group.", + }); + }); + + // The roles this file already understood, so the addition above is not paid for elsewhere. + test("still pairs a tool call with the result that answers it", () => { + const called: Message = { + id: "assistant-2", + role: "assistant", + content: "", + toolCalls: [ + { + id: "call-1", + type: "function", + function: { name: "botActivity", arguments: '{"days":7}' }, + }, + ], + }; + const answered: Message = { + id: "result-1", + role: "tool", + toolCallId: "call-1", + content: "42", + }; + + expect(toVisibleChatItems([called, answered])).toEqual([ + { + kind: "tool", + id: "call-1", + toolCall: called.toolCalls?.[0], + result: "42", + }, + ]); + }); + + /* + * The call that produces a generated interface is not a row of its own. + * + * Its renderer shows the waiting message and then returns nothing, so keeping the item left an + * empty child in a `gap-6` column and every generated interface gained a stray gap beneath it. + */ + test("drops the call that draws an interface, and keeps the interface", () => { + const drawing: Message = { + id: "assistant-3", + role: "assistant", + content: "", + toolCalls: [ + { + id: "call-2", + type: "function", + function: { name: "generateSandboxedUi", arguments: "{}" }, + }, + ], + }; + + expect(toVisibleChatItems([drawing, DRAWING])).toEqual([ + { kind: "activity", id: "activity-1", message: DRAWING }, + ]); + }); + + // Every other tool still gets its row: only the one whose output is the activity is dropped. + test("keeps a call from any other tool", () => { + const other: Message = { + id: "assistant-4", + role: "assistant", + content: "", + toolCalls: [ + { + id: "call-3", + type: "function", + function: { name: "botActivity", arguments: "{}" }, + }, + ], + }; + + expect(toVisibleChatItems([other]).map((item) => item.kind)).toEqual([ + "tool", + ]); + }); + + // Roles the transcript has nothing to draw for are still dropped rather than rendered empty. + test("drops a role it has nothing to show", () => { + const thinking: Message = { + id: "reasoning-1", + role: "reasoning", + content: "considering the grouping", + }; + + expect(toVisibleChatItems([thinking])).toEqual([]); + }); +}); diff --git a/docs/configuration.md b/docs/configuration.md index e9ec67d10..50eae047c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,6 +56,29 @@ at `agent-langgraph` on a laptop. | `APP_DIST_DIR` | unset | Where the built app is, when this process serves it. Set inside the container image; unset in development, where Vite serves the app. | | `AUDIT_RETENTION_DAYS` | unset | Whole number of days to keep audit rows; older ones are removed. Unset keeps the trail forever. | | `WORKER_SHARED_SECRET` | unset; `start.sh` uses a fixed local default | The secret the routines worker presents to fire a due routine. Without it the server refuses every handoff, whether or not a worker exists to send one. | +| `OPENBOT_GENERATIVE_UI_DISABLED` | unset (capability on) | `true` or `1` stops a Bot answering with an interface it wrote itself. | + +**`OPENBOT_GENERATIVE_UI_DISABLED`** turns off generated interfaces. Left unset, a Bot may answer +by writing the markup, styles and script for an interface and streaming it into the transcript, where +it renders in a sandboxed iframe. + +This is not the component catalogue. A component is something the deployment holds — compiled into +the build or authored in the playground — and an administrator grants it per Bot. A generated +interface has nothing to grant: it does not exist until the Bot writes it, and it is gone when the +conversation moves on. That is also why this is one switch for the deployment rather than a grant per +Bot. The interface is painted from activity events that only the runtime middleware emits, and the +tool the model calls is registered by the browser for every Bot the moment that middleware runs, so +enabling it for some Bots would leave the rest able to call the tool and draw nothing. + +The switch reaches both halves. The server stops passing `openGenerativeUI` to the runtime, and +`/api/capabilities` reports the capability as off so the app stops offering the tool. Turning off +only one half is the one configuration worth avoiding: a Bot would generate a whole interface that +nothing renders. + +What a generated interface can reach is what the sandbox hands it, and this deployment hands it +nothing — no session, no same-origin access to the app, no route into your data. It can load +libraries from a CDN, which is the reason a deployment that must not reach the public internet from a +browser tab would turn this off. **`AGENT_STALL_TIMEOUT_MS`** watches for the failure a Bot has that nothing else in the trail can show: a stream that stops producing anything. Every other audit row is something that happened, and diff --git a/server/src/app.ts b/server/src/app.ts index 205614469..8d57c53b7 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -201,6 +201,16 @@ export function createApp( context.json({ mode: config.runtime.mode, durableHistory: config.runtime.durableHistory, + /* + * Whether a Bot may answer with an interface it wrote itself. + * + * Projected because the browser holds half of this capability. The runtime middleware turns a + * generated interface into the events that paint it, and the SDK's provider registers the tool + * that produces one; a deployment that switched the runtime half off while the browser went on + * offering the tool would have Bots writing interfaces nothing ever draws. One flag, read by + * both halves, so off means off. + */ + generativeUi: config.generativeUi, /* * Which identity providers this deployment can sign somebody in with. * diff --git a/server/src/config.ts b/server/src/config.ts index dbe95e5db..540979936 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -221,6 +221,26 @@ export type DeploymentConfig = { singleUser: boolean; /** Names OpenBot on the analytics the runtime already sends. Off with OPENBOT_ACCESSIBILITY_DISABLED. */ accessibility: boolean; + /** + * Whether a Bot may answer with an interface it wrote itself. + * + * This is not the component catalogue. A component is something this deployment holds: it was + * either compiled into the build or authored in the playground, an administrator granted it to a + * Bot, and all a Bot decides is which of them to draw. Here there is nothing to grant, because + * there is nothing yet — the Bot writes the markup, the styles and the script for this one answer, + * and they are gone when the conversation moves on. + * + * A deployment switch rather than a per-Bot grant because the SDK offers no seam for one. The + * interface is painted from activity events that only the runtime middleware emits, and the tool + * the model calls is registered by the browser for every Bot the moment that middleware is on. + * Narrowing the middleware to some Bots would leave the rest able to call the tool and draw + * nothing at all, which is a worse answer than never offering it. + * + * What it runs is sandboxed by the SDK, in an iframe with no same-origin access to this app, so a + * generated interface reaches this deployment's data only through what the host hands it. This + * deployment hands it nothing. + */ + generativeUi: boolean; /** * Where the built app is, when this process serves it. * @@ -770,6 +790,22 @@ function accessibilityEnabled(environment: Environment): boolean { return off !== "true" && off !== "1"; } +/** + * Whether a Bot may draw an interface it wrote itself. + * + * On unless told otherwise, the same shape as OPENBOT_ACCESSIBILITY_DISABLED above it: a capability + * a fork gets without having to ask for it, and one an operator takes away in a single variable. + * + * The off switch has to reach the browser as well as the runtime, which is why this ends up on + * /api/capabilities rather than staying server-side. Turning off only the runtime half would leave + * the browser still offering the tool, and a Bot would generate a whole interface that nothing + * renders. See DeploymentConfig.generativeUi. + */ +function generativeUiEnabled(environment: Environment): boolean { + const off = optional(environment, "OPENBOT_GENERATIVE_UI_DISABLED"); + return off !== "true" && off !== "1"; +} + /** * How long the audit trail is kept. * @@ -840,6 +876,7 @@ export function loadConfig( configuredAuthProviders(auth).length > 0, ), accessibility: accessibilityEnabled(environment), + generativeUi: generativeUiEnabled(environment), ...(optional(environment, "APP_DIST_DIR") ? { appDistDir: optional(environment, "APP_DIST_DIR") as string } : {}), diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 891c53eee..fb0bd4a82 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -1086,6 +1086,23 @@ export function mountCopilotRuntime( ...(config.accessibility ? { telemetryProperties: { accessibility_title: "OpenBot" } } : {}), + /* + * What lets a Bot answer with an interface it wrote itself. + * + * This one flag is the whole difference between a Bot that draws and a Bot that describes + * markup it cannot show. The middleware it turns on does not give the model the tool — the + * browser does that — it reads the arguments of the `generateSandboxedUi` call as they stream + * and re-emits them as `open-generative-ui` activity events. Those events are the only thing + * that paints: the tool's own renderer shows the waiting message and then returns nothing. So a + * deployment with the browser half and not this one has Bots generating whole interfaces that + * never appear, which is the shape this capability arrived in. + * + * `true` rather than a list of Bots. The list narrows only the event transform, and the tool + * stays offered to every Bot regardless, so naming some Bots here would leave the others able to + * call it and draw nothing. Whether the capability exists at all is the switch this deployment + * has; see DeploymentConfig.generativeUi. + */ + ...(config.generativeUi ? { openGenerativeUI: true } : {}), // `identifyUser` is the Intelligence projection of the same person `identifyActor` returns: // one resolver decides both whose threads these are and whose coworkers exist. agents: createRequestAgents( diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index 0c8878114..385b38ec9 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -599,6 +599,45 @@ describe("accessibility", () => { ); }); +/** + * Whether a Bot may answer with an interface it wrote itself. + * + * Same shape as accessibility above, and tested to the same bar for the same reason: the off switch + * has a second reader. It is projected on /api/capabilities so the browser stops offering the tool + * too, so a value that silently failed to mean "off" would leave Bots generating interfaces nothing + * renders rather than merely leaving a capability on. + */ +describe("generated interfaces", () => { + test("are on when nothing is set", () => { + expect(loadConfig(baseEnvironment).generativeUi).toBe(true); + }); + + test.each(["true", "1"])( + "are off on OPENBOT_GENERATIVE_UI_DISABLED=%p", + (value) => { + expect( + loadConfig({ + ...baseEnvironment, + OPENBOT_GENERATIVE_UI_DISABLED: value, + }).generativeUi, + ).toBe(false); + }, + ); + + // Anything else is not a way of saying off, exactly as above. + test.each(["false", "no", "", "yes"])( + "stay on for OPENBOT_GENERATIVE_UI_DISABLED=%p", + (value) => { + expect( + loadConfig({ + ...baseEnvironment, + OPENBOT_GENERATIVE_UI_DISABLED: value, + }).generativeUi, + ).toBe(true); + }, + ); +}); + /** * Naming the private addresses an agent may live at. * diff --git a/server/tests/health.test.ts b/server/tests/health.test.ts index 24f112cc2..5e9d077ef 100644 --- a/server/tests/health.test.ts +++ b/server/tests/health.test.ts @@ -26,6 +26,9 @@ describe("runtime capabilities", () => { await expect(response.json()).resolves.toEqual({ mode: "intelligence", durableHistory: true, + // On unless an operator turned it off. The browser reads this to decide whether to offer the + // tool that generates an interface, so it has to be here and not only in the runtime. + generativeUi: true, // Names only. The sign-in screen reads this to know which buttons to draw. authProviders: ["google"], // A boolean, not a list: naming the registered providers would tell anybody who loads the @@ -47,12 +50,34 @@ describe("runtime capabilities", () => { expect(Object.keys(parsed)).toEqual([ "mode", "durableHistory", + "generativeUi", "authProviders", "ssoConfigured", ]); // The provider list is names, never the clients and secrets behind them. expect(body).not.toContain("google-client-secret"); }); + + /* + * Off has to reach the browser, not just the runtime. + * + * The app offers the model the tool that generates an interface, and it decides whether to from + * this field. A deployment that switched the runtime half off while this still said `true` would + * have Bots writing whole interfaces that nothing renders, which is the one configuration this + * capability must not be able to end up in. + */ + test("reports generated interfaces as off when the deployment disabled them", async () => { + const disabled = createApp( + loadConfig(testEnvironment({ OPENBOT_GENERATIVE_UI_DISABLED: "true" })), + ); + + const response = await disabled.request( + "http://openbot.local/api/capabilities", + ); + + expect(response.status).toBe(200); + expect((await response.json()).generativeUi).toBe(false); + }); }); describe("authentication availability", () => { From 67dad90d2b196f416565b57aab4f6cdd6691ad7b Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Mon, 31 Aug 2026 10:22:23 -0700 Subject: [PATCH 2/3] Never let a generated interface occupy nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK sizes the frame it draws into from one measurement taken inside the sandbox and writes it as an inline height. Two races make that number wrong. The measurement is queued alongside the expressions that build the interface, so when it runs first it measures an empty body, writes `height: 0px`, and because the container clips, the interface is invisible; nothing retries, because the measurement is one-shot. Separately, the effect that measures bails when the sandbox is still loading and its only dependency is "generation finished" — an interface restored from history is finished on its first render, so it bails and never runs again. A floor is the smallest thing that removes the failure that matters. `min-height` clamps above an inline height, so a collapsed measurement can no longer render nothing while a correct larger measurement is left alone. A reader gets an interface that is present and may be cut off, rather than a turn that looks like the Bot said nothing. Selected through the frame the sandbox library creates rather than through a wrapper of ours, because both chat surfaces build this container and only one of them is ours; a rule needing our wrapper would fix the transcript and leave the packaged chat broken. It compensates for arithmetic this repository does not own, so it says so and says when to delete it. --- app/src/styles.css | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/app/src/styles.css b/app/src/styles.css index 66d7cb190..2326b7d04 100644 --- a/app/src/styles.css +++ b/app/src/styles.css @@ -255,3 +255,37 @@ body { [data-slot="item-description"] { text-wrap: pretty; } + +/* + * A generated interface is never allowed to occupy nothing. + * + * The SDK sizes the frame it draws into from a single measurement taken inside the sandbox, and it + * writes the result as an inline height on the frame's own container. Two races in that code make + * the number wrong, and both were reproduced against @copilotkit/react-core 1.69.0: + * + * - The measurement is queued alongside the expressions that build the interface. When it runs + * first the body it measures is still empty, the container is set to `height: 0px`, and because + * the container also clips, the interface is invisible. Nothing retries: the measurement is + * one-shot and its listener is removed after the first reply. + * - The effect that measures bails out when the sandbox has not finished loading, and its only + * dependency is "generation finished". An interface restored from thread history is already + * finished on its first render, while the sandbox is still being imported, so the effect bails + * and never runs again. + * + * A floor is the smallest thing that removes the failure that matters. `min-height` clamps above an + * inline `height`, so a collapsed measurement can no longer render nothing, and a measurement that + * came back correct and larger is left alone. The reader gets an interface that is present and may + * be cut off, instead of a turn that looks like the Bot said nothing at all. + * + * Selected through the frame rather than through a class of ours because both chat surfaces render + * this container and only one of them is ours: the packaged chat builds it too, and a rule that + * needed our wrapper would fix the transcript and leave the other surface broken. `websandbox__frame` + * is the class the sandbox library puts on the frame it creates. + * + * DELETE THIS WHEN THE SDK IS FIXED. It compensates for someone else's arithmetic, so it should not + * outlive the bug: once the measurement re-runs on sandbox-ready and after the queue drains, this + * rule can only make small interfaces taller than they asked to be. + */ +:where(div:has(> iframe.websandbox__frame)) { + min-height: 200px; +} From 2a81fa29ba86fd26a3087bb8df46e8f3fafdf182 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Mon, 31 Aug 2026 10:26:54 -0700 Subject: [PATCH 3/3] Make drawing something a deployment asks for, not something it inherits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The switch was written as `OPENBOT_GENERATIVE_UI_DISABLED`, matching the accessibility flag beside it. That symmetry was wrong. Accessibility names a deployment in an analytics label, so defaulting it on costs a fork nothing it would mind. This decides whether a model may put code it wrote on somebody's screen and pull libraries from a CDN to run it. Written as a disable switch, absence is the permissive answer: every existing deployment acquires the capability by upgrading rather than by choosing it, and a deployment that builds its default branch automatically acquires it without anybody present to decide. That is the wrong direction for a capability whose failure mode is somebody else's code running in front of a signed-in person. So `OPENBOT_GENERATIVE_UI` instead, and only "true" or "1" count as yes. Anything else — a stray "false", an empty variable left by a template — leaves it off, because a value nobody intended should not turn a capability on. The old spelling is deliberately inert, so a deployment that set it does not read as having made a choice it has not made. Both halves still read one answer. Absent, the runtime is not given the middleware and /api/capabilities reports the capability off, so the browser never offers the tool and no Bot writes an interface nothing will draw. --- .env.example | 14 ++++++---- docs/configuration.md | 25 ++++++++++------- server/src/config.ts | 31 +++++++++++++++------- server/tests/config.test.ts | 53 +++++++++++++++++++++++-------------- server/tests/health.test.ts | 22 +++++++-------- 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 6c5c79036..f46d803a2 100644 --- a/.env.example +++ b/.env.example @@ -19,9 +19,13 @@ KEY_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= PORT=3001 SERVER_PORT=3001 TENANT_PACKAGE_DIR=../examples/fintech -# Whether a Bot may answer with an interface it wrote itself: markup, styles and a script it -# generates for that one answer, streamed into the transcript and rendered in a sandboxed iframe with -# no same-origin access to this app. On unless this says otherwise. +# Let a Bot answer with an interface it wrote itself: markup, styles and a script it generates for +# that one answer, streamed into the transcript and rendered in a sandboxed iframe with no +# same-origin access to this app. Off unless you set this to `true` or `1`. +# +# Asked for rather than inherited, unlike most of the switches in this file. It decides whether a +# model may put code it wrote on somebody's screen, so a deployment should choose it rather than +# acquire it by upgrading — including a deployment that builds its default branch automatically. # # This is not the component catalogue. A component is something this deployment holds and an # administrator grants per Bot; this has nothing to grant, because the Bot writes it on the spot and @@ -30,8 +34,8 @@ TENANT_PACKAGE_DIR=../examples/fintech # # What the interface can reach is what the sandbox hands it, and this deployment hands it nothing: no # session, no same-origin access, and no route into your data. It can load libraries from a CDN, so a -# deployment that must not reach the public internet from a browser tab is a reason to turn this off. -# OPENBOT_GENERATIVE_UI_DISABLED=true +# deployment that must not reach the public internet from a browser tab should leave this off. +# OPENBOT_GENERATIVE_UI=true # What this deployment calls itself, when more than one shares an Intelligence project. A copy of a # deployment made for development uses the same project key, and threads are listed per Bot with # nothing to say which deployment a conversation came from. The name goes into every thread id this diff --git a/docs/configuration.md b/docs/configuration.md index 50eae047c..738d1a1d8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,11 +56,18 @@ at `agent-langgraph` on a laptop. | `APP_DIST_DIR` | unset | Where the built app is, when this process serves it. Set inside the container image; unset in development, where Vite serves the app. | | `AUDIT_RETENTION_DAYS` | unset | Whole number of days to keep audit rows; older ones are removed. Unset keeps the trail forever. | | `WORKER_SHARED_SECRET` | unset; `start.sh` uses a fixed local default | The secret the routines worker presents to fire a due routine. Without it the server refuses every handoff, whether or not a worker exists to send one. | -| `OPENBOT_GENERATIVE_UI_DISABLED` | unset (capability on) | `true` or `1` stops a Bot answering with an interface it wrote itself. | +| `OPENBOT_GENERATIVE_UI` | unset (capability off) | `true` or `1` lets a Bot answer with an interface it wrote itself. | -**`OPENBOT_GENERATIVE_UI_DISABLED`** turns off generated interfaces. Left unset, a Bot may answer -by writing the markup, styles and script for an interface and streaming it into the transcript, where -it renders in a sandboxed iframe. +**`OPENBOT_GENERATIVE_UI`** turns on generated interfaces. Set it, and a Bot may answer by writing +the markup, styles and script for an interface and streaming it into the transcript, where it renders +in a sandboxed iframe. Left unset, a Bot answers in prose and with the components this deployment +holds, as before. + +It is asked for rather than inherited, which is deliberate and unlike most switches here. This one +decides whether a model may put code it wrote on somebody's screen and load libraries from a CDN to +run it, so a deployment should choose it rather than acquire it by upgrading — including a deployment +that builds its default branch automatically. Only `true` or `1` count as yes; anything else leaves it +off. This is not the component catalogue. A component is something the deployment holds — compiled into the build or authored in the playground — and an administrator grants it per Bot. A generated @@ -70,15 +77,15 @@ Bot. The interface is painted from activity events that only the runtime middlew tool the model calls is registered by the browser for every Bot the moment that middleware runs, so enabling it for some Bots would leave the rest able to call the tool and draw nothing. -The switch reaches both halves. The server stops passing `openGenerativeUI` to the runtime, and -`/api/capabilities` reports the capability as off so the app stops offering the tool. Turning off -only one half is the one configuration worth avoiding: a Bot would generate a whole interface that -nothing renders. +The switch reaches both halves. The server passes `openGenerativeUI` to the runtime, and +`/api/capabilities` reports the capability so the app offers the tool. The halves disagreeing is the +one configuration worth avoiding: runtime-only means the tool is never offered, and browser-only +means a Bot generates a whole interface that nothing renders. What a generated interface can reach is what the sandbox hands it, and this deployment hands it nothing — no session, no same-origin access to the app, no route into your data. It can load libraries from a CDN, which is the reason a deployment that must not reach the public internet from a -browser tab would turn this off. +browser tab should leave this unset. **`AGENT_STALL_TIMEOUT_MS`** watches for the failure a Bot has that nothing else in the trail can show: a stream that stops producing anything. Every other audit row is something that happened, and diff --git a/server/src/config.ts b/server/src/config.ts index 540979936..0bcfc9d2a 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -236,9 +236,14 @@ export type DeploymentConfig = { * Narrowing the middleware to some Bots would leave the rest able to call the tool and draw * nothing at all, which is a worse answer than never offering it. * + * Off until a deployment sets OPENBOT_GENERATIVE_UI. A capability that runs code a model wrote is + * one an operator should choose, not one they should discover after an upgrade — and a deployment + * that builds its default branch automatically would otherwise acquire it without a decision. + * * What it runs is sandboxed by the SDK, in an iframe with no same-origin access to this app, so a * generated interface reaches this deployment's data only through what the host hands it. This - * deployment hands it nothing. + * deployment hands it nothing. It can load libraries from a CDN, which is the part a deployment + * that must not reach the public internet from a browser tab needs to weigh. */ generativeUi: boolean; /** @@ -793,17 +798,25 @@ function accessibilityEnabled(environment: Environment): boolean { /** * Whether a Bot may draw an interface it wrote itself. * - * On unless told otherwise, the same shape as OPENBOT_ACCESSIBILITY_DISABLED above it: a capability - * a fork gets without having to ask for it, and one an operator takes away in a single variable. + * ASKED FOR, NOT INHERITED, which is the one place this deliberately breaks the symmetry with + * OPENBOT_ACCESSIBILITY_DISABLED above it. That flag names a deployment out of an analytics label, + * so defaulting it on costs a fork nothing it would mind. This one decides whether a model may put + * code it wrote on somebody's screen and pull libraries from a CDN to run it. Written as a disable + * switch, absence would be the permissive answer, and a deployment acquires the capability by + * upgrading rather than by choosing it — which is exactly how a deployment that auto-deploys its + * default branch would find out. + * + * Only "true" or "1" turn it on. Anything else is not a way of saying yes, and a value nobody + * intended should leave a capability off rather than on. * - * The off switch has to reach the browser as well as the runtime, which is why this ends up on - * /api/capabilities rather than staying server-side. Turning off only the runtime half would leave - * the browser still offering the tool, and a Bot would generate a whole interface that nothing - * renders. See DeploymentConfig.generativeUi. + * The answer has to reach the browser as well as the runtime, which is why it ends up on + * /api/capabilities rather than staying server-side. Enabling only the runtime half would leave the + * browser never offering the tool; enabling only the browser half would have a Bot generate a whole + * interface that nothing renders. See DeploymentConfig.generativeUi. */ function generativeUiEnabled(environment: Environment): boolean { - const off = optional(environment, "OPENBOT_GENERATIVE_UI_DISABLED"); - return off !== "true" && off !== "1"; + const on = optional(environment, "OPENBOT_GENERATIVE_UI"); + return on === "true" || on === "1"; } /** diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index 385b38ec9..394124fcd 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -608,34 +608,47 @@ describe("accessibility", () => { * renders rather than merely leaving a capability on. */ describe("generated interfaces", () => { - test("are on when nothing is set", () => { - expect(loadConfig(baseEnvironment).generativeUi).toBe(true); + /* + * The default is the whole point of this block. Written as a disable switch, an upgrade would hand + * every existing deployment the ability to run code a model wrote, and a deployment that builds + * its default branch automatically would acquire it without anybody deciding to. + */ + test("are off when nothing is set", () => { + expect(loadConfig(baseEnvironment).generativeUi).toBe(false); }); - test.each(["true", "1"])( - "are off on OPENBOT_GENERATIVE_UI_DISABLED=%p", + test.each(["true", "1"])("are on for OPENBOT_GENERATIVE_UI=%p", (value) => { + expect( + loadConfig({ ...baseEnvironment, OPENBOT_GENERATIVE_UI: value }) + .generativeUi, + ).toBe(true); + }); + + /* + * Anything else is not a way of saying yes. A value nobody intended — a stray "false", an empty + * variable left behind by a template — should leave the capability off, which is the direction + * that cannot surprise anybody. + */ + test.each(["false", "no", "", "yes", "TRUE", "on"])( + "stay off for OPENBOT_GENERATIVE_UI=%p", (value) => { expect( - loadConfig({ - ...baseEnvironment, - OPENBOT_GENERATIVE_UI_DISABLED: value, - }).generativeUi, + loadConfig({ ...baseEnvironment, OPENBOT_GENERATIVE_UI: value }) + .generativeUi, ).toBe(false); }, ); - // Anything else is not a way of saying off, exactly as above. - test.each(["false", "no", "", "yes"])( - "stay on for OPENBOT_GENERATIVE_UI_DISABLED=%p", - (value) => { - expect( - loadConfig({ - ...baseEnvironment, - OPENBOT_GENERATIVE_UI_DISABLED: value, - }).generativeUi, - ).toBe(true); - }, - ); + // The old spelling was a disable switch. It must not still work, or a deployment that set it + // would read as having made a choice it has not made under the new name. + test("ignore the disable switch this replaced", () => { + expect( + loadConfig({ + ...baseEnvironment, + OPENBOT_GENERATIVE_UI_DISABLED: "false", + }).generativeUi, + ).toBe(false); + }); }); /** diff --git a/server/tests/health.test.ts b/server/tests/health.test.ts index 5e9d077ef..47912f342 100644 --- a/server/tests/health.test.ts +++ b/server/tests/health.test.ts @@ -26,9 +26,9 @@ describe("runtime capabilities", () => { await expect(response.json()).resolves.toEqual({ mode: "intelligence", durableHistory: true, - // On unless an operator turned it off. The browser reads this to decide whether to offer the + // Off until a deployment asks for it. The browser reads this to decide whether to offer the // tool that generates an interface, so it has to be here and not only in the runtime. - generativeUi: true, + generativeUi: false, // Names only. The sign-in screen reads this to know which buttons to draw. authProviders: ["google"], // A boolean, not a list: naming the registered providers would tell anybody who loads the @@ -59,24 +59,24 @@ describe("runtime capabilities", () => { }); /* - * Off has to reach the browser, not just the runtime. + * The answer has to reach the browser, not just the runtime. * * The app offers the model the tool that generates an interface, and it decides whether to from - * this field. A deployment that switched the runtime half off while this still said `true` would - * have Bots writing whole interfaces that nothing renders, which is the one configuration this - * capability must not be able to end up in. + * this field. The two halves disagreeing is the one configuration this capability must not be able + * to end up in: runtime-only means the tool is never offered, browser-only means a Bot writes a + * whole interface that nothing renders. */ - test("reports generated interfaces as off when the deployment disabled them", async () => { - const disabled = createApp( - loadConfig(testEnvironment({ OPENBOT_GENERATIVE_UI_DISABLED: "true" })), + test("reports generated interfaces as on when the deployment asked for them", async () => { + const enabled = createApp( + loadConfig(testEnvironment({ OPENBOT_GENERATIVE_UI: "true" })), ); - const response = await disabled.request( + const response = await enabled.request( "http://openbot.local/api/capabilities", ); expect(response.status).toBe(200); - expect((await response.json()).generativeUi).toBe(false); + expect((await response.json()).generativeUi).toBe(true); }); });