feat: enable generative UI, A2UI actions, tables and forms (S8)

.env.example

@@ -22,11 +22,7 @@ SERVER_PORT=3001
 TENANT_PACKAGE_DIR=../examples/fintech
 # Let a Bot answer with an interface it wrote itself: markup, styles and a script it generates for
 # that one answer, streamed into the transcript and rendered in a sandboxed iframe with no
-# same-origin access to this app. Off unless you set this to `true` or `1`.
-#
-# Asked for rather than inherited, unlike most of the switches in this file. It decides whether a
-# model may put code it wrote on somebody's screen, so a deployment should choose it rather than
-# acquire it by upgrading — including a deployment that builds its default branch automatically.
+# same-origin access to this app. On by default; set this to `false` or `0` to opt out.
 #
 # This is not the component catalogue. A component is something this deployment holds and an
 # administrator grants per Bot; this has nothing to grant, because the Bot writes it on the spot and
@@ -35,8 +31,8 @@ TENANT_PACKAGE_DIR=../examples/fintech
 #
 # What the interface can reach is what the sandbox hands it, and this deployment hands it nothing: no
 # session, no same-origin access, and no route into your data. It can load libraries from a CDN, so a
-# deployment that must not reach the public internet from a browser tab should leave this off.
-# OPENBOT_GENERATIVE_UI=true
+# deployment that must not reach the public internet from a browser tab should set this to false.
+# OPENBOT_GENERATIVE_UI=false
 # What this deployment calls itself, when more than one shares an Intelligence project. A copy of a
 # deployment made for development uses the same project key, and threads are listed per Bot with
 # nothing to say which deployment a conversation came from. The name goes into every thread id this

CHANGELOG.md

@@ -8,6 +8,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### Generated interfaces, tables and forms
+
+Generative UI is enabled by default; set `OPENBOT_GENERATIVE_UI=false` or `0` to disable it.
+Bots can render A2UI interfaces, compare records in sortable tables, and collect related answers in
+a form that waits for submission. LangGraph receives the component schemas needed to draw these
+interfaces correctly.
+
+The playground rejects invalid JSON before saving or publishing, confirms successful saves, and
+shows published custom components in the administrator's gallery.
+
 ### A Bot's computer is rebuilt when it holds a token the deployment has stopped using
 
 A computer checks every caller against the `COMPUTER_TOKEN` it was created with, and holds that one

agent-bot/src/history.ts

@@ -17,6 +17,12 @@ export function toProviderMessages(
 ): OpenAI.Chat.ChatCompletionMessageParam[] {
   const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [
     { role: "system", content: COMPUTER_GUIDANCE },
+    // AG-UI application context is separate from history. The A2UI catalog and tool instructions
+    // arrive here; omitting them leaves the model guessing component names and action schemas.
+    ...(input.context ?? []).map(({ description, value }) => ({
+      role: "system" as const,
+      content: `${description}\n${value}`,
+    })),
   ];
 
   /*

agent-bot/tests/history.test.ts

@@ -31,6 +31,37 @@ function withoutGuidance(messages: ReturnType<typeof toProviderMessages>) {
   return messages.slice(1);
 }
 
+test("passes AG-UI catalog context to the model while preserving prompt and history order", () => {
+  const run = input([
+    { id: "standing", role: "system", content: "Help with travel planning." },
+    { id: "request", role: "user", content: "Draw a trip card." },
+  ]);
+  const withoutContext = toProviderMessages(run);
+  const catalog = JSON.stringify({
+    components: { Card: { properties: { component: { const: "Card" } } } },
+  });
+  run.context = [
+    { description: "A2UI Component Schema", value: catalog },
+    {
+      description: "A2UI render tool usage guide",
+      value: "Actions use event.name.",
+    },
+  ];
+
+  expect(toProviderMessages(run)).toEqual([
+    withoutContext[0],
+    { role: "system", content: `A2UI Component Schema\n${catalog}` },
+    {
+      role: "system",
+      content: "A2UI render tool usage guide\nActions use event.name.",
+    },
+    ...withoutContext.slice(1),
+  ]);
+  expect(run.messages).toHaveLength(2);
+  run.context = [];
+  expect(toProviderMessages(run)).toEqual(withoutContext);
+});
+
 describe("a tool call nothing ever answered", () => {
   test("is answered, so the next turn is not refused outright", () => {
     const messages = withoutGuidance(
@@ -241,13 +272,10 @@ describe("a tool call restored from the thread store", () => {
       ],
     } as never);
 
-    const withCalls = messages.find(
-      (message: Record<string, unknown>) => message.tool_calls,
-    ) as Record<string, unknown>;
-    const call = (withCalls.tool_calls as Array<Record<string, unknown>>)[0];
-    const fn = call.function as Record<string, unknown>;
+    const withCalls = messages.find((message) => message.role === "assistant");
+    const fn = withCalls?.tool_calls?.[0]?.function;
 
-    expect(fn.name).toBe("computer_navigate");
-    expect(fn.arguments).toBe('{"url":"https://news.ycombinator.com"}');
+    expect(fn?.name).toBe("computer_navigate");
+    expect(fn?.arguments).toBe('{"url":"https://news.ycombinator.com"}');
   });
 });

agent-langgraph-agui/src/main.py

@@ -15,7 +15,13 @@
 from langgraph.checkpoint.memory import MemorySaver
 from langgraph.graph import START, MessagesState, StateGraph
 
-from .tool_runtime import ToolAwareAgent, bind_tools, execute_tools, next_step
+from .tool_runtime import (
+    ToolAwareAgent,
+    bind_tools,
+    execute_tools,
+    model_messages,
+    next_step,
+)
 
 TOKEN_HEADER = "x-openbot-agent-token"
 
@@ -152,7 +158,8 @@ def _model():
 
 
 async def answer(state: MessagesState):
-    return {"messages": [await bind_tools(_model()).ainvoke(state["messages"])]}
+    messages = model_messages(state["messages"])
+    return {"messages": [await bind_tools(_model()).ainvoke(messages)]}
 
 
 builder = StateGraph(MessagesState)

agent-langgraph-agui/src/tool_runtime.py

@@ -13,8 +13,8 @@
 from dataclasses import dataclass, field
 
 import httpx
-from ag_ui.core import EventType, RunAgentInput, RunErrorEvent, Tool
-from langchain_core.messages import ToolMessage
+from ag_ui.core import Context, EventType, RunAgentInput, RunErrorEvent, Tool
+from langchain_core.messages import SystemMessage, ToolMessage
 from langgraph.graph import END
 
 from .parallel_tools import ParallelToolAgent
@@ -23,6 +23,7 @@
 @dataclass(frozen=True)
 class RunTools:
     tools: tuple[Tool, ...] = ()
+    context: tuple[Context, ...] = ()
     deployment: frozenset[str] = frozenset()
     assertion: str = field(default="", repr=False)
 
@@ -45,6 +46,7 @@ async def run(self, input: RunAgentInput):
         assertion = props.get("openbotRun", "")
         context = RunTools(
             tools=tuple(input.tools or []),
+            context=tuple(input.context or []),
             deployment=frozenset(name for name in names if isinstance(name, str))
             if isinstance(names, list)
             else frozenset(),
@@ -81,6 +83,22 @@ async def run(self, input: RunAgentInput):
             _current.reset(token)
 
 
+def model_messages(messages):
+    """Pass AG-UI application context to the model without checkpointing it.
+
+    The maintained integration carries context separately from messages. Our
+    graph uses MessagesState, so its answer node must explicitly include the
+    current catalog/guidelines instead of silently discarding them.
+    """
+    return [
+        *[
+            SystemMessage(content=f"{entry.description}\n{entry.value}")
+            for entry in current_tools().context
+        ],
+        *messages,
+    ]
+
+
 def bind_tools(model):
     tools = current_tools().tools
     if not tools:

agent-langgraph-agui/tests/test_tool_protocol.py

@@ -232,6 +232,51 @@ def snapshot(events):
     )
 
 
+@pytest.mark.asyncio
+async def test_a2ui_catalog_context_reaches_model_without_entering_history(boundary):
+    body = run_input(
+        [],
+        messages=[{
+            "id": "context-request", "role": "user", "content": "Draw a trip card"
+        }],
+    )
+    catalog = json.dumps({
+        "catalogId": "https://a2ui.org/specification/v0_9/basic_catalog.json",
+        "components": {"Card": {"properties": {"component": {"const": "Card"}}}},
+    })
+    body["context"] = [
+        {
+            "description": (
+                "A2UI Component Schema — available components for generating UI surfaces. "
+                "Use these component names and properties when creating A2UI operations."
+            ),
+            "value": catalog,
+        },
+        {
+            "description": "A2UI render tool usage guide",
+            "value": "Use component: Card, not type: card. Actions use event.name.",
+        },
+    ]
+
+    events = await run_protocol(body)
+    model_messages = boundary["model"][0]["messages"]
+    system = [
+        message["content"] for message in model_messages
+        if message["role"] == "system"
+    ]
+    assert any(catalog in content for content in system)
+    assert any("Actions use event.name." in content for content in system)
+    assert catalog not in json.dumps(snapshot(events))
+    assert "synthetic-run-assertion" not in json.dumps(model_messages)
+
+    # A later request on the same graph thread uses its current context, not a checkpointed catalog.
+    body["runId"] = str(uuid4())
+    body["messages"] = [{"id": "context-next", "role": "user", "content": "Continue"}]
+    body["context"] = []
+    await run_protocol(body)
+    assert catalog not in json.dumps(boundary["model"][-1]["messages"])
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("name", ["computer_navigate", "computer_run_command"])
 async def test_surface_tool_calls_end_then_consume_actual_client_result(boundary, name):

agent-langgraph/src/history.ts

@@ -23,7 +23,15 @@ export { NO_ANSWER_CAME };
 
 /** Translate the conversation AG-UI carries into LangChain's message classes. */
 export function toLangChainMessages(input: RunAgentInput): BaseMessage[] {
-  const messages: BaseMessage[] = [new SystemMessage(COMPUTER_GUIDANCE)];
+  const messages: BaseMessage[] = [
+    new SystemMessage(COMPUTER_GUIDANCE),
+    // AG-UI carries application context separately from conversation history. CopilotKit puts
+    // the A2UI catalog and tool instructions here; dropping it leaves the model guessing the
+    // component schema and can strand the renderer on an invalid, never-painted surface.
+    ...(input.context ?? []).map(
+      ({ description, value }) => new SystemMessage(`${description}\n${value}`),
+    ),
+  ];
 
   /*
    * Which calls in this history were ever answered.

agent-langgraph/tests/history.test.ts

@@ -1,6 +1,10 @@
 import { describe, expect, test } from "bun:test";
-import { AIMessage, ToolMessage } from "@langchain/core/messages";
 import type { RunAgentInput } from "@ag-ui/core";
+import {
+  AIMessage,
+  SystemMessage,
+  ToolMessage,
+} from "@langchain/core/messages";
 import { NO_ANSWER_CAME, toLangChainMessages } from "../src/history";
 
 /**
@@ -30,6 +34,37 @@ const assistantAsking = {
   ],
 };
 
+test("passes the caller's A2UI catalog and tool instructions to the model", () => {
+  // The live failure emitted `type: "card"` instead of `component: "Card"`: the model saw the
+  // permissive render_a2ui tool schema, but this adapter had discarded its actual catalog context.
+  const catalog = JSON.stringify({
+    catalogId: "https://a2ui.org/specification/v0_9/basic_catalog.json",
+    components: {
+      Card: {
+        properties: { component: { const: "Card" }, child: { type: "string" } },
+      },
+    },
+  });
+  const instructions =
+    "Use flat components with component names from the catalog. Button actions use event.name and event.context.";
+  const run = input([
+    { role: "user", content: "Show a Trip preferences card." },
+  ]);
+  run.context = [
+    { description: "A2UI Component Schema", value: catalog },
+    { description: "A2UI render tool usage guide", value: instructions },
+  ];
+  const messages = toLangChainMessages(run);
+  const system = messages.filter((message) => message instanceof SystemMessage);
+  expect(system.map((message) => message.content)).toContain(
+    `A2UI Component Schema\n${catalog}`,
+  );
+  expect(system.map((message) => message.content)).toContain(
+    `A2UI render tool usage guide\n${instructions}`,
+  );
+  expect(messages.at(-1)?.content).toBe("Show a Trip preferences card.");
+});
+
 describe("history with a tool call nobody answered", () => {
   test("closes it, so the next turn is not rejected", () => {
     const messages = toLangChainMessages(

app/package.json

@@ -17,6 +17,7 @@
     "@ag-ui/core": "0.0.59",
     "@base-ui/react": "^1.6.0",
     "@better-auth/sso": "^1.7.1",
+    "@copilotkit/a2ui-renderer": "1.70.1",
     "@copilotkit/react-core": "1.70.1",
     "@fontsource-variable/inter": "^5.3.0",
     "@shadcn/react": "^0.3.0",