diff --git a/.gitignore b/.gitignore index 2cd52b59..f1a2b98b 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,8 @@ agent_bundle/ **/.craftbot/ app/data/.file_index/ .playwright-mcp +**/test-results/ +**/playwright-report/ # Sidecar Node runtime (install.py downloads it when the system Node is too old for Agent App) runtime/ diff --git a/agent_core/core/event_stream/event.py b/agent_core/core/event_stream/event.py index a658b2f0..4f2fd5db 100644 --- a/agent_core/core/event_stream/event.py +++ b/agent_core/core/event_stream/event.py @@ -172,6 +172,7 @@ class Event: platform: Optional[str] = None continue_work: Optional[bool] = None question: Optional[Dict[str, Any]] = None + ui_artifact: Optional[Dict[str, Any]] = None schema_actions: Optional[List[str]] = None def display_text(self) -> Optional[str]: @@ -206,6 +207,7 @@ def to_dict(self) -> Dict[str, Any]: "platform": self.platform, "continue_work": self.continue_work, "question": self.question, + "ui_artifact": self.ui_artifact, "schema_actions": self.schema_actions, } @@ -247,6 +249,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Event": platform=data.get("platform"), continue_work=data.get("continue_work"), question=data.get("question"), + ui_artifact=data.get("ui_artifact"), schema_actions=data.get("schema_actions"), ) diff --git a/agent_core/core/impl/event_stream/event_stream.py b/agent_core/core/impl/event_stream/event_stream.py index ee428dce..039b22b0 100644 --- a/agent_core/core/impl/event_stream/event_stream.py +++ b/agent_core/core/impl/event_stream/event_stream.py @@ -255,6 +255,7 @@ def log( platform: Optional[str] = None, continue_work: Optional[bool] = None, question: Optional[dict] = None, + ui_artifact: Optional[dict] = None, schema_actions: Optional[List[str]] = None, ) -> int: """ @@ -320,6 +321,7 @@ def log( platform=platform, continue_work=continue_work, question=question, + ui_artifact=ui_artifact, schema_actions=schema_actions, ) rec = EventRecord(event=ev) diff --git a/agent_core/core/impl/event_stream/manager.py b/agent_core/core/impl/event_stream/manager.py index 623c36ca..79960eb0 100644 --- a/agent_core/core/impl/event_stream/manager.py +++ b/agent_core/core/impl/event_stream/manager.py @@ -361,6 +361,7 @@ def log( platform: Optional[str] = None, continue_work: Optional[bool] = None, question: Optional[dict] = None, + ui_artifact: Optional[dict] = None, schema_actions: Optional[List[str]] = None, task_id: str | None = None, ) -> int: @@ -413,6 +414,7 @@ def log( platform=platform, continue_work=continue_work, question=question, + ui_artifact=ui_artifact, schema_actions=schema_actions, ) diff --git a/app/data/action/render_ui.py b/app/data/action/render_ui.py new file mode 100644 index 00000000..e7ebe18c --- /dev/null +++ b/app/data/action/render_ui.py @@ -0,0 +1,130 @@ +from agent_core import action + + +@action( + name="render_ui", + description=( + "Create a self-contained HTML document with inline CSS/JavaScript in the output workspace. " + "Choose this when manipulating inputs, exploring alternatives, inspecting changing data or " + "tracking progress materially helps the user's goal; no explicit UI request is needed. " + "Prefer text when reading suffices; never render solely for decoration or topic. " + "Respect requested formats. " + "Use semantic, keyboard-usable HTML and addEventListener; every control must work. " + "No backend, npm/imports, external scripts, inline event handlers or secrets. " + "Never navigate or request permissions; preventDefault on form submits. Other resources " + "are blocked by CSP. For API fetches declare exact public HTTPS origins in connect_origins " + "(omit offline); require credential-free browser CORS, loading/recoverable errors, actual " + "data timestamps and verified units. " + "Restore window.craftbot.state; call window.craftbot.saveState(object) on changes. " + "State is JSON: store timer deadlines, not interval counts; serialize timestamps and " + "reconstruct Dates. To revise, pass artifact_id plus complete replacement HTML; " + "preserve state keys and normalize obsolete saved enum values. You cannot inspect or " + "change saved preview state through this action. " + "Design around the main activity: useful defaults, immediate feedback, a strong focal " + "value/result and one clear primary action. Keep labeled controls/units near their results; " + "make selection and progress obvious. Use open sections, whitespace and subtle dividers " + "instead of nested bordered cards or generic dashboards. Keep advanced settings/tips " + "behind disclosures and give empty/error states actionable next steps. " + "Use 15–16px body text, deliberate hierarchy and restrained accents. For live host " + "light/dark styling use --cb-bg, --cb-surface, --cb-text, --cb-muted, --cb-border, " + "--cb-accent and --cb-font; bespoke palettes must consistently style page/text/controls " + "without host-token fallbacks overriding them. " + "Fit a full-height, resizable 440–900px canvas and usable 360px layout using document-width " + "responsive CSS. The host supplies title, phone/fullscreen/revision controls: don't duplicate " + "them, add faux browser chrome or repeat the title with an intro. " + "Use purposeful 150–250ms opacity/transform transitions; honor prefers-reduced-motion. " + "No perpetual decoration, ticking-number animation or delayed controls. " + "Keep code concise with shared styles/functions so the complete document fits one response. " + "After rendering, send at most one brief usage sentence; don't re-ask for values adjustable " + "in the interface." + ), + default=True, + action_sets=["core"], + parallelizable=False, + input_schema={ + "title": { + "type": "string", + "description": "Short interface title (1–120 characters).", + }, + "html": { + "type": "string", + "description": "Complete HTML/CSS/JS, at most 512 KB.", + }, + "artifact_id": { + "type": "string", + "description": "Optional ID returned by an earlier render_ui in this session; omit for a new interface.", + }, + "connect_origins": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional list of at most 8 exact public HTTPS API origins (default HTTPS port). No paths, query strings, credentials, IP addresses, local hosts or wildcards. Omit for offline output.", + }, + }, + output_schema={ + "status": {"type": "string"}, + "artifact_id": {"type": "string"}, + "revision": {"type": "integer"}, + }, + test_payload={ + "title": "Counter", + "html": "", + "simulated_mode": True, + }, +) +async def render_ui(input_data: dict) -> dict: + # CraftBot executes the serialized function body in a fresh namespace. + import asyncio + + from agent_core.core.session import MAIN_SESSION_ID + from app.generative_ui import make_artifact + + try: + artifact = make_artifact(input_data) + except ValueError as exc: + return {"status": "error", "message": str(exc)} + + if not input_data.get("simulated_mode", False): + from app.internal_action_interface import InternalActionInterface as iai + + session_id = input_data.get("_session_id") or MAIN_SESSION_ID + if input_data.get("artifact_id"): + streams = iai.state_manager.event_stream_manager + if not streams.has_stream(session_id): + return { + "status": "error", + "message": "This session has no UI artifact stream.", + } + stream = streams.get_stream_by_id(session_id) + previous = [ + event.ui_artifact + for event in stream.as_list() + if event.ui_artifact and event.ui_artifact.get("id") == artifact["id"] + ] + if not previous: + from app.usage.chat_storage import get_chat_storage + + stored = await asyncio.to_thread( + get_chat_storage().get_latest_ui_artifact, + session_id, + artifact["id"], + ) + if stored: + previous = [stored] + if not previous: + return { + "status": "error", + "message": "No artifact with this ID exists in this session.", + } + artifact["revision"] = max(a["revision"] for a in previous) + 1 + await iai.do_chat( + artifact["title"], + platform="CraftBot Interface", + session_id=session_id, + continue_work=True, + ui_artifact=artifact, + ) + return { + "status": "success", + "artifact_id": artifact["id"], + "revision": artifact["revision"], + } diff --git a/app/generative_ui.py b/app/generative_ui.py new file mode 100644 index 00000000..e6ea32f5 --- /dev/null +++ b/app/generative_ui.py @@ -0,0 +1,73 @@ +"""The frontend-only artifact contract. Browser isolation lives in the renderer.""" + +import re +import uuid + +MAX_HTML_BYTES = 512_000 +MAX_CONNECT_ORIGINS = 8 + + +def normalize_connect_origins(value: object) -> list[str]: + """Accept exact public HTTPS origins, never CSP expressions or local hosts.""" + if not isinstance(value, list) or len(value) > MAX_CONNECT_ORIGINS: + raise ValueError("connect_origins must be a list of at most 8 HTTPS origins.") + origins = set() + for origin in value: + match = ( + re.fullmatch(r"https://([A-Za-z0-9.-]+)(?::443)?/?", origin) + if isinstance(origin, str) + else None + ) + host = match[1].lower() if match else "" + labels = host.split(".") + if ( + len(host) > 253 + or len(labels) < 2 + or not re.fullmatch(r"[a-z]{2,63}", labels[-1]) + or any( + not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) + for label in labels + ) + or labels[-1] + in { + "localhost", + "local", + "internal", + "lan", + "home", + "corp", + "arpa", + "onion", + "test", + "invalid", + } + ): + raise ValueError( + "connect_origins must contain exact public HTTPS origins, without paths, credentials or wildcards." + ) + origins.add(f"https://{host}") + return sorted(origins) + + +def make_artifact(data: dict, revision: int = 1) -> dict: + title = data.get("title") + html = data.get("html") + artifact_id = data.get("artifact_id") or uuid.uuid4().hex + connect_origins = normalize_connect_origins(data.get("connect_origins", [])) + if not isinstance(title, str) or not 1 <= len(title.strip()) <= 120: + raise ValueError("title must contain 1–120 characters.") + if not isinstance(html, str) or not html.strip(): + raise ValueError("html must be a complete HTML document.") + if len(html.encode("utf-8")) > MAX_HTML_BYTES: + raise ValueError(f"html exceeds {MAX_HTML_BYTES} bytes.") + if not isinstance(artifact_id, str) or not re.fullmatch( + r"[A-Za-z0-9_-]{1,80}", artifact_id + ): + raise ValueError("artifact_id must contain 1–80 letters, digits, _ or -.") + return { + "id": artifact_id, + "title": title.strip(), + "html": html, + "revision": revision, + "connect_origins": connect_origins, + } diff --git a/app/internal_action_interface.py b/app/internal_action_interface.py index e3cd5b9a..5db489b8 100644 --- a/app/internal_action_interface.py +++ b/app/internal_action_interface.py @@ -390,6 +390,7 @@ async def do_chat( continue_work: bool = False, suggested_responses: Optional[List[str]] = None, allow_free_text: bool = True, + ui_artifact: Optional[dict] = None, ) -> None: """Record an agent-authored chat message to the event stream. @@ -421,6 +422,7 @@ async def do_chat( continue_work=continue_work, suggested_responses=suggested_responses, allow_free_text=allow_free_text, + ui_artifact=ui_artifact, ) @staticmethod diff --git a/app/state/state_manager.py b/app/state/state_manager.py index c30660d9..93bd0bbb 100644 --- a/app/state/state_manager.py +++ b/app/state/state_manager.py @@ -138,6 +138,7 @@ def record_agent_message( continue_work: bool = False, suggested_responses: Optional[List[str]] = None, allow_free_text: bool = True, + ui_artifact: Optional[dict] = None, ) -> None: """Record an agent message to a session's event stream. @@ -182,6 +183,7 @@ def record_agent_message( platform=platform, continue_work=continue_work, question=question, + ui_artifact=ui_artifact, task_id=target, ) diff --git a/app/ui_layer/adapters/base.py b/app/ui_layer/adapters/base.py index 1a2e120e..c61949f9 100644 --- a/app/ui_layer/adapters/base.py +++ b/app/ui_layer/adapters/base.py @@ -312,6 +312,7 @@ def _handle_agent_message(self, event: UIEvent) -> None: continue_work=bool(event.data.get("continue_work", False)), is_question=is_question, allow_free_text=allow_free_text, + ui_artifact=event.data.get("ui_artifact"), ) ) @@ -447,6 +448,7 @@ async def _display_chat_message( is_question: bool = False, allow_free_text: bool = True, details: Optional[str] = None, + ui_artifact: Optional[dict] = None, ) -> None: """ Display a chat message. @@ -485,6 +487,7 @@ async def _display_chat_message( # bubble must not add the "Please select a response" banner. requires_choice=not is_question, details=details, + ui_artifact=ui_artifact, ) ) diff --git a/app/ui_layer/adapters/browser_adapter.py b/app/ui_layer/adapters/browser_adapter.py index 05d87fed..7c473f7e 100644 --- a/app/ui_layer/adapters/browser_adapter.py +++ b/app/ui_layer/adapters/browser_adapter.py @@ -342,6 +342,7 @@ def _stored_to_chat_message(stored) -> ChatMessage: allow_free_text=stored.allow_free_text, requires_choice=not stored.is_question, details=stored.details, + ui_artifact=stored.ui_artifact, ) async def append_message(self, message: ChatMessage) -> None: @@ -386,6 +387,7 @@ async def append_message(self, message: ChatMessage) -> None: is_question=message.is_question, allow_free_text=message.allow_free_text, details=message.details, + ui_artifact=message.ui_artifact, ) await self.run_storage(self._storage.insert_message, stored) except Exception: diff --git a/app/ui_layer/browser/frontend/package-lock.json b/app/ui_layer/browser/frontend/package-lock.json index 1ba478c3..6b87f425 100644 --- a/app/ui_layer/browser/frontend/package-lock.json +++ b/app/ui_layer/browser/frontend/package-lock.json @@ -14,6 +14,7 @@ "driver.js": "^1.8.0", "i18next": "^23.16.8", "lucide-react": "^0.344.0", + "parse5": "^8.0.1", "prism-react-renderer": "^2.4.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -26,6 +27,7 @@ "remark-gfm": "^4.0.0" }, "devDependencies": { + "@playwright/test": "^1.64.0", "@types/react": "^18.2.55", "@types/react-dom": "^18.2.19", "@types/react-grid-layout": "^1.3.5", @@ -978,6 +980,22 @@ "node": ">= 8" } }, + "node_modules/@playwright/test": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.0.tgz", + "integrity": "sha512-oTN9FNYuN6bqQMwiho6jJKLCG2/QOYCixGXuCjRmzOq23eODeukuY2XiAVoVpAECbuezMt1MHQjAiQxK8F+Y+g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@reduxjs/toolkit": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", @@ -2226,6 +2244,18 @@ "dev": true, "license": "ISC" }, + "node_modules/entities": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "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/fb55/entities?sponsor=1" + } + }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -4248,6 +4278,18 @@ "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", "license": "MIT" }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "license": "MIT", + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "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/inikulin/parse5?sponsor=1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4308,6 +4350,35 @@ "url": "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/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz", + "integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.8", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", diff --git a/app/ui_layer/browser/frontend/package.json b/app/ui_layer/browser/frontend/package.json index c14323fa..9a6947d6 100644 --- a/app/ui_layer/browser/frontend/package.json +++ b/app/ui_layer/browser/frontend/package.json @@ -7,6 +7,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", + "test:generative-ui": "playwright test", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0" }, "dependencies": { @@ -16,6 +17,7 @@ "driver.js": "^1.8.0", "i18next": "^23.16.8", "lucide-react": "^0.344.0", + "parse5": "^8.0.1", "prism-react-renderer": "^2.4.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -28,6 +30,7 @@ "remark-gfm": "^4.0.0" }, "devDependencies": { + "@playwright/test": "^1.64.0", "@types/react": "^18.2.55", "@types/react-dom": "^18.2.19", "@types/react-grid-layout": "^1.3.5", diff --git a/app/ui_layer/browser/frontend/playwright.config.ts b/app/ui_layer/browser/frontend/playwright.config.ts new file mode 100644 index 00000000..7dbc0ab5 --- /dev/null +++ b/app/ui_layer/browser/frontend/playwright.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + testMatch: '**/*.test.ts', + fullyParallel: true, + use: { baseURL: 'http://127.0.0.1:7935', headless: true, trace: 'retain-on-failure' }, + webServer: { command: 'npm run dev -- --host 127.0.0.1 --port 7935', url: 'http://127.0.0.1:7935/tests/generative-ui.html', reuseExistingServer: !process.env.CI }, +}) diff --git a/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx b/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx index 17da1cb6..3ba10714 100644 --- a/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx +++ b/app/ui_layer/browser/frontend/src/components/Chat/Chat.tsx @@ -11,6 +11,7 @@ import { useToast } from '../../contexts/ToastContext' import { SlashCommandAutocomplete, AttachmentPreviewModal, PlaybookModal } from '../ui' import type { SlashCommandAutocompleteHandle } from '../ui' import { ChatMessageItem } from '../../pages/Chat/ChatMessage' +import { OutputWorkspace } from '../GenerativeUI/OutputWorkspace' import { TypingIndicatorRow } from '../../pages/Chat/TypingIndicator' import { ReasoningBlock, ActionBlock, ChunkHeaderRow } from '../activity/ActivityBlocks' import { normalizeActionName } from '../activity/actionNames' @@ -1255,6 +1256,7 @@ export function Chat({ sessionId, placeholder }: ChatProps) { } return ( +
@@ -1668,5 +1670,6 @@ export function Chat({ sessionId, placeholder }: ChatProps) { /> setPlaybookOpen(false)} />
+ ) } diff --git a/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.module.css b/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.module.css new file mode 100644 index 00000000..ba9011d6 --- /dev/null +++ b/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.module.css @@ -0,0 +1,58 @@ +.artifact { display: flex; flex: 1; flex-direction: column; min-height: 0; min-width: 0; } +.toolbar { display: grid; grid-template-columns: minmax(0, 1fr) auto; grid-template-rows: 30px 36px; align-items: center; gap: 8px; padding: 10px 16px; border-bottom: 1px solid var(--border-primary); background: var(--bg-primary); flex-shrink: 0; } +.toolbarTitle { min-width: 0; grid-column: 1; grid-row: 1; } +.previewControls, .devices, .actions { display: flex; align-items: center; gap: 4px; } +.previewControls { gap: 10px; grid-column: 1 / -1; grid-row: 2; justify-content: space-between; } +.devices { padding: 3px; border: 1px solid var(--border-primary); border-radius: 8px; background: var(--bg-secondary); gap: 2px; } +.toolbar button, .menu summary { display: flex; align-items: center; justify-content: center; gap: 6px; min-height: 28px; padding: 4px 8px; border: 0; border-radius: 5px; background: transparent; color: var(--text-secondary); font-size: 12px; cursor: pointer; white-space: nowrap; } +.devices button[aria-pressed=true] { background: var(--bg-primary); box-shadow: 0 1px 3px rgba(0,0,0,.12); color: var(--text-primary); } +.toolbar button:hover, .menu summary:hover { background: var(--bg-hover); color: var(--text-primary); } +.toolbar button:focus-visible, .menu summary:focus-visible, .resume:focus-visible, .error button:focus-visible { outline: 2px solid var(--text-secondary); outline-offset: 2px; } +.toolbarActions { display: flex; justify-content: flex-end; grid-column: 2; grid-row: 1; } +.canvas { flex: 1; display: flex; justify-content: center; min-height: 0; min-width: 0; padding: 16px; background: var(--bg-secondary); } +.preview { position: relative; display: flex; flex-direction: column; width: 100%; max-width: 100%; min-height: 0; overflow: hidden; border: 1px solid var(--border-primary); border-radius: 12px; background: var(--bg-primary); box-shadow: 0 4px 20px rgba(0,0,0,.035); transition: max-width 280ms cubic-bezier(.22, 1, .36, 1), border-radius 280ms ease, box-shadow 280ms ease; } +.phone { padding: 20px; } +.phone .preview { max-width: 390px; border-radius: 24px; box-shadow: 0 12px 36px rgba(0,0,0,.1); } +.frame { flex: 1; display: block; border: 0; width: 100%; min-height: 0; height: 100%; } +.menu { position: relative; } +.menu summary { list-style: none; } +.menu summary::-webkit-details-marker { display: none; } +.menuItems { position: absolute; top: calc(100% + 9px); right: 0; z-index: 5; display: grid; gap: 3px; width: 240px; padding: 6px; border: 1px solid var(--border-primary); border-radius: 10px; background: var(--bg-elevated); box-shadow: var(--shadow-md); } +.menuItems button { justify-content: flex-start; padding: 9px 10px; } +.resetConfirmation { padding: 8px; font-size: 13px; color: var(--text-primary); } +.resetConfirmation p { margin: 6px 0 14px; line-height: 1.5; font-size: 12px; color: var(--text-secondary); } +.resetConfirmation > div { display: flex; justify-content: flex-end; gap: 6px; } +.resetConfirmation .destructive { color: var(--color-error); background: var(--color-error-bg); } +.loading { position: absolute; inset: 0; z-index: 1; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 14px; background: var(--bg-primary); color: var(--text-secondary); font-size: 13px; pointer-events: none; } +.loading svg { animation: spin 1s linear infinite; } +.paused { display: flex; flex: 1; flex-direction: column; align-items: center; justify-content: center; padding: 28px; text-align: center; } +.statusIcon { display: grid; place-items: center; width: 54px; height: 54px; border-radius: 16px; background: var(--bg-secondary); color: var(--text-secondary); } +.paused h3 { margin: 18px 0 8px; font-size: 17px; font-weight: 600; color: var(--text-primary); } +.paused p { margin: 0 0 22px; max-width: 280px; line-height: 1.6; color: var(--text-secondary); font-size: 13px; } +.resume, .error button { display: flex; align-items: center; justify-content: center; gap: 7px; border: 1px solid var(--border-primary); border-radius: 8px; padding: 9px 14px; background: var(--bg-secondary); color: var(--text-primary); font: inherit; font-size: 13px; cursor: pointer; } +.resume:hover, .error button:hover { background: var(--bg-hover); } +.error { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border-bottom: 1px solid var(--border-primary); background: var(--color-warning-bg); color: var(--text-primary); font-size: 12px; } +.error > svg { flex-shrink: 0; color: var(--color-warning); } +.error > div { flex: 1; min-width: 0; } +.error summary { margin-top: 5px; color: var(--text-secondary); cursor: pointer; } +.error p { margin: 8px 0 0; white-space: pre-wrap; overflow-wrap: anywhere; } +.error button { flex-shrink: 0; padding: 5px 8px; font-size: 12px; } +@keyframes spin { to { transform: rotate(360deg); } } +.canvas { transition: padding 280ms cubic-bezier(.22, 1, .36, 1); } +.toolbar button, .menu summary, .resume, .error button { transition: background-color 140ms ease, color 140ms ease, box-shadow 140ms ease; } +.menu[open] .menuItems { animation: menuEnter 160ms cubic-bezier(.22, 1, .36, 1); transform-origin: top right; } +.frameReady, .paused, .error, .resetConfirmation { animation: previewReveal 200ms ease-out; } +@keyframes menuEnter { from { opacity: 0; transform: translateY(-5px) scale(.97); } to { opacity: 1; transform: translateY(0) scale(1); } } +@keyframes previewReveal { from { opacity: 0; } to { opacity: 1; } } +@media (prefers-reduced-motion: reduce) { + .loading svg, .menu[open] .menuItems, .frameReady, .paused, .error, .resetConfirmation { animation: none; } + .preview, .canvas, .toolbar button, .menu summary, .resume, .error button { transition: none; } +} +@container output-pane (max-width: 640px) { + .toolbar { padding: 10px 16px; } + .canvas { padding: 10px; } + .phone { padding: 12px; } +} +@media (max-width: 768px) { + .toolbarTitle { padding-left: 36px; } +} diff --git a/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.tsx b/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.tsx new file mode 100644 index 00000000..1644c339 --- /dev/null +++ b/app/ui_layer/browser/frontend/src/components/GenerativeUI/GenerativeUI.tsx @@ -0,0 +1,135 @@ +import { useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode } from 'react' +import { useTranslation } from 'react-i18next' +import { Monitor, Smartphone, MoreHorizontal, RotateCcw, RefreshCw, Pause, Play, Loader2, AlertCircle } from 'lucide-react' +import type { GenerativeUIArtifact } from '../../types' +import { usePersistedState } from '../../hooks' +import { UI_STATE } from '../../store/uiState' +import { BRIDGE_CHANNEL, buildDocument, readHostTheme, validState } from './runtime' +import styles from './GenerativeUI.module.css' + +interface Props { artifact: GenerativeUIArtifact; sessionId: string; toolbarTitle?: ReactNode; toolbarActions?: ReactNode } + +export function GenerativeUI({ artifact, sessionId, toolbarTitle, toolbarActions }: Props) { + const { t } = useTranslation('chat') + const timeoutText = useRef('') + timeoutText.current = t('generativeUI.timeout') + const [state, saveState] = usePersistedState(UI_STATE.chat.artifactState(JSON.stringify([sessionId, artifact.id]))) + const stateRef = useRef(state) + stateRef.current = state + const frame = useRef(null) + const options = useRef(null) + const [attempt, setAttempt] = useState(0) + const [viewport, setViewport] = useState<'desktop' | 'phone'>('desktop') + const [paused, setPaused] = useState(false) + const [status, setStatus] = useState<'loading' | 'ready' | 'error'>('loading') + const [error, setError] = useState('') + const [confirmReset, setConfirmReset] = useState(false) + useLayoutEffect(() => { + if (confirmReset) options.current?.querySelector('button')?.focus() + }, [confirmReset]) + useEffect(() => { + const dismiss = (event: PointerEvent) => { + if (options.current?.open && !options.current.contains(event.target as Node)) options.current.open = false + } + const dismissFromPreview = () => { + if (document.activeElement === frame.current && options.current) options.current.open = false + } + document.addEventListener('pointerdown', dismiss) + window.addEventListener('blur', dismissFromPreview) + return () => { document.removeEventListener('pointerdown', dismiss); window.removeEventListener('blur', dismissFromPreview) } + }, []) + // Saved state intentionally does not rebuild the document on every interaction. + const runtime = useMemo(() => { + const token = crypto.randomUUID().replace(/-/g, '') + return { token, html: buildDocument(artifact, stateRef.current, token, window.location.origin) } + }, [artifact, attempt, sessionId]) + + // Register before the frame's first queued messages can arrive. Translation + // changes must not restart readiness tracking for an already loaded frame. + useLayoutEffect(() => { + if (paused) return + setStatus('loading') + setError('') + const timer = window.setTimeout(() => { + setStatus(current => current === 'loading' ? 'error' : current) + setError(current => current || timeoutText.current) + }, 10_000) + const receive = (event: MessageEvent) => { + const data = event.data + // srcdoc sandbox frames have origin "null". Check the exact live Window, + // per-mount token and message schema instead of trusting that origin. + if (event.source !== frame.current?.contentWindow || !data || data.channel !== BRIDGE_CHANNEL || data.token !== runtime.token) return + switch (data.type) { + case 'ready': + window.clearTimeout(timer) + setStatus(current => current === 'error' ? current : 'ready') + break + case 'state': + if (validState(data.state)) saveState(data.state) + break + case 'error': + window.clearTimeout(timer) + if (typeof data.message === 'string') { setError(data.message.slice(0, 1000)); setStatus('error') } + break + } + } + window.addEventListener('message', receive) + const syncTheme = () => frame.current?.contentWindow?.postMessage({ channel: BRIDGE_CHANNEL, token: runtime.token, type: 'theme', theme: readHostTheme() }, '*') + const observer = new MutationObserver(syncTheme) + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }) + return () => { observer.disconnect(); window.clearTimeout(timer); window.removeEventListener('message', receive) } + }, [runtime, paused, saveState]) + + const restart = (reset = false) => { + if (options.current?.open) { options.current.open = false; options.current.querySelector('summary')?.focus() } + setConfirmReset(false) + if (reset) { stateRef.current = {}; saveState({}) } + setPaused(false) + setAttempt(value => value + 1) + } + const preview = ( +
+ {paused ?
+ +

{t('generativeUI.pausedTitle')}

{t('generativeUI.paused')}

+ +
: <> + {status === 'loading' &&
{t('generativeUI.loading')}
} + {status === 'error' &&
+
{t('generativeUI.error')}
{t('generativeUI.errorDetails')}

{error}

+ +
} +

`) + await expect(preview(page).locator('#result')).toHaveText('parent-blocked,storage-blocked') + await expect(page.getByRole('alert')).toContainText('Blocked resource:') + expect(outgoing).toEqual([]) + await expect(page.locator('iframe')).toHaveAttribute('sandbox', 'allow-scripts allow-forms') +}) + +test('runtime errors are visible, forged messages are ignored and oversized state is rejected', async ({ page }) => { + await open(page) + await page.evaluate(() => window.postMessage({ channel: 'craftbot-generative-ui', type: 'error', message: 'forged', token: document.querySelector('iframe')?.srcdoc.match(/const token = "([^"]+)/)?.[1] }, '*')) + await expect(page.getByRole('alert')).toHaveCount(0) + await custom(page, ``) + await expect(page.getByRole('alert')).toContainText('State exceeds 64 KB') + await custom(page, `

Broken example

`) + await expect(page.getByRole('alert')).toContainText('Example runtime failure') + await previewAction(page, 'Reload preview') + await expect(page.getByRole('alert')).toContainText('Example runtime failure') +}) + +test('document attributes and state strings survive rendering without executing injected markup', async ({ page }) => { + await open(page) + await custom(page, '

') + await expect(preview(page).locator('html')).toHaveAttribute('lang', 'fr') + await expect(preview(page).locator('body')).toHaveClass('test-body') + await expect(preview(page).locator('body')).toHaveCSS('background-color', 'rgb(1, 2, 3)') + await expect(preview(page).getByRole('button', { name: 'Save text' })).toHaveCSS('color', 'rgb(23, 21, 18)') + await preview(page).getByRole('button', { name: 'Save text' }).click() + await previewAction(page, 'Reload preview') + await expect(preview(page).locator('#value')).toHaveText('') + await expect(page.getByRole('alert')).toHaveCount(0) +}) diff --git a/app/ui_layer/browser/frontend/tests/harness.tsx b/app/ui_layer/browser/frontend/tests/harness.tsx new file mode 100644 index 00000000..2af021fd --- /dev/null +++ b/app/ui_layer/browser/frontend/tests/harness.tsx @@ -0,0 +1,61 @@ +// Development-only fixture: exercises the real chat message and preview, without +// a model/provider or a live CraftBot database. Vite does not bundle it for release. +import React, { useEffect, useState } from 'react' +import { createRoot } from 'react-dom/client' +import { configureStore } from '@reduxjs/toolkit' +import { Provider } from 'react-redux' +import ui from '../src/store/slices/uiSlice' +import agent from '../src/store/slices/agentSlice' +import { createUiPersistenceMiddleware, loadPersistedUiState } from '../src/store/uiState' +import { OutputWorkspace } from '../src/components/GenerativeUI/OutputWorkspace' +import { ChatMessageItem } from '../src/pages/Chat/ChatMessage' +import type { GenerativeUIArtifact, ChatMessage } from '../src/types' +import i18n from '../src/i18n/config' +import '../src/styles/global.css' +import cooking from './fixtures/cooking.html?raw' +import weather from './fixtures/weather.html?raw' +import design from './fixtures/design.html?raw' + +const store = configureStore({ reducer: { ui, agent }, preloadedState: { ui: loadPersistedUiState(), agent: { ...agent(undefined, { type: '@@init' }), profilePictureUrl: 'data:image/svg+xml,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"%3E%3Ccircle cx="16" cy="16" r="14" fill="%23FF4F18"/%3E%3C/svg%3E' } }, middleware: getDefault => getDefault().concat(createUiPersistenceMiddleware()) }) +const examples: Record = { + Cooking: { id: 'cooking', title: 'Cooking guide', html: cooking, revision: 1, connect_origins: [] }, + Weather: { id: 'weather', title: 'Weather explorer', html: weather, revision: 1, connect_origins: ['https://api.open-meteo.com', 'https://geocoding-api.open-meteo.com'] }, + Design: { id: 'design', title: 'Design playground', html: design, revision: 1, connect_origins: [] }, +} +function Harness() { + const [artifact, setArtifact] = useState(examples.Cooking) + const [visible, setVisible] = useState(true) + const [messageVisible, setMessageVisible] = useState(true) + const [history, setHistory] = useState([]) + const [sessionId, setSessionId] = useState('demo') + useEffect(() => { + const receive = (event: Event) => setArtifact((event as CustomEvent).detail) + const receiveHistory = (event: Event) => setHistory((event as CustomEvent).detail) + window.addEventListener('fixture-history', receiveHistory) + const changeLanguage = (event: Event) => { void i18n.changeLanguage((event as CustomEvent).detail) } + window.addEventListener('fixture-artifact', receive) + window.addEventListener('fixture-language', changeLanguage) + return () => { + window.removeEventListener('fixture-artifact', receive) + window.removeEventListener('fixture-history', receiveHistory) + window.removeEventListener('fixture-language', changeLanguage) + } + }, []) + const messages: ChatMessage[] = [...history, artifact].map((item, index) => ({ sender: 'CraftBot', content: item.title, style: 'agent', timestamp: index + 1, messageId: item.id + item.revision, sessionId, uiArtifact: item })) + return
+

CraftBot · development preview

+

An answer you can use.

+

Explore three frontend-only answers. Every control runs in the preview.

+ +
+ {visible &&
{messageVisible && messages.map(message => {}} onOpenFolder={() => {}} />)}
} +
+
+} +createRoot(document.getElementById('root')!).render() diff --git a/app/ui_layer/components/types.py b/app/ui_layer/components/types.py index 47d98491..ea8c67ab 100644 --- a/app/ui_layer/components/types.py +++ b/app/ui_layer/components/types.py @@ -123,6 +123,7 @@ class ChatMessage: # e.g. the raw body of an incoming integration message on the # "📩 Incoming …" system stub (PR #419). details: Optional[str] = None + ui_artifact: Optional[dict] = None def __post_init__(self) -> None: """Generate message_id if not provided; normalize session id.""" @@ -181,6 +182,8 @@ def to_dict(self) -> dict: data["allowFreeText"] = self.allow_free_text if self.details: data["details"] = self.details + if self.ui_artifact: + data["uiArtifact"] = self.ui_artifact return data diff --git a/app/ui_layer/events/transformer.py b/app/ui_layer/events/transformer.py index 0df2c94a..40392263 100644 --- a/app/ui_layer/events/transformer.py +++ b/app/ui_layer/events/transformer.py @@ -116,6 +116,7 @@ def _build_agent_message( "session_id": session_id, "continue_work": bool(event.continue_work), "question": event.question, + "ui_artifact": event.ui_artifact, }, timestamp=ts, task_id=session_id, diff --git a/app/usage/chat_storage.py b/app/usage/chat_storage.py index 24200b3f..89673540 100644 --- a/app/usage/chat_storage.py +++ b/app/usage/chat_storage.py @@ -27,7 +27,7 @@ _ROW_COLUMNS = ( "message_id, sender, content, style, timestamp, attachments, " "session_id, options, option_selected, continue_work, " - "is_question, allow_free_text, details" + "is_question, allow_free_text, details, ui_artifact" ) @@ -57,6 +57,7 @@ class StoredChatMessage: # Expandable payload behind a disclosure (e.g. the raw body of an # incoming integration message on the "📩 Incoming …" system stub). details: Optional[str] = None + ui_artifact: Optional[Dict[str, Any]] = None def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for JSON serialization.""" @@ -81,6 +82,8 @@ def to_dict(self) -> Dict[str, Any]: result["allowFreeText"] = self.allow_free_text if self.details: result["details"] = self.details + if self.ui_artifact: + result["uiArtifact"] = self.ui_artifact return result @@ -99,6 +102,7 @@ def _row_to_message(row) -> StoredChatMessage: is_question=bool(row[10]), allow_free_text=bool(row[11]), details=row[12], + ui_artifact=json.loads(row[13]) if row[13] else None, ) @@ -197,6 +201,8 @@ def _init_db(self) -> None: ) if "details" not in columns: cursor.execute("ALTER TABLE chat_messages ADD COLUMN details TEXT") + if "ui_artifact" not in columns: + cursor.execute("ALTER TABLE chat_messages ADD COLUMN ui_artifact TEXT") cursor.execute(""" CREATE INDEX IF NOT EXISTS idx_chat_session @@ -220,8 +226,8 @@ def insert_message(self, message: StoredChatMessage) -> int: cursor.execute( """ INSERT OR REPLACE INTO chat_messages - (message_id, sender, content, style, timestamp, attachments, session_id, options, option_selected, continue_work, is_question, allow_free_text, details) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (message_id, sender, content, style, timestamp, attachments, session_id, options, option_selected, continue_work, is_question, allow_free_text, details, ui_artifact) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( message.message_id, @@ -237,11 +243,26 @@ def insert_message(self, message: StoredChatMessage) -> int: 1 if message.is_question else 0, 1 if message.allow_free_text else 0, message.details, + json.dumps(message.ui_artifact) if message.ui_artifact else None, ), ) conn.commit() return cursor.lastrowid + def get_latest_ui_artifact( + self, session_id: str, artifact_id: str + ) -> Optional[dict]: + """Find a revision even after its event has been folded out of context.""" + with sqlite3.connect(self._db_path) as conn: + row = conn.execute( + "SELECT ui_artifact FROM chat_messages " + "WHERE session_id = ? AND ui_artifact IS NOT NULL " + "AND json_extract(ui_artifact, '$.id') = ? " + "ORDER BY json_extract(ui_artifact, '$.revision') DESC LIMIT 1", + (session_id, artifact_id), + ).fetchone() + return json.loads(row[0]) if row else None + def get_messages( self, session_id: Optional[str] = None, diff --git a/docs/generative-ui.md b/docs/generative-ui.md new file mode 100644 index 00000000..592b9a1e --- /dev/null +++ b/docs/generative-ui.md @@ -0,0 +1,283 @@ +# Generative UI (issue #479) + +Generative UI turns an agent answer into an interactive HTML/CSS/JavaScript +experience in a dedicated output workspace beside the conversation. The generated document requires no backend, database, +package installation, build, or server process. CraftBot's existing backend +records the artifact alongside the chat message. + +## Agent contract + +The agent chooses the output format based on the user's goal. Use `render_ui` +when manipulating inputs, exploring alternatives, inspecting changing information, +or tracking progress materially helps accomplish the task. Prefer text when +reading alone meets the need, and respect an explicitly requested output format. +Topic categories do not trigger UI generation; the examples in this document are +test cases, not a list of supported or preferred use cases. + +Call `render_ui` with `title`, `html`, and optionally `connect_origins` listing +the exact public HTTPS API origins needed by that output. +The action returns `artifact_id` and `revision`. To revise an existing answer, +send its ID with the complete replacement document in the same session. +Older revisions remain in history and can be selected in the output workspace. Revision lookup uses the session's event +stream and falls back to saved chat history after context folding or restart. + +HTML is capped at 512,000 UTF-8 bytes. Titles are capped at 120 characters. +Use inline CSS and JavaScript, `addEventListener`, semantic controls, responsive +layouts, and loading/error states. External scripts, module dependencies, +inline event attributes, browser permissions, and API secrets are unsupported. +For forms, handle `submit` and call `preventDefault()`; no form is sent externally. + +The conversation contains a compact Open output reference. One selected document +runs in a separate full-height output pane, independently of virtualized message +rows. New outputs open automatically; older revisions are selectable. Closing +the pane returns to conversation and stays closed across reloads for that session, +until the user reopens it or a new output arrives. Loading older history does not +reopen it. Desktop shows conversation and output side by side; narrow screens +switch between them. Phone preview and fullscreen resize the canvas without +reloading the document. Drag the divider to adjust the conversation/output split; +Arrow keys, Home and End also resize it, and double-click restores the default. +The split is remembered as a browser preference. At workspace widths below 860px, +the output takes the available space instead of squeezing both views together. +One adaptive header holds the output title, device preview, revision selector, +reload and panel controls. The preview sits on a separate inset canvas. +Pause shows a direct Resume button and retains saved progress. Reset asks for an +in-panel confirmation before clearing only that output's state. Preview errors +show a retry action with technical details behind a disclosure. +Panel entry/exit and fullscreen animate the shared grid tracks, so the chat, +divider, output and its toolbar move together. The toolbar keeps the same two-row +composition in split and fullscreen views. Preview sizing and menus use short eased motion. +Resizing by dragging stays immediate, and existing document state is retained. +Motion is disabled when the browser requests reduced motion. Generated documents +are also instructed to honor that preference and keep controls responsive. + +Generated content receives a small base stylesheet and the host's public design +tokens: `--cb-bg`, `--cb-surface`, `--cb-text`, `--cb-muted`, `--cb-border`, +`--cb-accent`, and `--cb-font`. Use these for neutral surfaces, typography and +controls so light/dark mode stays consistent with CraftBot. Theme updates cross +the validated bridge without reloading the document or resetting progress. +`window.craftbot.theme` exposes the same visual tokens, without other host state. +The generation guidance favors deliberate typography, a strong focal point, +task-specific compositions, and progressive disclosure instead of nested +dashboards or collections of forms. Custom design experiments can still +choose their own palettes. Existing artifacts with hardcoded colors need a +revision to adopt the tokens; the runtime does not overwrite their authored CSS. + +The runtime supplies a small synchronous API before generated scripts run: + +```javascript +const state = { servings: 2, completedSteps: [], ...window.craftbot.state }; +servingsInput.addEventListener('change', () => { + state.servings = Number(servingsInput.value); + window.craftbot.saveState(state); +}); +``` + +State must be a JSON object of at most 65,536 UTF-8 bytes. Save changes when +they happen; state is not scraped from the DOM. Use absolute deadlines for +timers, rather than counting interval callbacks. Dates must be stored as JSON +timestamps and reconstructed on restore. State is shared by an artifact's +revisions in one chat session; preserve compatible state keys when revising. + +State lives in CraftBot's Redux UI store and sessionStorage. It survives chat +navigation, virtualization/remounting, expansion, preview reloads, and page +reloads in the same browser tab. Closing the tab clears this first version's +progress; artifact code and metadata remain in SQLite chat history. Reset clears +progress for that session/artifact. Pause removes the iframe; resume restores +saved state, and does not guarantee background timers or notifications. + +## Data access + +Offline interfaces receive `connect-src 'none'`. An output may declare up to eight +exact public HTTPS API origins in `connect_origins`. These become its CSP +connection allowlist; undeclared origins and redirects to them stay blocked. +Origins use the default HTTPS port and contain no paths, query strings, +credentials, IP addresses, local hostnames, or wildcard expressions. The backend +normalizes and validates the list; the renderer independently validates persisted +metadata and fails closed if it is invalid. No providers or task categories are +hardcoded in the action or renderer. APIs must permit browser CORS without +private credentials. Other resource types remain blocked as described below. +Older artifacts without `connect_origins` load offline. + +This is a browser connection policy, not a DNS firewall. Hostname validation +does not resolve DNS or prevent public-looking names resolving to private +addresses. Stronger network isolation remains part of hostile-code hardening. + +## Runtime boundaries and remaining hardening + +The preview uses an opaque-origin iframe with `allow-scripts allow-forms` and no +same-origin, popup, top-navigation, download, camera, microphone, geolocation, +or clipboard delegation. `allow-forms` enables local submit events; +`form-action 'none'` independently blocks network submission. + +Host markup parsing uses parse5's pure syntax tree, which does not start image or +frame requests and preserves document/body attributes. Policy overrides, nested frames, external scripts, and inline +event handlers are removed before preview insertion. A CSP precedes generated +content: external scripts, frames, workers, fonts, media and other resource +types are blocked; images can use only data/blob sources. Inline scripts and +styles are supported, but JavaScript evaluation through eval/Function is not. +Bridge messages must come from the exact live iframe window and carry its +per-mount token; payload types and state sizes are checked. No host credentials +or arbitrary tool calls are exposed. Runtime failures appear above the preview. + +**This is an initial browser runtime, not a complete hostile-code sandbox.** +An iframe can navigate its own browsing context through JavaScript; current +browser CSP does not provide a portable way to prohibit all such requests. +Intercepting links and stripping refresh metadata prevents ordinary navigation, +but is not an adversarial egress guarantee. An infinite loop can also hang the +browser renderer and prevent its pause/reload controls from responding. +Before claiming strict network containment and bounded CPU execution (and closing +the corresponding requirements in #479), choose an execution environment that +can enforce navigation and resource limits, or restrict generated behavior to a +trusted component/action vocabulary. These limits must not be described as solved +by iframe sandbox flags or a host watchdog. + +The implementation favors generated HTML for custom interactions. Alternatives +researched include [AI SDK's tool-to-component pattern](https://ai-sdk.dev/docs/ai-sdk-ui/generative-user-interfaces), +[A2UI's declarative component protocol](https://a2ui.org/introduction/what-is-a2ui/), +and [OpenUI's controlled component libraries](https://github.com/thesysdev/openui). +Those trade free-form behavior for more controlled execution. + +## Development preview and verification + +From `app/ui_layer/browser/frontend`: + +```sh +npm ci +npm run dev -- --host 127.0.0.1 --port 7935 +# Open http://127.0.0.1:7935/tests/generative-ui.html + +npx playwright install chromium +npm run test:generative-ui +npx tsc --noEmit +npm run build +``` + +The development-only harness uses the actual chat message and artifact renderer +with an isolated Redux store. It does not start a model, write a live database, +or enter the production build. Its cooking, weather and design documents live +under `frontend/tests/fixtures/`. The weather fixture calls the live service when +used manually; automated tests intercept responses to cover success and failure +deterministically. + +Python regression checks from the repository root: + +```sh +python -m pytest tests/test_generative_ui.py tests/test_chat_storage_sessions.py tests/test_ui_event_delivery.py tests/test_browser_ui_buffers.py -q +ruff check . +python -m compileall -q app agent_core agents decorators skills +``` + +Coverage includes artifact validation, event transport, additive SQLite migration, +session scoping, revision lookup, cooking progress/timers, expand/reset/pause/retry, +page reload, local temperature conversion, API failure states, design interaction, +keyboard controls, narrow layouts, parent/storage isolation, external resource +blocking, forged bridge messages, oversized state and visible runtime errors. +This does not substitute for a hostile-code containment audit or broad model +evaluation. + +## Live agent evaluation — 8 October 2026 + +Ran the production browser interface on localhost:7945 with the configured +OpenAI `gpt-5.2-2025-12-11` model, using separate test history and no connected +MCP servers. The messages below never named Generative UI, `render_ui`, HTML, +JavaScript, or the artifact runtime. The agent received its normal core action +catalog and chose the response format itself. + +| Natural request | Agent's choice | Live checks | +| --- | --- | --- | +| Help me cook tomato pasta for two, track completed steps, and adjust quantities if a friend joins. | `render_ui`, offline cook-along | Selecting 3 servings changed pasta from 200 g to 300 g and tomatoes from 400 g to 600 g. Checklist completion, notes, servings and an active timer survived a browser reload. | +| Check London weather for a walk, with Celsius/Fahrenheit and a way to check again before leaving. | `render_ui` with `weather` | Current conditions and a six-hour forecast loaded from the live API. Refresh updated the fetch time. Both temperature units appeared together. | +| Compare warm orange/muted green themes, heading sizes, spacing, and mobile/desktop for a coffee-shop landing page experiment. | `render_ui`, offline design experiment | Palette, heading-size and spacing controls changed the preview. The mobile selector updated the device state. | +| What's the capital of Portugal? Please answer in one sentence. | `send_message`, plain text | Replied: “The capital of Portugal is Lisbon.” No artifact was created. | + +The first cooking run exposed a runtime integration bug: action registration +serializes only the function body, so module-level helper imports were absent +in the executor's namespace. Moved the imports inside `render_ui` and changed +the transport/history regression test to execute the registry's serialized +action through `ActionExecutor`. The fresh cooking run then succeeded. + +A production reload also exposed a false timeout banner on a functioning +preview during history hydration. Each generated document now receives a fresh +iframe, the message listener registers in a layout effect, and a load handshake +recovers an early ready message. Translation synchronization also no longer +restarts readiness tracking for an already loaded iframe. Browser regressions +replace artifacts during hydration and change the app language, then advance +beyond the timeout to verify the previews remain ready. + +The first generated weather document incorrectly treated the API's default +wind speed as m/s. A natural follow-up asking the agent to double-check the +units produced revision 2 with corrected km/h and mph values. Expansion also +loaded that revision successfully. This demonstrates revision delivery, but +also a remaining generated-content quality risk: a functioning interface can +misinterpret API data. [Open-Meteo's unit parameters](https://open-meteo.com/en/docs) +and returned unit metadata should be checked before conversions. + +The cooking agent also asked a redundant text question about servings after +providing a working servings selector. Further evaluation should cover these +handoff choices, first-generation API accuracy, and whether the agent avoids +claiming to observe or change preview state it cannot access. + +## Presentation revision + +After feedback on both the chat framing and generated content, the host wrapper +was reduced to the answer itself and a small footer. Theme tokens and a base +stylesheet now help answers blend into the conversation. Generation guidance +explicitly targets the inline container width and favors plain lists, a clear +primary action, and optional details behind disclosures. + +A natural follow-up in the live cooking conversation produced a single-step +layout. Its first revision still stacked into a 960px preview at the actual +708px chat width. A further refinement produced a 386px preview with compact +ingredients and timer beside the current step, optional notes, and a single +Done + Next action. Advancing a step and changing to three servings worked, +and the values survived a page reload. Switching between light and dark themes +updated the answer without losing progress; the timer also started successfully. +These results also show that design guidance improves defaults but does not +guarantee the quality of every generated layout. + +## Dedicated output workspace + +Further feedback replaced inline rendering with a separate output pane. Chat +keeps an Open output reference, while one selected revision runs outside the +virtualized timeline. The canvas fills the available height and can take the +entire content area or a phone-width viewport. Its selection/closed state is +scoped to the conversation, and the runtime state remains scoped to the artifact. + +Four additional browser regressions cover output lifecycle, revision switching, +viewport/fullscreen changes without document reload, and session-specific closed +state while loading older history. All 14 browser tests pass; TypeScript and the +production build pass. + +Live refinements of the pancake and Edinburgh weather examples opened as revision +2 automatically. Cooking progress, four-serving quantities, and paused timer +state survived reload. Weather selection, units and refresh worked. The original +weather document had persisted Date objects as strings, then called getTime on +restore; the agent revised its JSON restoration logic and the new version loaded +without that error. Guidance now explicitly describes JSON timestamps. + +The larger portfolio revision exceeded the model response budget repeatedly, +producing incomplete action JSON. A smaller composition was requested rather +than treating the failed generation as a usable output. Complete-document +generation still depends on the provider's output budget. + +The smaller portfolio composition succeeded. Its third revision corrected a +palette conflict between the host theme and the document's Sand/Ink themes. +Both theme buttons and project detail/back navigation worked in the live output +workspace. The final Sand view was checked full-screen. The runtime's native +controls now inherit authored text colors, covered by a browser assertion. + +After removing topic-based selection examples and layout instructions, fresh +live requests initially produced conversational answers. Describing a need for +self-paced answer checking, repeatedly changing cost inputs, or trying furniture +positions prompted the agent to select `render_ui` without naming the action or +prescribing a UI. These are observations of model decisions, not deterministic +selection guarantees. The practice set's answer feedback and final summary +worked; a revision replaced a blocked browser confirmation with an in-page reset, +which cleared the saved answers and score. The cost split updated for a one-night +guest and changed accommodation costs, and disclosed rounding differences. +The room planner required revisions for collapsed CSS sizing and a DOM-ID/state +variable initialization collision. The final revision rendered without the error; +X/Y controls moved the selected furniture, and its corrected total 160×200cm bed +footprint cleared the desk/bookcase. Desktop and phone views were inspected. +Dragging was not confirmed by the browser driver; direct position controls were +the verified way to move items. diff --git a/tests/test_generative_ui.py b/tests/test_generative_ui.py new file mode 100644 index 00000000..a1de3808 --- /dev/null +++ b/tests/test_generative_ui.py @@ -0,0 +1,250 @@ +"""Artifact validation, delivery, history migrations and session-safe revisions.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from agent_core.core.event_stream.event import Event, EventType +from agent_core.core.action_framework.registry import ActionRegistry +from agent_core.core.impl.action.executor import ActionExecutor +from agent_core.core.impl.event_stream.manager import EventStreamManager +from app.data.action.render_ui import render_ui +from app.generative_ui import MAX_HTML_BYTES, make_artifact +from app.internal_action_interface import InternalActionInterface +from app.state.state_manager import StateManager +from app.ui_layer.adapters.base import InterfaceAdapter +from app.ui_layer.adapters.browser_adapter import BrowserChatComponent +from app.ui_layer.components.types import ChatMessage +from app.ui_layer.events.transformer import EventTransformer +from app.usage.chat_storage import ChatStorage, StoredChatMessage + + +def artifact(): + return make_artifact( + {"title": "Interactive steps", "html": ""} + ) + + +@pytest.mark.parametrize( + "change", + [ + {"title": " "}, + {"title": "a" * 121}, + {"html": ""}, + {"html": 123}, + {"html": "é" * (MAX_HTML_BYTES // 2 + 1)}, + {"artifact_id": "../other"}, + {"connect_origins": "https://api.example.com"}, + {"connect_origins": [{}]}, + {"connect_origins": ["https://api.example.com"] * 9}, + ], +) +def test_rejects_invalid_artifacts(change): + with pytest.raises(ValueError): + make_artifact({"title": "Example", "html": "

Hello

", **change}) + + +@pytest.mark.parametrize( + "origin", + [ + "http://api.example.com", + "https://api.example.com/path", + "https://api.example.com?secret=value", + "https://user:pass@api.example.com", + "https://*.example.com", + "https://api.example.com; connect-src *", + "https://api.example.com\n", + "https://127.0.0.1", + "https://[::1]", + "https://localhost", + "https://app.localhost", + "https://app.local", + "https://app.internal", + "https://api.example.com:7946", + "https://-bad.example.com", + "https://api..example.com", + ], +) +def test_rejects_unsafe_connect_origins(origin): + with pytest.raises(ValueError): + make_artifact( + {"title": "API output", "html": "

Data

", "connect_origins": [origin]} + ) + + +def test_connect_origins_are_explicit_normalized_and_persisted(tmp_path): + data = make_artifact( + { + "title": "API output", + "html": "

Data

", + "connect_origins": [ + "https://API.Example.com:443/", + "https://api.example.com", + "https://data.example.org", + ], + } + ) + assert data["connect_origins"] == [ + "https://api.example.com", + "https://data.example.org", + ] + storage = ChatStorage(str(tmp_path / "api-output.db")) + storage.insert_message( + StoredChatMessage( + message_id="api-output", + sender="Agent", + content="API output", + style="agent", + timestamp=1, + session_id="api-test", + ui_artifact=data, + ) + ) + assert ( + storage.get_messages(session_id="api-test")[0].ui_artifact["connect_origins"] + == data["connect_origins"] + ) + + +def test_artifact_delivery_and_history_round_trip(tmp_path, monkeypatch): + """Exercise the production state → event → adapter → SQLite → wire path.""" + streams = EventStreamManager(llm=SimpleNamespace()) + stream = streams.create_stream("cooking") + manager = StateManager(streams) + monkeypatch.setattr(manager, "bump_event_stream", lambda: None) + monkeypatch.setattr(InternalActionInterface, "state_manager", manager) + result = asyncio.run( + ActionExecutor().execute_atomic_action( + SimpleNamespace(**ActionRegistry().find_action_by_name("render_ui")), + { + "title": "Steps", + "html": "", + "_session_id": "cooking", + }, + ) + ) + event = Event.from_dict(stream.as_list()[-1].to_dict()) + assert event.ui_artifact["id"] == result["artifact_id"] + ui_event = EventTransformer.transform(event, session_id="cooking") + component = SimpleNamespace(append_message=AsyncMock()) + # InterfaceAdapter is abstract: exercise its method on a minimal receiver. + receiver = SimpleNamespace(chat_component=component) + asyncio.run( + InterfaceAdapter._display_chat_message( + receiver, + "CraftBot", + "Steps", + "agent", + session_id="cooking", + ui_artifact=ui_event.data["ui_artifact"], + ) + ) + message = component.append_message.call_args.args[0] + storage = ChatStorage(str(tmp_path / "chat.db")) + storage.insert_message( + StoredChatMessage( + message_id=message.message_id, + sender=message.sender, + content=message.content, + style=message.style, + timestamp=message.timestamp, + session_id=message.session_id, + ui_artifact=message.ui_artifact, + ) + ) + restored = BrowserChatComponent._stored_to_chat_message( + storage.get_recent_messages(session_id="cooking")[0] + ) + assert restored.to_dict()["uiArtifact"] == message.to_dict()["uiArtifact"] + assert restored.session_id == "cooking" + assert storage.get_recent_messages(session_id="other") == [] + assert ( + storage.get_latest_ui_artifact("cooking", message.ui_artifact["id"]) + == message.ui_artifact + ) + assert storage.get_latest_ui_artifact("other", message.ui_artifact["id"]) is None + # The migration is additive/idempotent; ordinary messages retain their shape. + ChatStorage(str(tmp_path / "chat.db")) + assert "uiArtifact" not in ChatMessage("user", "Hello", "user").to_dict() + assert ( + "uiArtifact" + not in StoredChatMessage("old", "user", "Hello", "user", 0).to_dict() + ) + + +def test_render_action_revisions_are_session_scoped(monkeypatch): + original = artifact() + event = Event( + "Steps", + "agent message", + "INFO", + event_type=EventType.AGENT_MESSAGE, + ui_artifact=original, + ) + streams = { + "mine": SimpleNamespace(as_list=lambda: [event]), + "other": SimpleNamespace(as_list=lambda: []), + } + manager = SimpleNamespace( + event_stream_manager=SimpleNamespace( + get_stream_by_id=streams.get, has_stream=lambda sid: sid in streams + ) + ) + monkeypatch.setattr(InternalActionInterface, "state_manager", manager) + delivery = AsyncMock() + monkeypatch.setattr(InternalActionInterface, "do_chat", delivery) + monkeypatch.setattr( + "app.usage.chat_storage.get_chat_storage", + lambda: SimpleNamespace(get_latest_ui_artifact=lambda sid, aid: None), + ) + data = {"artifact_id": original["id"], "title": "Revised", "html": "

Revised

"} + result = asyncio.run(render_ui({**data, "_session_id": "mine"})) + assert result["revision"] == 2 + assert delivery.call_args.kwargs["ui_artifact"]["html"] == "

Revised

" + assert delivery.call_args.kwargs["session_id"] == "mine" + assert asyncio.run(render_ui({**data, "_session_id": "other"}))["status"] == "error" + assert delivery.call_count == 1 + + +def test_simulated_action_does_not_deliver(monkeypatch): + delivery = AsyncMock() + monkeypatch.setattr(InternalActionInterface, "do_chat", delivery) + assert ( + asyncio.run( + render_ui({"title": "Demo", "html": "

Hello

", "simulated_mode": True}) + )["status"] + == "success" + ) + delivery.assert_not_called() + + +def test_revision_lookup_survives_context_folding(tmp_path, monkeypatch): + original = {**artifact(), "revision": 7} + storage = ChatStorage(str(tmp_path / "chat.db")) + storage.insert_message( + StoredChatMessage( + "old", + "CraftBot", + "Steps", + "agent", + 1, + session_id="mine", + ui_artifact=original, + ) + ) + streams = SimpleNamespace( + has_stream=lambda sid: True, + get_stream_by_id=lambda sid: SimpleNamespace(as_list=lambda: []), + ) + monkeypatch.setattr( + InternalActionInterface, + "state_manager", + SimpleNamespace(event_stream_manager=streams), + ) + monkeypatch.setattr("app.usage.chat_storage.get_chat_storage", lambda: storage) + monkeypatch.setattr(InternalActionInterface, "do_chat", AsyncMock()) + data = {"artifact_id": original["id"], "title": "Revised", "html": "

Updated

"} + assert asyncio.run(render_ui({**data, "_session_id": "mine"}))["revision"] == 8 + assert asyncio.run(render_ui({**data, "_session_id": "other"}))["status"] == "error"