feat #479: add interactive Generative UI outputs

.gitignore

@@ -67,6 +67,8 @@ agent_bundle/
 **/.craftbot/
 app/data/.file_index/
 .playwright-mcp
+**/test-results/
+**/playwright-report/
 # Sidecar Node runtime (install.py downloads it when the system Node is too old for Agent App)
 runtime/
 

agent_core/core/event_stream/event.py

@@ -172,6 +172,7 @@ class Event:
     platform: Optional[str] = None
     continue_work: Optional[bool] = None
     question: Optional[Dict[str, Any]] = None
+    ui_artifact: Optional[Dict[str, Any]] = None
     schema_actions: Optional[List[str]] = None
 
     def display_text(self) -> Optional[str]:
@@ -206,6 +207,7 @@ def to_dict(self) -> Dict[str, Any]:
             "platform": self.platform,
             "continue_work": self.continue_work,
             "question": self.question,
+            "ui_artifact": self.ui_artifact,
             "schema_actions": self.schema_actions,
         }
 
@@ -247,6 +249,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Event":
             platform=data.get("platform"),
             continue_work=data.get("continue_work"),
             question=data.get("question"),
+            ui_artifact=data.get("ui_artifact"),
             schema_actions=data.get("schema_actions"),
         )
 

agent_core/core/impl/event_stream/event_stream.py

@@ -255,6 +255,7 @@ def log(
         platform: Optional[str] = None,
         continue_work: Optional[bool] = None,
         question: Optional[dict] = None,
+        ui_artifact: Optional[dict] = None,
         schema_actions: Optional[List[str]] = None,
     ) -> int:
         """
@@ -320,6 +321,7 @@ def log(
             platform=platform,
             continue_work=continue_work,
             question=question,
+            ui_artifact=ui_artifact,
             schema_actions=schema_actions,
         )
         rec = EventRecord(event=ev)

agent_core/core/impl/event_stream/manager.py

@@ -361,6 +361,7 @@ def log(
         platform: Optional[str] = None,
         continue_work: Optional[bool] = None,
         question: Optional[dict] = None,
+        ui_artifact: Optional[dict] = None,
         schema_actions: Optional[List[str]] = None,
         task_id: str | None = None,
     ) -> int:
@@ -413,6 +414,7 @@ def log(
             platform=platform,
             continue_work=continue_work,
             question=question,
+            ui_artifact=ui_artifact,
             schema_actions=schema_actions,
         )
 

app/data/action/render_ui.py

@@ -0,0 +1,130 @@
+from agent_core import action
+
+
+@action(
+    name="render_ui",
+    description=(
+        "Create a self-contained HTML document with inline CSS/JavaScript in the output workspace. "
+        "Choose this when manipulating inputs, exploring alternatives, inspecting changing data or "
+        "tracking progress materially helps the user's goal; no explicit UI request is needed. "
+        "Prefer text when reading suffices; never render solely for decoration or topic. "
+        "Respect requested formats. "
+        "Use semantic, keyboard-usable HTML and addEventListener; every control must work. "
+        "No backend, npm/imports, external scripts, inline event handlers or secrets. "
+        "Never navigate or request permissions; preventDefault on form submits. Other resources "
+        "are blocked by CSP. For API fetches declare exact public HTTPS origins in connect_origins "
+        "(omit offline); require credential-free browser CORS, loading/recoverable errors, actual "
+        "data timestamps and verified units. "
+        "Restore window.craftbot.state; call window.craftbot.saveState(object) on changes. "
+        "State is JSON: store timer deadlines, not interval counts; serialize timestamps and "
+        "reconstruct Dates. To revise, pass artifact_id plus complete replacement HTML; "
+        "preserve state keys and normalize obsolete saved enum values. You cannot inspect or "
+        "change saved preview state through this action. "
+        "Design around the main activity: useful defaults, immediate feedback, a strong focal "
+        "value/result and one clear primary action. Keep labeled controls/units near their results; "
+        "make selection and progress obvious. Use open sections, whitespace and subtle dividers "
+        "instead of nested bordered cards or generic dashboards. Keep advanced settings/tips "
+        "behind disclosures and give empty/error states actionable next steps. "
+        "Use 15–16px body text, deliberate hierarchy and restrained accents. For live host "
+        "light/dark styling use --cb-bg, --cb-surface, --cb-text, --cb-muted, --cb-border, "
+        "--cb-accent and --cb-font; bespoke palettes must consistently style page/text/controls "
+        "without host-token fallbacks overriding them. "
+        "Fit a full-height, resizable 440–900px canvas and usable 360px layout using document-width "
+        "responsive CSS. The host supplies title, phone/fullscreen/revision controls: don't duplicate "
+        "them, add faux browser chrome or repeat the title with an intro. "
+        "Use purposeful 150–250ms opacity/transform transitions; honor prefers-reduced-motion. "
+        "No perpetual decoration, ticking-number animation or delayed controls. "
+        "Keep code concise with shared styles/functions so the complete document fits one response. "
+        "After rendering, send at most one brief usage sentence; don't re-ask for values adjustable "
+        "in the interface."
+    ),
+    default=True,
+    action_sets=["core"],
+    parallelizable=False,
+    input_schema={
+        "title": {
+            "type": "string",
+            "description": "Short interface title (1–120 characters).",
+        },
+        "html": {
+            "type": "string",
+            "description": "Complete HTML/CSS/JS, at most 512 KB.",
+        },
+        "artifact_id": {
+            "type": "string",
+            "description": "Optional ID returned by an earlier render_ui in this session; omit for a new interface.",
+        },
+        "connect_origins": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": "Optional list of at most 8 exact public HTTPS API origins (default HTTPS port). No paths, query strings, credentials, IP addresses, local hosts or wildcards. Omit for offline output.",
+        },
+    },
+    output_schema={
+        "status": {"type": "string"},
+        "artifact_id": {"type": "string"},
+        "revision": {"type": "integer"},
+    },
+    test_payload={
+        "title": "Counter",
+        "html": "<button>Count</button>",
+        "simulated_mode": True,
+    },
+)
+async def render_ui(input_data: dict) -> dict:
+    # CraftBot executes the serialized function body in a fresh namespace.
+    import asyncio
+
+    from agent_core.core.session import MAIN_SESSION_ID
+    from app.generative_ui import make_artifact
+
+    try:
+        artifact = make_artifact(input_data)
+    except ValueError as exc:
+        return {"status": "error", "message": str(exc)}
+
+    if not input_data.get("simulated_mode", False):
+        from app.internal_action_interface import InternalActionInterface as iai
+
+        session_id = input_data.get("_session_id") or MAIN_SESSION_ID
+        if input_data.get("artifact_id"):
+            streams = iai.state_manager.event_stream_manager
+            if not streams.has_stream(session_id):
+                return {
+                    "status": "error",
+                    "message": "This session has no UI artifact stream.",
+                }
+            stream = streams.get_stream_by_id(session_id)
+            previous = [
+                event.ui_artifact
+                for event in stream.as_list()
+                if event.ui_artifact and event.ui_artifact.get("id") == artifact["id"]
+            ]
+            if not previous:
+                from app.usage.chat_storage import get_chat_storage
+
+                stored = await asyncio.to_thread(
+                    get_chat_storage().get_latest_ui_artifact,
+                    session_id,
+                    artifact["id"],
+                )
+                if stored:
+                    previous = [stored]
+            if not previous:
+                return {
+                    "status": "error",
+                    "message": "No artifact with this ID exists in this session.",
+                }
+            artifact["revision"] = max(a["revision"] for a in previous) + 1
+        await iai.do_chat(
+            artifact["title"],
+            platform="CraftBot Interface",
+            session_id=session_id,
+            continue_work=True,
+            ui_artifact=artifact,
+        )
+    return {
+        "status": "success",
+        "artifact_id": artifact["id"],
+        "revision": artifact["revision"],
+    }

app/generative_ui.py

@@ -0,0 +1,73 @@
+"""The frontend-only artifact contract. Browser isolation lives in the renderer."""
+
+import re
+import uuid
+
+MAX_HTML_BYTES = 512_000
+MAX_CONNECT_ORIGINS = 8
+
+
+def normalize_connect_origins(value: object) -> list[str]:
+    """Accept exact public HTTPS origins, never CSP expressions or local hosts."""
+    if not isinstance(value, list) or len(value) > MAX_CONNECT_ORIGINS:
+        raise ValueError("connect_origins must be a list of at most 8 HTTPS origins.")
+    origins = set()
+    for origin in value:
+        match = (
+            re.fullmatch(r"https://([A-Za-z0-9.-]+)(?::443)?/?", origin)
+            if isinstance(origin, str)
+            else None
+        )
+        host = match[1].lower() if match else ""
+        labels = host.split(".")
+        if (
+            len(host) > 253
+            or len(labels) < 2
+            or not re.fullmatch(r"[a-z]{2,63}", labels[-1])
+            or any(
+                not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label)
+                for label in labels
+            )
+            or labels[-1]
+            in {
+                "localhost",
+                "local",
+                "internal",
+                "lan",
+                "home",
+                "corp",
+                "arpa",
+                "onion",
+                "test",
+                "invalid",
+            }
+        ):
+            raise ValueError(
+                "connect_origins must contain exact public HTTPS origins, without paths, credentials or wildcards."
+            )
+        origins.add(f"https://{host}")
+    return sorted(origins)
+
+
+def make_artifact(data: dict, revision: int = 1) -> dict:
+    title = data.get("title")
+    html = data.get("html")
+    artifact_id = data.get("artifact_id") or uuid.uuid4().hex
+    connect_origins = normalize_connect_origins(data.get("connect_origins", []))
+    if not isinstance(title, str) or not 1 <= len(title.strip()) <= 120:
+        raise ValueError("title must contain 1–120 characters.")
+    if not isinstance(html, str) or not html.strip():
+        raise ValueError("html must be a complete HTML document.")
+    if len(html.encode("utf-8")) > MAX_HTML_BYTES:
+        raise ValueError(f"html exceeds {MAX_HTML_BYTES} bytes.")
+    if not isinstance(artifact_id, str) or not re.fullmatch(
+        r"[A-Za-z0-9_-]{1,80}", artifact_id
+    ):
+        raise ValueError("artifact_id must contain 1–80 letters, digits, _ or -.")
+    return {
+        "id": artifact_id,
+        "title": title.strip(),
+        "html": html,
+        "revision": revision,
+        "connect_origins": connect_origins,
+    }

app/internal_action_interface.py

@@ -390,6 +390,7 @@ async def do_chat(
         continue_work: bool = False,
         suggested_responses: Optional[List[str]] = None,
         allow_free_text: bool = True,
+        ui_artifact: Optional[dict] = None,
     ) -> None:
         """Record an agent-authored chat message to the event stream.
 
@@ -421,6 +422,7 @@ async def do_chat(
             continue_work=continue_work,
             suggested_responses=suggested_responses,
             allow_free_text=allow_free_text,
+            ui_artifact=ui_artifact,
         )
 
     @staticmethod

app/state/state_manager.py

@@ -138,6 +138,7 @@ def record_agent_message(
         continue_work: bool = False,
         suggested_responses: Optional[List[str]] = None,
         allow_free_text: bool = True,
+        ui_artifact: Optional[dict] = None,
     ) -> None:
         """Record an agent message to a session's event stream.
 
@@ -182,6 +183,7 @@ def record_agent_message(
             platform=platform,
             continue_work=continue_work,
             question=question,
+            ui_artifact=ui_artifact,
             task_id=target,
         )
 

app/ui_layer/adapters/base.py

@@ -312,6 +312,7 @@ def _handle_agent_message(self, event: UIEvent) -> None:
                 continue_work=bool(event.data.get("continue_work", False)),
                 is_question=is_question,
                 allow_free_text=allow_free_text,
+                ui_artifact=event.data.get("ui_artifact"),
             )
         )
 
@@ -447,6 +448,7 @@ async def _display_chat_message(
         is_question: bool = False,
         allow_free_text: bool = True,
         details: Optional[str] = None,
+        ui_artifact: Optional[dict] = None,
     ) -> None:
         """
         Display a chat message.
@@ -485,6 +487,7 @@ async def _display_chat_message(
                 # bubble must not add the "Please select a response" banner.
                 requires_choice=not is_question,
                 details=details,
+                ui_artifact=ui_artifact,
             )
         )
 

app/ui_layer/adapters/browser_adapter.py

@@ -342,6 +342,7 @@ def _stored_to_chat_message(stored) -> ChatMessage:
             allow_free_text=stored.allow_free_text,
             requires_choice=not stored.is_question,
             details=stored.details,
+            ui_artifact=stored.ui_artifact,
         )
 
     async def append_message(self, message: ChatMessage) -> None:
@@ -386,6 +387,7 @@ async def append_message(self, message: ChatMessage) -> None:
                     is_question=message.is_question,
                     allow_free_text=message.allow_free_text,
                     details=message.details,
+                    ui_artifact=message.ui_artifact,
                 )
                 await self.run_storage(self._storage.insert_message, stored)
             except Exception: