feat: Mini Browser — a live Chromium that agents drive and users can take over

.dockerignore

@@ -45,6 +45,10 @@ app/config/settings.json
 app/config/mcp_config.json
 app/config/skills_config.json
 app/config/scheduler_config.json
+# Mini Browser's Chromium profile: cookies and signed-in sessions, plus its
+# lock file (its password vault is in .credentials, above).
+**/mini_browser_profile
+**/mini_browser_profile.lock
 
 # Per-install agent state
 app/data/.usage

.gitignore

@@ -85,3 +85,9 @@ npm-cache/
 playwright-browsers/
 
 hf-cache/
+
+# Mini Browser's persistent Chromium profile (cookies, signed-in sessions)
+# and its cross-process lock file. Its password vault lives in .credentials/
+# (ignored above). Never commit.
+**/mini_browser_profile/
+**/mini_browser_profile.lock

agent_core/core/impl/action/router.py

@@ -57,6 +57,7 @@ def __init__(
         action_library: ActionLibraryProtocol,
         llm_interface: LLMInterfaceProtocol,
         context_engine: ContextEngineProtocol,
+        prompt_filter: Optional[Callable[[str], str]] = None,
     ):
         """
         Initialize the router responsible for selecting or creating actions.
@@ -65,10 +66,25 @@ def __init__(
             action_library: Repository for storing and retrieving action definitions.
             llm_interface: LLM client used to reason about which action to run.
             context_engine: Provider of system prompts and context formatting.
+            prompt_filter: Optional host hook applied to every rendered
+                action-selection prompt, e.g. to drop a capability rule while
+                that capability is unavailable. A failing filter is skipped.
         """
         self.action_library = action_library
         self.llm_interface = llm_interface
         self.context_engine = context_engine
+        self.prompt_filter = prompt_filter
+
+    def _filter_prompt(self, prompt: str) -> str:
+        """``prompt`` through the host's prompt_filter (unchanged on failure)."""
+        if self.prompt_filter is None:
+            return prompt
+        try:
+            filtered = self.prompt_filter(prompt)
+        except Exception as e:
+            logger.debug(f"[ACTION] prompt filter failed: {e}")
+            return prompt
+        return filtered if isinstance(filtered, str) else prompt
 
     @profile("action_router_select_action", OperationCategory.ACTION_ROUTING)
     async def select_action_in_session(
@@ -127,26 +143,30 @@ async def select_action_in_session(
             integration_essentials = ""
 
         decision_prompt_name = "SELECT_ACTION"
-        static_prompt = SELECT_ACTION_PROMPT.format(
-            session_state=session_state,
-            event_stream="",  # Empty for static prompt
-            query=query,
-            action_candidates=self._format_candidates(action_candidates),
-            integration_essentials=integration_essentials,
+        static_prompt = self._filter_prompt(
+            SELECT_ACTION_PROMPT.format(
+                session_state=session_state,
+                event_stream="",  # Empty for static prompt
+                query=query,
+                action_candidates=self._format_candidates(action_candidates),
+                integration_essentials=integration_essentials,
+            )
         )
         candidates_text = self._format_candidates(action_candidates)
 
         def render_prompt() -> str:
             # Rendered from the CURRENT stream, so a fold made while deciding
             # (see _prompt_for_decision) is reflected in what is sent.
-            return SELECT_ACTION_PROMPT.format(
-                session_state=session_state,
-                event_stream=self.context_engine.get_event_stream(
-                    session_id=session_id
-                ),
-                query=query,
-                action_candidates=candidates_text,
-                integration_essentials=integration_essentials,
+            return self._filter_prompt(
+                SELECT_ACTION_PROMPT.format(
+                    session_state=session_state,
+                    event_stream=self.context_engine.get_event_stream(
+                        session_id=session_id
+                    ),
+                    query=query,
+                    action_candidates=candidates_text,
+                    integration_essentials=integration_essentials,
+                )
             )
 
         full_prompt = render_prompt()

agent_core/core/impl/session/manager.py

@@ -2,10 +2,10 @@
 """
 Shared SessionManager for agent_core.
 
-Owns the registry of persistent sessions (main / chat / agent_app), their
-loaded capabilities (action sets + skills), todos, run budgets, workspace
-directories, and their LLM session caches. Runtime-specific behavior is
-injected via hooks:
+Owns the registry of persistent sessions (main / chat / agent_app /
+mini_browser), their loaded capabilities (action sets + skills), todos, run
+budgets, workspace directories, and their LLM session caches.
+Runtime-specific behavior is injected via hooks:
 
 State hooks:
 - get_agent_property / set_agent_property: session-scoped state access
@@ -103,16 +103,22 @@ def main(self) -> Optional[Session]:
         return self.sessions.get(MAIN_SESSION_ID)
 
     def list_sessions(self, include_archived: bool = False) -> List[Session]:
-        """All sessions: main first, then agent_app, then chats newest-first."""
+        """All sessions: main first, then the Mini Browser session, then
+        agent_app, then chats — newest-first within each type."""
         sessions = [
             s for s in self.sessions.values() if include_archived or not s.archived
         ]
 
-        type_rank = {SessionType.MAIN: 0, SessionType.AGENT_APP: 1, SessionType.CHAT: 2}
+        type_rank = {
+            SessionType.MAIN: 0,
+            SessionType.MINI_BROWSER: 1,
+            SessionType.AGENT_APP: 2,
+            SessionType.CHAT: 3,
+        }
 
         # Newest-first within each type bucket (two-pass stable sort)
         sessions.sort(key=lambda s: s.last_active_at, reverse=True)
-        sessions.sort(key=lambda s: type_rank.get(s.type, 3))
+        sessions.sort(key=lambda s: type_rank.get(s.type, len(type_rank)))
         return sessions
 
     # ─────────────────────── Creation ─────────────────────────────────────────
@@ -142,9 +148,10 @@ def create_session(
         Create a new persistent session.
 
         Args:
-            session_type: main | chat | agent_app.
+            session_type: main | chat | agent_app | mini_browser.
             title: Sidebar title ("New chat" placeholder until auto-titled).
-            session_id: Explicit id (main / agent-app); random hex otherwise.
+            session_id: Explicit id (main / agent-app / mini-browser); random
+                hex otherwise.
             action_sets: Extra action sets to load on top of core.
             selected_skills: Skills to preload (slash-command entry, Agent App).
             agent_app_project_id: Backing project for agent_app sessions.

agent_core/core/prompts/action.py

@@ -92,6 +92,12 @@
   needed only for the USER'S OWN account (their Gmail, their Slack), never
   for public data. NEVER tell the user you cannot do something that
   http_request can do.
+- Websites you must USE, not just read (log in, click through, fill or
+  submit forms, search inside a site, shop or book, the user's account on a
+  site with no connected integration, pages web_fetch could not read) go
+  through the Mini Browser: load the 'mini_browser' action set (a real
+  browser with the user's saved logins, which the user can watch). Reading
+  public pages stays web_search / web_fetch.
 - Your system prompt contains a Capability Catalog of every action set and
   skill available. Only your session's loaded sets are in <actions> below.
 - Need a capability that isn't loaded (documents, images, an integration,
@@ -305,6 +311,21 @@
 {integration_essentials}
 """
 
+# The Mini Browser bullet of SELECT_ACTION_PROMPT's capability rules, verbatim.
+# The host application removes it from the rendered prompt (the ActionRouter's
+# ``prompt_filter`` hook) while the browser is not usable: steering every
+# agent to an action set that is not registered here, or to a browser that
+# cannot launch here, wastes turns on every matching task.
+MINI_BROWSER_RULE = """\
+- Websites you must USE, not just read (log in, click through, fill or
+  submit forms, search inside a site, shop or book, the user's account on a
+  site with no connected integration, pages web_fetch could not read) go
+  through the Mini Browser: load the 'mini_browser' action set (a real
+  browser with the user's saved logins, which the user can watch). Reading
+  public pages stays web_search / web_fetch.
+"""
+
 __all__ = [
+    "MINI_BROWSER_RULE",
     "SELECT_ACTION_PROMPT",
 ]

agent_core/core/prompts/context.py

@@ -15,7 +15,7 @@
 AGENT_INFO_PROMPT = """
 <context>
 You are a highly capable proactive and general AI agent that can perform virtually ANY computer-based task (until proven can't). Your capabilities include:
-- Full control over a virtual machine (CLI commands, browser automation)
+- Full control over a virtual machine (CLI commands, a live Mini Browser for using websites)
 - Full access to the operating system, file system, and internet
 - Your own persistent agent file system for memory, notes, and workspace files
 - Access to MCP (Model Context Protocol) tools that extend your abilities
@@ -56,7 +56,7 @@
 
 Adaptive Execution:
 - If you lack information during execution, STOP and go back to collect more
-- Before replying "I don't know", "I can't do that", or reaching for generic web search: check what you ALREADY have — stored memory, connected integrations, and your Agent App apps often hold the answer or the capability
+- Before replying "I don't know", "I can't do that", or reaching for generic web search: check what you ALREADY have — stored memory, connected integrations, your Agent App apps, and the Mini Browser (websites the user is signed in to) often hold the answer or the capability
 - If verification fails, analyze why and either re-execute or gather more info
 - Never assume work is done without verification
 </working_ethic>

agent_core/core/session/session.py

@@ -25,8 +25,12 @@ class SessionType:
     MAIN = "main"
     CHAT = "chat"
     AGENT_APP = "agent_app"
+    # The dedicated chat behind the Mini Browser page (app/mini_browser). It
+    # behaves like a chat, but the backend creates it (browser actions
+    # preloaded) and it is not listed with the sidebar's chats.
+    MINI_BROWSER = "mini_browser"
 
-    ALL = (MAIN, CHAT, AGENT_APP)
+    ALL = (MAIN, CHAT, AGENT_APP, MINI_BROWSER)
 
 
 # The singleton main session id. All ambient input (integrations, scheduler,
@@ -41,7 +45,7 @@ class Session:
 
     Attributes:
         id: Unique identifier (``main`` for the main session).
-        type: One of SessionType.ALL — main | chat | agent_app.
+        type: One of SessionType.ALL — main | chat | agent_app | mini_browser.
         title: Human-readable title shown in the sidebar (auto-generated
             for chat sessions after the first exchange, renamable).
         created_at: ISO timestamp when the session was created.